From 15f3dfded48e15924a9562d07fa7b30dccdf0e41 Mon Sep 17 00:00:00 2001 From: Jonas Date: Thu, 4 May 2023 15:07:55 +0200 Subject: [PATCH 001/131] upload files --- .../git-patches/.gitignore | 2 + .../git-patches/git-patches.py | 45 ++++++++++ .../strip-body/.gitignore | 2 + .../strip-body/RemoveFunctionBodyCheck.cpp | 85 +++++++++++++++++++ .../strip-body/RemoveFunctionBodyCheck.h | 23 +++++ .../strip-body/strip-body.py | 25 ++++++ .../strip-body/test-orginal.c | 53 ++++++++++++ 7 files changed, 235 insertions(+) create mode 100644 scripts/incremental-test-generation/git-patches/.gitignore create mode 100644 scripts/incremental-test-generation/git-patches/git-patches.py create mode 100644 scripts/incremental-test-generation/strip-body/.gitignore create mode 100644 scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp create mode 100644 scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h create mode 100644 scripts/incremental-test-generation/strip-body/strip-body.py create mode 100644 scripts/incremental-test-generation/strip-body/test-orginal.c diff --git a/scripts/incremental-test-generation/git-patches/.gitignore b/scripts/incremental-test-generation/git-patches/.gitignore new file mode 100644 index 0000000000..a0db2c2466 --- /dev/null +++ b/scripts/incremental-test-generation/git-patches/.gitignore @@ -0,0 +1,2 @@ +testGit/* +testGitDif/* \ No newline at end of file diff --git a/scripts/incremental-test-generation/git-patches/git-patches.py b/scripts/incremental-test-generation/git-patches/git-patches.py new file mode 100644 index 0000000000..31f6470eac --- /dev/null +++ b/scripts/incremental-test-generation/git-patches/git-patches.py @@ -0,0 +1,45 @@ +import os +import shutil +import subprocess +import tempfile + +script_location = os.path.dirname(__file__) +repoURL = 'https://github.com/cmatsuoka/figlet.git' +repoPath = os.path.join(script_location, '/testGit') +repoDifPath = os.path.join(script_location, '/testGitDif') +paths = [repoPath, repoDifPath] + +# Clear the directories +for path in paths: + if os.path.exists(path): + shutil.rmtree(path) + +# Clone the repository from repo URL into the folder ./testGit +print("Repo Path: " + repoPath) +subprocess.run(['git', 'clone', repoURL, repoPath]) + +# Change the working directory to the repository directory +os.chdir(repoPath) + +# Get a list of commit hashes using git log +output = subprocess.run(["git", "log", "--reverse", "--format=%H"], capture_output=True, text=True) +commit_hashes = output.stdout.strip().split('\n') + +# Create the directory ./testGitDif if it does not exist +os.makedirs(repoDifPath, exist_ok=True) + +# Create a diff file for each commit and store them in ./testGitDif +for i, commit_hash in enumerate(commit_hashes): + # Generate the diff using git diff + diff_output = subprocess.run(['git', 'diff', commit_hash + '^', commit_hash], capture_output=True, text=True) + diff = diff_output.stdout.strip() + + # Write the diff to a file in the repoDifPath + diff_path = os.path.join(repoDifPath, '{:06d}_{}.diff'.format(i, commit_hash)) + with open(diff_path, 'w') as f: + f.write(diff) + + # Update the paths to account for the change in working directory + repoPath = os.path.abspath(repoPath) + repoDifPath = os.path.abspath(repoDifPath) + diff --git a/scripts/incremental-test-generation/strip-body/.gitignore b/scripts/incremental-test-generation/strip-body/.gitignore new file mode 100644 index 0000000000..d419251dfd --- /dev/null +++ b/scripts/incremental-test-generation/strip-body/.gitignore @@ -0,0 +1,2 @@ +test.c +test-origninal.c \ No newline at end of file diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp new file mode 100644 index 0000000000..8e419af4ea --- /dev/null +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -0,0 +1,85 @@ +/* + +• sudo apt install ninja-build +• sudo apt install ccache +• sudo apt install lld +• "git clone https://github.com/llvm/llvm-project.git" +• Move to directory "llvm-project/clang-tools-extra" +• Run "clang-tidy/add_new_check.py readability remove-function-body" +• Replace "./clang-tidy/readability/RemoveFunctionBodyCheck.cpp" with the content of this file +• Update "./clang-tidy/readability/RemoveFunctionBodyCheck.h" with the the corresponding file in this directory +• "cd .." to llvm-project/ +• "mkdir build" +• "cd build" +• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_PARALLEL_LINK_JOBS=1 ../llvm" + Additional Options: -DLLVM_PARALLEL_COMPILE_JOBS=2 +• "ninja all" +• Use "bin/clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" + For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName' or 'functionName1, functionName2' to specify the target functions + +*/ + +#include "RemoveFunctionBodyCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { + if (RemoveOnlyFunctionName.empty()) { + Finder->addMatcher(functionDecl(isDefinition()).bind("remove_function_body"), this); + } else { + std::vector names; + std::istringstream iss(RemoveOnlyFunctionName); + for (std::string s; std::getline(iss, s, ','); ) { + // trim leading and trailing whitespace from function name + s.erase(0, s.find_first_not_of(" ")); + s.erase(s.find_last_not_of(" ") + 1); + if (!s.empty()) { + names.push_back(s); + } + } + for (const auto& name : names) { + Finder->addMatcher(functionDecl(hasName(name), isDefinition()).bind("remove_function_body"), this); + } + } +} + +void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { + auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); + // Remove the function body + std::string Replacement = " {\n\t// Stripped function of its body\n"; + const auto ReturnType = MatchedDecl->getReturnType(); + if (!ReturnType->isVoidType()) { + Replacement += "\treturn "; + if (ReturnType->isPointerType() || ReturnType->isNullPtrType()) { + Replacement += "0"; + } else if (ReturnType->isIntegralType(*Result.Context) || ReturnType->isCharType()) { + Replacement += "0"; + } else if (ReturnType->isFloatingType()) { + Replacement += "0.0"; + } else if (const RecordType *RT = ReturnType->getAsStructureType()) { + Replacement += "(struct " + RT->getDecl()->getNameAsString() + "){}"; + } else if (const RecordType *RT = ReturnType->getAsUnionType()) { + Replacement += "(union " + RT->getDecl()->getNameAsString() + "){}"; + } else { + Replacement += "/* TODO: Add generic return value for " + ReturnType.getAsString() + " */"; + } + Replacement += ";\n"; + } + SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); + SourceLocation End = MatchedDecl->getBodyRBrace(); + auto Range = CharSourceRange::getCharRange(Start, End); + diag(Start, "Function %0 has been stripped of its body") + << MatchedDecl + << FixItHint::CreateReplacement(Range, Replacement); +} + +void RemoveFunctionBodyCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "RemoveOnlyFunctionName", RemoveOnlyFunctionName); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h new file mode 100644 index 0000000000..cfc808448d --- /dev/null +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h @@ -0,0 +1,23 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Removes the body of a function and adds when needed generic return statements. +class RemoveFunctionBodyCheck : public ClangTidyCheck { + const std::string RemoveOnlyFunctionName; + +public: + RemoveFunctionBodyCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + RemoveOnlyFunctionName(Options.get("RemoveOnlyFunctionName", "")){} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H diff --git a/scripts/incremental-test-generation/strip-body/strip-body.py b/scripts/incremental-test-generation/strip-body/strip-body.py new file mode 100644 index 0000000000..9f33432630 --- /dev/null +++ b/scripts/incremental-test-generation/strip-body/strip-body.py @@ -0,0 +1,25 @@ +import os + +# Change these variables to match your setup +CLANG_PATH = "~/BA/Clang/llvm-project/build/bin" +CHECKS_PATH = "~/BA/removeBody.so" + +# The name of the file to check +file_name = "test.c" + +# Build the clang command to run the removeBody check +clang_command = [ + "clang", + "-Xclang", + "-load", + "-Xclang", + CHECKS_PATH, + "-Xclang", + "-add-plugin", + "-Xclang", + "removeBody", + file_name, +] + +# Run the command +os.system(" ".join(clang_command)) \ No newline at end of file diff --git a/scripts/incremental-test-generation/strip-body/test-orginal.c b/scripts/incremental-test-generation/strip-body/test-orginal.c new file mode 100644 index 0000000000..550bee2360 --- /dev/null +++ b/scripts/incremental-test-generation/strip-body/test-orginal.c @@ -0,0 +1,53 @@ +#include +#include + +typedef struct { + int x; + int y; +} Point; + +typedef union { + int i; + float f; +} IntOrFloat; + +typedef enum { + RED, + GREEN, + BLUE +} Color; + +Point* create_point(int x, int y) { + Point* p = malloc(sizeof(Point)); + p->x = x; + p->y = y; + return p; +} + +IntOrFloat* create_int_or_float(int i, float f) { + IntOrFloat* io = malloc(sizeof(IntOrFloat)); + io->i = i; + io->f = f; + return io; +} + +Color get_color() { + return BLUE; +} + +int main() { + Point* p = create_point(10, 20); + printf("Point: (%d, %d)\n", p->x, p->y); + + IntOrFloat* io = create_int_or_float(42, 3.14f); + printf("Int or float: %d, %f\n", io->i, io->f); + + Color c = get_color(); + printf("Color: %d\n", c); + + free(p); + free(io); + + return 0; +} + From 9de3f7afbab4dc7adfac35bce46d1e00db66532e Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 4 May 2023 15:18:17 +0200 Subject: [PATCH 002/131] sync --- scripts/incremental-test-generation/strip-body/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/strip-body/.gitignore b/scripts/incremental-test-generation/strip-body/.gitignore index d419251dfd..d3c4e87b2f 100644 --- a/scripts/incremental-test-generation/strip-body/.gitignore +++ b/scripts/incremental-test-generation/strip-body/.gitignore @@ -1,2 +1,2 @@ test.c -test-origninal.c \ No newline at end of file +test-original.c \ No newline at end of file From e8cf76ce9fcc6df96fba035b69f2643ad6856dec Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 4 May 2023 15:19:47 +0200 Subject: [PATCH 003/131] update gitignore --- .../strip-body/.gitignore | 2 +- .../strip-body/test-orginal.c | 53 ------------------- 2 files changed, 1 insertion(+), 54 deletions(-) delete mode 100644 scripts/incremental-test-generation/strip-body/test-orginal.c diff --git a/scripts/incremental-test-generation/strip-body/.gitignore b/scripts/incremental-test-generation/strip-body/.gitignore index d3c4e87b2f..4f68c87b67 100644 --- a/scripts/incremental-test-generation/strip-body/.gitignore +++ b/scripts/incremental-test-generation/strip-body/.gitignore @@ -1,2 +1,2 @@ test.c -test-original.c \ No newline at end of file +test-clean.c \ No newline at end of file diff --git a/scripts/incremental-test-generation/strip-body/test-orginal.c b/scripts/incremental-test-generation/strip-body/test-orginal.c deleted file mode 100644 index 550bee2360..0000000000 --- a/scripts/incremental-test-generation/strip-body/test-orginal.c +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include - -typedef struct { - int x; - int y; -} Point; - -typedef union { - int i; - float f; -} IntOrFloat; - -typedef enum { - RED, - GREEN, - BLUE -} Color; - -Point* create_point(int x, int y) { - Point* p = malloc(sizeof(Point)); - p->x = x; - p->y = y; - return p; -} - -IntOrFloat* create_int_or_float(int i, float f) { - IntOrFloat* io = malloc(sizeof(IntOrFloat)); - io->i = i; - io->f = f; - return io; -} - -Color get_color() { - return BLUE; -} - -int main() { - Point* p = create_point(10, 20); - printf("Point: (%d, %d)\n", p->x, p->y); - - IntOrFloat* io = create_int_or_float(42, 3.14f); - printf("Int or float: %d, %f\n", io->i, io->f); - - Color c = get_color(); - printf("Color: %d\n", c); - - free(p); - free(io); - - return 0; -} - From 5b03ce86e6944f42527f059f290564f8c86d8ec1 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Fri, 5 May 2023 12:58:04 +0200 Subject: [PATCH 004/131] update build instructions --- .../strip-body/RemoveFunctionBodyCheck.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp index 8e419af4ea..7864fa1f68 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -13,8 +13,8 @@ • "cd build" • "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_PARALLEL_LINK_JOBS=1 ../llvm" Additional Options: -DLLVM_PARALLEL_COMPILE_JOBS=2 -• "ninja all" -• Use "bin/clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" +• "sudo ninja install" +• Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName' or 'functionName1, functionName2' to specify the target functions */ From 6091f1d6dd5caf062fafa91d09c7d237584735cf Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 5 May 2023 14:41:52 +0200 Subject: [PATCH 005/131] Adjust evalAssert to offer __goblint_check(exp) with the option trans.goblint-check --- src/transform/evalAssert.ml | 7 +++++-- src/util/options.schema.json | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index a86b12f7fa..b0d4969a52 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -25,8 +25,11 @@ module EvalAssert = struct (* should asserts be surrounded by __VERIFIER_atomic_{begin,end}? *) let surroundByAtomic = true + (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp)*) + let goblintCheck = GobConfig.get_bool "trans.goblint-check" + (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) - let ass = makeVarinfo true "__VERIFIER_assert" (TVoid []) + let ass = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) let atomicBegin = makeVarinfo true "__VERIFIER_atomic_begin" (TVoid []) let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) @@ -59,7 +62,7 @@ module EvalAssert = struct | `Lifted e -> let es = WitnessUtil.InvariantExp.process_exp e in let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv ass); ("exp", Fe e)]) es in - if surroundByAtomic then + if surroundByAtomic && not goblintCheck then let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in abegin :: (asserts @ [aend]) diff --git a/src/util/options.schema.json b/src/util/options.schema.json index 2ff2e8bf58..620721c03e 100644 --- a/src/util/options.schema.json +++ b/src/util/options.schema.json @@ -1461,6 +1461,12 @@ "description": "Output filename for transformations that output a transformed file.", "type":"string", "default": "transformed.c" + }, + "goblint-check" : { + "title": "trans.goblint-check", + "description": "Write __Goblint_Check(exp) in the file instead of __VERIFIER_assert(exp).", + "type": "boolean", + "default": false } }, "additionalProperties": false From 11113314cfba3cf1c4bff5e592fe9074306447e9 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Fri, 5 May 2023 15:30:48 +0200 Subject: [PATCH 006/131] Change number of jobs to improve build speed --- .../strip-body/RemoveFunctionBodyCheck.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp index 7864fa1f68..59655c49df 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -11,8 +11,8 @@ • "cd .." to llvm-project/ • "mkdir build" • "cd build" -• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_PARALLEL_LINK_JOBS=1 ../llvm" - Additional Options: -DLLVM_PARALLEL_COMPILE_JOBS=2 +• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_PARALLEL_COMPILE_JOBS=8 -DLLVM_PARALLEL_LINK_JOBS=8 ../llvm" + You can vary the linker and compiler jobs depending on your memory to avoid swapping. You can see the free memory with "free -h --giga" • "sudo ninja install" • Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName' or 'functionName1, functionName2' to specify the target functions From 2ea12fcdd604429d4291be2c2fea9c87aa091a98 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 5 May 2023 16:33:36 +0200 Subject: [PATCH 007/131] Add build target option for further speedup --- .../strip-body/RemoveFunctionBodyCheck.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp index 59655c49df..939f49b826 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -11,8 +11,10 @@ • "cd .." to llvm-project/ • "mkdir build" • "cd build" -• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_PARALLEL_COMPILE_JOBS=8 -DLLVM_PARALLEL_LINK_JOBS=8 ../llvm" - You can vary the linker and compiler jobs depending on your memory to avoid swapping. You can see the free memory with "free -h --giga" +• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_PARALLEL_COMPILE_JOBS=5 -DLLVM_PARALLEL_LINK_JOBS=5 ../llvm" + Make sure to set the right build target (AMDGPU, ARM, AVR, BPF, Hexagon, Lanai, LoongArch, Mips, MSP430, NVPTX, PowerPC, RISCV, Sparc, SystemZ, VE, WebAssembly, X86, XCore) + You can vary the linker and compiler jobs depending on your memory and CPU to improve the performance for "sudo ninja install" + You can see the free memory with "free -h --giga" • "sudo ninja install" • Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName' or 'functionName1, functionName2' to specify the target functions From a0ca99c8202c62ed48b1cf76060c2ebca458b6f5 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 9 May 2023 23:20:55 +0200 Subject: [PATCH 008/131] Adding index to matched functionDecl --- .../strip-body/RemoveFunctionBodyCheck.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp index 939f49b826..ae325f15c0 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -51,6 +51,18 @@ void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { } void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { + void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { + const FunctionDecl *FD = Result.Nodes.getNodeAs("functionDecl"); + const std::vector &Decls = Result.Nodes.getNodeAsAll("remove_function_body"); // Filter by binding + int index = -1; + for (int i = 0; i < Decls.size(); i++) { + if (FD == Decls[i]) { + index = i; + break; + } + } + assert(index >= 0 && "Matched function not found in translation unit"); + auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); // Remove the function body std::string Replacement = " {\n\t// Stripped function of its body\n"; @@ -75,8 +87,8 @@ void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); SourceLocation End = MatchedDecl->getBodyRBrace(); auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "Function %0 has been stripped of its body") - << MatchedDecl + diag(Start, "Function %0 with index %1 has been stripped of its body") + << MatchedDecl << index << FixItHint::CreateReplacement(Range, Replacement); } From 922dd23fc1ded6e55061957c6a6c5a4937f0dca4 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 10 May 2023 19:24:34 +0200 Subject: [PATCH 009/131] Adopt evalAssert.ml so the option is checked correctly at runtime --- src/transform/evalAssert.ml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index b0d4969a52..c2b89982cf 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -25,11 +25,6 @@ module EvalAssert = struct (* should asserts be surrounded by __VERIFIER_atomic_{begin,end}? *) let surroundByAtomic = true - (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp)*) - let goblintCheck = GobConfig.get_bool "trans.goblint-check" - - (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) - let ass = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) let atomicBegin = makeVarinfo true "__VERIFIER_atomic_begin" (TVoid []) let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) @@ -40,6 +35,8 @@ module EvalAssert = struct (* TODO: handle witness.invariant.loop-head *) val emit_after_lock = GobConfig.get_bool "witness.invariant.after-lock" val emit_other = GobConfig.get_bool "witness.invariant.other" + (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp) *) + val goblintCheck = GobConfig.get_bool "trans.goblint-check" method! vstmt s = let is_lock exp args = @@ -52,6 +49,10 @@ module EvalAssert = struct | _ -> false in + (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) + (* Moved in method because Options can not be checked in the module *) + let ass = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) in + let make_assert ~node loc lval = let lvals = match lval with | None -> CilLval.Set.top () From 5f16ec1766242457993099427af2a84805326e74 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 16 May 2023 17:52:49 +0200 Subject: [PATCH 010/131] Index mutations with -line-filter option --- .../strip-body/RemoveFunctionBodyCheck.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp index ae325f15c0..61486f8c23 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp @@ -16,8 +16,9 @@ You can vary the linker and compiler jobs depending on your memory and CPU to improve the performance for "sudo ninja install" You can see the free memory with "free -h --giga" • "sudo ninja install" -• Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" test.c --" - For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName' or 'functionName1, functionName2' to specify the target functions +• Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" -line-filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]' test.c --" + For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName1, functionName2' to specify the target functions + Additionaly you can specify which lines should be considered with the -line-filter option. To find all possible lines run without -fix --fix-errors. */ @@ -51,18 +52,6 @@ void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { } void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { - void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { - const FunctionDecl *FD = Result.Nodes.getNodeAs("functionDecl"); - const std::vector &Decls = Result.Nodes.getNodeAsAll("remove_function_body"); // Filter by binding - int index = -1; - for (int i = 0; i < Decls.size(); i++) { - if (FD == Decls[i]) { - index = i; - break; - } - } - assert(index >= 0 && "Matched function not found in translation unit"); - auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); // Remove the function body std::string Replacement = " {\n\t// Stripped function of its body\n"; @@ -88,7 +77,7 @@ void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { SourceLocation End = MatchedDecl->getBodyRBrace(); auto Range = CharSourceRange::getCharRange(Start, End); diag(Start, "Function %0 with index %1 has been stripped of its body") - << MatchedDecl << index + << MatchedDecl << FixItHint::CreateReplacement(Range, Replacement); } From ad9f0d9fbc1180a067c227f0ef74d271baab3318 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 16 May 2023 18:00:31 +0200 Subject: [PATCH 011/131] Fix Goblint Option Retrieving by adding Unit to constant --- src/transform/evalAssert.ml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index c2b89982cf..28d400a0b6 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -25,6 +25,11 @@ module EvalAssert = struct (* should asserts be surrounded by __VERIFIER_atomic_{begin,end}? *) let surroundByAtomic = true + (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp) *) + let goblintCheck () = GobConfig.get_bool "trans.goblint-check" + + (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) + let ass () = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) in let atomicBegin = makeVarinfo true "__VERIFIER_atomic_begin" (TVoid []) let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) @@ -35,8 +40,6 @@ module EvalAssert = struct (* TODO: handle witness.invariant.loop-head *) val emit_after_lock = GobConfig.get_bool "witness.invariant.after-lock" val emit_other = GobConfig.get_bool "witness.invariant.other" - (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp) *) - val goblintCheck = GobConfig.get_bool "trans.goblint-check" method! vstmt s = let is_lock exp args = @@ -49,10 +52,6 @@ module EvalAssert = struct | _ -> false in - (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) - (* Moved in method because Options can not be checked in the module *) - let ass = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) in - let make_assert ~node loc lval = let lvals = match lval with | None -> CilLval.Set.top () From eba6d4660f7b68b82dc5f29a0e3efa96a625a1a6 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 12:58:55 +0200 Subject: [PATCH 012/131] Move Check Creation Info to md --- .../.gitignore | 0 .../clang-mutations/README.md | 70 +++++++++++++++++++ .../RemoveFunctionBodyCheck.cpp | 24 ------- .../RemoveFunctionBodyCheck.h | 0 .../strip-body.py | 0 5 files changed, 70 insertions(+), 24 deletions(-) rename scripts/incremental-test-generation/{strip-body => clang-mutations}/.gitignore (100%) create mode 100644 scripts/incremental-test-generation/clang-mutations/README.md rename scripts/incremental-test-generation/{strip-body => clang-mutations}/RemoveFunctionBodyCheck.cpp (61%) rename scripts/incremental-test-generation/{strip-body => clang-mutations}/RemoveFunctionBodyCheck.h (100%) rename scripts/incremental-test-generation/{strip-body => clang-mutations}/strip-body.py (100%) diff --git a/scripts/incremental-test-generation/strip-body/.gitignore b/scripts/incremental-test-generation/clang-mutations/.gitignore similarity index 100% rename from scripts/incremental-test-generation/strip-body/.gitignore rename to scripts/incremental-test-generation/clang-mutations/.gitignore diff --git a/scripts/incremental-test-generation/clang-mutations/README.md b/scripts/incremental-test-generation/clang-mutations/README.md new file mode 100644 index 0000000000..ab357886ee --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/README.md @@ -0,0 +1,70 @@ +# Create Clang-Tidy Checks for mutations + +In this document is described how you can create all the Clang-Tidy checks needed for generating the code mutations. + + +## Dependencies + +For building Clang you need to install some dependencies: + + - [ ] `sudo apt install ninja-build` + - [ ] `sudo apt install ccache` + - [ ] `sudo apt install lld` + +## Cloning the repository + + - [ ] There are two alternatives for getting the repository + +For creating all the checks by yourself clone the **Official Clang Repository**: +`git clone https://github.com/llvm/llvm-project.git` +Alternatively you can clone the **Fork** with all the additional checks ready: +`git clone https://github.com/J2000A/llvm-project.git` + +## Creating the checks + + - [ ] When you cloned the Official Clang Repository you need to add the checks. Otherwise you can skip this part. + - [ ] Move to the directory `llvm-project/clang-tools-extra` + +In this directory are the implementations of the checks with their corresponding `.cpp` and `.h` file. In the following we will use the **>>check-name<<** of each check. You can get it by the filename without the word "Check" at the end and changing the capital letters to small ones with a minus in front (e.g. the >>check-name<< of `RemoveFunctionBodyCheck.cpp` is `remove-function-body`). Repeat the following steps for all new checks. + + - [ ] Run `clang-tidy/add_new_check.py readability >>check-name<<` + - [ ] Replace `./clang-tidy/readability/>>check-name<<.cpp` with the implementation in this directory + - [ ] Replace `./clang-tidy/readability/>>check-name<<.h` with the header file in this directory + +Now you have added all the check we need for the mutations. + +## Build + +The first build can take a while (up to multiple hours). But you can increase the performance by changing the parallel compile and link jobs. For me using the value 5 for both of them got me the fastest results. When using too many jobs the memory becomes a bottleneck. You can check the memory status with `free -h --giga`. +Additionally you may need to change the build target. Avaiable targets are: AMDGPU, ARM, AVR, BPF, Hexagon, Lanai, LoongArch, Mips, MSP430, NVPTX, PowerPC, RISCV, Sparc, SystemZ, VE, WebAssembly, X86, XCore + + - [ ] Move to the directory `llvm-project/` + - [ ] `mkdir build && cd build` + - [ ] `cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_PARALLEL_COMPILE_JOBS=5 -DLLVM_PARALLEL_LINK_JOBS=5 ../llvm` + - [ ] `sudo ninja install` + +## Running Clang-Tidy + +We will use the **>>check-name<<** again as defined in "Creating the checks". + +**Example:** Create the mutation "remove function body" on a file "test.c" in lines "4" and "14" when the function name is "foo": +`clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'foo'}}" -line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]' test.c --` + +**The command consists of the following components:** + + - [ ] Clang-Tidy +`clang-tidy` The command itself. + + - [ ] General Options + `-checks=-*,>>check-name<<` Deactivating all checks except >>check-name<<. +`-fix --fix-errors` Applying the mutations. +`-line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]'` Apply the mutations only on line 4 and 14. + + - [ ] Special Options +`-config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'foo1, foo2'}}"` Special option for the **remove-function-body** check to only remove the function body of functions named foo1 and foo2. + + - [ ] Filename +`test.c --` The filename. + +## Workflow +First run the check without `-fix --fix-errors` to see where mutations are possible without applying them. Remember the lines where you actually want to apply the mutation. Make a copy of the input file that you will mutate. The run the check again with `-fix --fix-errors` and `-line filter=...` on the copied file to apply only specific mutations and not all at ones. \ No newline at end of file diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp similarity index 61% rename from scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp rename to scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp index 61486f8c23..26704d8829 100644 --- a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp @@ -1,27 +1,3 @@ -/* - -• sudo apt install ninja-build -• sudo apt install ccache -• sudo apt install lld -• "git clone https://github.com/llvm/llvm-project.git" -• Move to directory "llvm-project/clang-tools-extra" -• Run "clang-tidy/add_new_check.py readability remove-function-body" -• Replace "./clang-tidy/readability/RemoveFunctionBodyCheck.cpp" with the content of this file -• Update "./clang-tidy/readability/RemoveFunctionBodyCheck.h" with the the corresponding file in this directory -• "cd .." to llvm-project/ -• "mkdir build" -• "cd build" -• "cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_PARALLEL_COMPILE_JOBS=5 -DLLVM_PARALLEL_LINK_JOBS=5 ../llvm" - Make sure to set the right build target (AMDGPU, ARM, AVR, BPF, Hexagon, Lanai, LoongArch, Mips, MSP430, NVPTX, PowerPC, RISCV, Sparc, SystemZ, VE, WebAssembly, X86, XCore) - You can vary the linker and compiler jobs depending on your memory and CPU to improve the performance for "sudo ninja install" - You can see the free memory with "free -h --giga" -• "sudo ninja install" -• Use "clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: ''}}" -line-filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]' test.c --" - For RemoveOnlyFunctionName you can use '' to strip all bodies. Alternativly use 'functionName1, functionName2' to specify the target functions - Additionaly you can specify which lines should be considered with the -line-filter option. To find all possible lines run without -fix --fix-errors. - -*/ - #include "RemoveFunctionBodyCheck.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" diff --git a/scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h similarity index 100% rename from scripts/incremental-test-generation/strip-body/RemoveFunctionBodyCheck.h rename to scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h diff --git a/scripts/incremental-test-generation/strip-body/strip-body.py b/scripts/incremental-test-generation/clang-mutations/strip-body.py similarity index 100% rename from scripts/incremental-test-generation/strip-body/strip-body.py rename to scripts/incremental-test-generation/clang-mutations/strip-body.py From 1592efe16767486442907c6bf3a5f73007b5eff2 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 15:27:43 +0200 Subject: [PATCH 013/131] Bugfix --- scripts/incremental-test-generation/clang-mutations/README.md | 2 +- .../clang-mutations/RemoveFunctionBodyCheck.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/README.md b/scripts/incremental-test-generation/clang-mutations/README.md index ab357886ee..7dfb04be8b 100644 --- a/scripts/incremental-test-generation/clang-mutations/README.md +++ b/scripts/incremental-test-generation/clang-mutations/README.md @@ -56,7 +56,7 @@ We will use the **>>check-name<<** again as defined in "Creating the checks". `clang-tidy` The command itself. - [ ] General Options - `-checks=-*,>>check-name<<` Deactivating all checks except >>check-name<<. + `-checks=-*,readability->>check-name<<` Deactivating all checks except >>check-name<<. `-fix --fix-errors` Applying the mutations. `-line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]'` Apply the mutations only on line 4 and 14. diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp index 26704d8829..a9db0bb87a 100644 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp @@ -52,7 +52,7 @@ void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); SourceLocation End = MatchedDecl->getBodyRBrace(); auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "Function %0 with index %1 has been stripped of its body") + diag(Start, "Function %0 has been stripped of its body") << MatchedDecl << FixItHint::CreateReplacement(Range, Replacement); } From 03868cbb1abff86d08184a0ec464f2cc0836e9e1 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 15:28:10 +0200 Subject: [PATCH 014/131] Add Unary Operator Inversion Mutation --- .../UnaryOperatorInversionCheck.cpp | 41 +++++++++++++++++++ .../UnaryOperatorInversionCheck.h | 19 +++++++++ 2 files changed, 60 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp new file mode 100644 index 0000000000..52e978875c --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp @@ -0,0 +1,41 @@ +//===--- UnaryOperatorInversionCheck.cpp - clang-tidy ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "UnaryOperatorInversionCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void UnaryOperatorInversionCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher(stmt(ifStmt()).bind("unary-operator-inversion"), this); +} + +// Replace if(e) with if(!(e)) +void UnaryOperatorInversionCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedDecl = Result.Nodes.getNodeAs("unary-operator-inversion"); + // Get locations + SourceLocation Start = MatchedDecl->getLParenLoc().getLocWithOffset(1); + SourceLocation End = MatchedDecl->getRParenLoc(); + auto Range = CharSourceRange::getCharRange(Start, End); + // Get the current expression in the if statement + const SourceManager *SM = Result.SourceManager; + const char *StartPtr = SM->getCharacterData(Start); + const char *EndPtr = SM->getCharacterData(End); + size_t Length = EndPtr - StartPtr; + std::string Expression(StartPtr, Length); + // Invert the expression with the unary operator + std::string Replacement = "!(" + Expression + ")" " /* Inverted if statement */"; + diag(Start, "If Statement %0 was inverted") + << Expression + << FixItHint::CreateReplacement(Range, Replacement); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h new file mode 100644 index 0000000000..d56556539e --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h @@ -0,0 +1,19 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Unary Operator Inversion for if statements +class UnaryOperatorInversionCheck : public ClangTidyCheck { +public: + UnaryOperatorInversionCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H From 4f0299bb848181f61fde64f6c09716d6315e78e5 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:07:43 +0200 Subject: [PATCH 015/131] Add [MUTATION] to comments --- .../clang-mutations/RemoveFunctionBodyCheck.cpp | 2 +- .../clang-mutations/UnaryOperatorInversionCheck.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp index a9db0bb87a..596dd41d97 100644 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp @@ -30,7 +30,7 @@ void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); // Remove the function body - std::string Replacement = " {\n\t// Stripped function of its body\n"; + std::string Replacement = " {\n\t// [MUTATION] Stripped function of its body\n"; const auto ReturnType = MatchedDecl->getReturnType(); if (!ReturnType->isVoidType()) { Replacement += "\treturn "; diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp index 52e978875c..396a506c27 100644 --- a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp @@ -32,7 +32,7 @@ void UnaryOperatorInversionCheck::check(const MatchFinder::MatchResult &Result) size_t Length = EndPtr - StartPtr; std::string Expression(StartPtr, Length); // Invert the expression with the unary operator - std::string Replacement = "!(" + Expression + ")" " /* Inverted if statement */"; + std::string Replacement = "!(" + Expression + ")" + " /* [MUTATION] Inverted if statement */"; diag(Start, "If Statement %0 was inverted") << Expression << FixItHint::CreateReplacement(Range, Replacement); From dd69c3a26736f4f2b10f1ae05c4d41529fc79103 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:07:56 +0200 Subject: [PATCH 016/131] Add test files --- .../sample-files/comparisons.c | 40 ++++++++++++++++++ .../clang-mutations/sample-files/functions.c | 40 ++++++++++++++++++ .../clang-mutations/sample-files/threads.c | 42 +++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c create mode 100644 scripts/incremental-test-generation/clang-mutations/sample-files/functions.c create mode 100644 scripts/incremental-test-generation/clang-mutations/sample-files/threads.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c b/scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c new file mode 100644 index 0000000000..ad5cd2cb77 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c @@ -0,0 +1,40 @@ +#include + +int main() { + int a = 10; + int b = 20; + + if (a < b) { + printf("a is less than b\n"); + } + + if (a <= b) { + printf("a is less than or equal to b\n"); + } + + if (a > b) { + printf("a is greater than b\n"); + } + + if (a >= b) { + printf("a is greater than or equal to b\n"); + } + + if (a == b) { + printf("a is equal to b\n"); + } + + if (a != b) { + printf("a is not equal to b\n"); + } + + if (a && b) { + printf("Both a and b are non-zero\n"); + } + + if (a || b) { + printf("Either a or b is non-zero\n"); + } + + return 0; +} diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/functions.c b/scripts/incremental-test-generation/clang-mutations/sample-files/functions.c new file mode 100644 index 0000000000..c96a2eec9e --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/functions.c @@ -0,0 +1,40 @@ +#include + +// Function to calculate the sum of two numbers +int sum(int a, int b) { + return a + b; +} + +// Function to calculate the difference of two numbers +int difference(int a, int b) { + return a - b; +} + +// Function to calculate the product of two numbers +int product(int a, int b) { + return a * b; +} + +// Function to calculate the quotient of two numbers +float divide(int a, int b) { + if (b != 0) { + return (float)a / b; + } else { + printf("Error: Division by zero is not allowed.\n"); + return 0; + } +} + +int main() { + int num1, num2; + + printf("Enter two numbers: "); + scanf("%d %d", &num1, &num2); + + printf("Sum: %d\n", sum(num1, num2)); + printf("Difference: %d\n", difference(num1, num2)); + printf("Product: %d\n", product(num1, num2)); + printf("Quotient: %.2f\n", divide(num1, num2)); + + return 0; +} diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c b/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c new file mode 100644 index 0000000000..13b3117c59 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c @@ -0,0 +1,42 @@ +#include +#include + +#define NUM_THREADS 5 + +void *threadFunction(void *arg) { + int threadID = *(int *)arg; + printf("Thread %d is running.\n", threadID); + // Perform some task in the thread + printf("Thread %d completed.\n", threadID); + pthread_exit(NULL); +} + +int main() { + pthread_t threads[NUM_THREADS]; + int threadArgs[NUM_THREADS]; + int i, result; + int j = 1; + + // Create threads + for (i = 0; i < NUM_THREADS; i++) { + threadArgs[i] = i; + result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); + if (result != 0) { + printf("Error creating thread %d. Exiting program.\n", i); + return -1; + } + } + + // Wait for threads to finish + for (i = 0; i < NUM_THREADS; i++) { + result = pthread_join(threads[i], NULL); + if (result != 0) { + printf("Error joining thread %d. Exiting program.\n", i); + return -1; + } + } + + printf("All threads have completed. Exiting program.\n"); + + return 0; +} From c63a024631c5e8dff732f2bb043294d9f8cf5fa2 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:08:10 +0200 Subject: [PATCH 017/131] Add ROR Mutation --- .../RelationalOperatorReplacementCheck.cpp | 51 +++++++++++++++++++ .../RelationalOperatorReplacementCheck.h | 19 +++++++ 2 files changed, 70 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h diff --git a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp new file mode 100644 index 0000000000..9b675c87f6 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp @@ -0,0 +1,51 @@ +#include "RelationalOperatorReplacementCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void RelationalOperatorReplacementCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher(stmt(expr(binaryOperator())).bind("relational-operator-replacement"), this); +} + +// ">=" <-> ">" and "=<" <-> "<" +void RelationalOperatorReplacementCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedDecl = Result.Nodes.getNodeAs("relational-operator-replacement"); + + int OperatorSize; + std::string Replacement; + + if (MatchedDecl->getOpcode() == BO_GE) { + // GE -> G + OperatorSize = 2; + Replacement = ">"; + } else if (MatchedDecl->getOpcode() == BO_GT) { + // G -> GE + OperatorSize = 1; + Replacement = ">="; + } else if (MatchedDecl->getOpcode() == BO_LE) { + // LE -> L + OperatorSize = 2; + Replacement = "<"; + } else if (MatchedDecl->getOpcode() == BO_LT) { + // LE -> L + OperatorSize = 1; + Replacement = "<="; + } else { + return; + } + Replacement += " /* [MUTATION] Replaced Relational Operator */"; + + // Get locations + SourceLocation Start = MatchedDecl->getOperatorLoc(); + SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(OperatorSize); + auto Range = CharSourceRange::getCharRange(Start, End); + + diag(Start, "Replaced Relational Operator %0") + << MatchedDecl->getOpcodeStr() + << FixItHint::CreateReplacement(Range, Replacement); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h new file mode 100644 index 0000000000..c0aeee12cc --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h @@ -0,0 +1,19 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Relational Operator Replacement +class RelationalOperatorReplacementCheck : public ClangTidyCheck { +public: + RelationalOperatorReplacementCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H From 1e71c0b7c019587646865baa9625bfb7ea786f67 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:34:51 +0200 Subject: [PATCH 018/131] Remove function body in one line --- .../clang-mutations/RemoveFunctionBodyCheck.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp index 596dd41d97..b1c97fb642 100644 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp @@ -29,11 +29,12 @@ void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); + // Remove the function body - std::string Replacement = " {\n\t// [MUTATION] Stripped function of its body\n"; + std::string Replacement = " { "; const auto ReturnType = MatchedDecl->getReturnType(); if (!ReturnType->isVoidType()) { - Replacement += "\treturn "; + Replacement += "return "; if (ReturnType->isPointerType() || ReturnType->isNullPtrType()) { Replacement += "0"; } else if (ReturnType->isIntegralType(*Result.Context) || ReturnType->isCharType()) { @@ -45,12 +46,15 @@ void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { } else if (const RecordType *RT = ReturnType->getAsUnionType()) { Replacement += "(union " + RT->getDecl()->getNameAsString() + "){}"; } else { - Replacement += "/* TODO: Add generic return value for " + ReturnType.getAsString() + " */"; + Replacement += "/* [TODO]: Add generic return value for " + ReturnType.getAsString() + " */"; } - Replacement += ";\n"; + Replacement += "; "; } + Replacement += "/* [MUTATION] Stripped function of its body */ }"; + + // Get locations SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); - SourceLocation End = MatchedDecl->getBodyRBrace(); + SourceLocation End = MatchedDecl->getBodyRBrace().getLocWithOffset(1); auto Range = CharSourceRange::getCharRange(Start, End); diag(Start, "Function %0 has been stripped of its body") << MatchedDecl From ac053d88bff66ca84fb93b22d54c86e07218c8ae Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:35:06 +0200 Subject: [PATCH 019/131] Remove unnecesary comments --- .../clang-mutations/UnaryOperatorInversionCheck.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp index 396a506c27..24fa4a66e8 100644 --- a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp @@ -1,11 +1,3 @@ -//===--- UnaryOperatorInversionCheck.cpp - clang-tidy ---------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - #include "UnaryOperatorInversionCheck.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" From 5958abb1a5ffae484993d6e7eecf112a42e41a7e Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 18 May 2023 16:35:18 +0200 Subject: [PATCH 020/131] Add Logical Connector Mutation --- .../LogicalConnectorReplacementCheck.cpp | 40 +++++++++++++++++++ .../LogicalConnectorReplacementCheck.h | 19 +++++++++ 2 files changed, 59 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h diff --git a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp new file mode 100644 index 0000000000..edee5399fa --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp @@ -0,0 +1,40 @@ +#include "LogicalConnectorReplacementCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void LogicalConnectorReplacementCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher(stmt(expr(binaryOperator())).bind("logical-connector-replacement"), this); +} + +// "||" <-> "&&" +void LogicalConnectorReplacementCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedDecl = Result.Nodes.getNodeAs("logical-connector-replacement"); + + std::string Replacement; + + if (MatchedDecl->getOpcode() == BO_LOr) { + // || -> && + Replacement = "&&"; + } else if (MatchedDecl->getOpcode() == BO_LAnd) { + // && -> || + Replacement = "||"; + } else { + return; + } + Replacement += " /* [MUTATION] Replaced Logical Connector */"; + + // Get locations + SourceLocation Start = MatchedDecl->getOperatorLoc(); + SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(2); + auto Range = CharSourceRange::getCharRange(Start, End); + + diag(Start, "Replaced Logical Connector %0") + << MatchedDecl->getOpcodeStr() + << FixItHint::CreateReplacement(Range, Replacement); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h new file mode 100644 index 0000000000..9dd76e2856 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h @@ -0,0 +1,19 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Logical Connector Replacement +class LogicalConnectorReplacementCheck : public ClangTidyCheck { +public: + LogicalConnectorReplacementCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H From 6702b5600e9f9ed94b7e296c793ca40b998fd725 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 19 May 2023 13:47:18 +0200 Subject: [PATCH 021/131] Add Constant Replacement Check --- .../ConstantReplacementCheck.cpp | 40 +++++++++++++++++++ .../ConstantReplacementCheck.h | 19 +++++++++ 2 files changed, 59 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h diff --git a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp new file mode 100644 index 0000000000..e5cbad7960 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp @@ -0,0 +1,40 @@ +#include "ConstantReplacementCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/Lex/Lexer.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void ConstantReplacementCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher(integerLiteral(isExpansionInMainFile(), unless(anyOf(equals(0), equals(1)))).bind("constant-replacement"), this); +} + +// Replaces all Integer Literals != 0 with 1 +void ConstantReplacementCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedDecl = Result.Nodes.getNodeAs("constant-replacement"); + + // Get old int value + llvm::APInt value = MatchedDecl->getValue(); + std::string valueAsString; + llvm::SmallString<8> buffer; + value.toString(buffer, 10, true); + valueAsString = buffer.str(); + // Get locations + SourceLocation Start = MatchedDecl->getBeginLoc(); + auto Range = MatchedDecl->getSourceRange(); + // Check if it is a Macro + std::string MacroInfo = ""; + if (MatchedDecl->getLocation().isMacroID()) { + std::string MacroId = Lexer::getImmediateMacroName(MatchedDecl->getLocation(), *Result.SourceManager, Result.Context->getLangOpts()).str(); + MacroInfo = "[MACRO][" + MacroId + "]"; + } + // Replace with 1 + std::string Replacement = "1 /* [MUTATION]" + MacroInfo + " Replaced Constant " + valueAsString + " */"; + diag(Start, (MacroInfo.empty() ? "" : (MacroInfo + " ")) + "Replaced Constant %0 with 1") + << valueAsString + << FixItHint::CreateReplacement(Range, Replacement); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h new file mode 100644 index 0000000000..182c389aad --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h @@ -0,0 +1,19 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Constant Replacement +class ConstantReplacementCheck : public ClangTidyCheck { +public: + ConstantReplacementCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H From 5583a91af08ad9e1f91b707ca8de28141d7319f2 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 19 May 2023 14:30:39 +0200 Subject: [PATCH 022/131] Add Mutation Type to Brackets --- .../clang-mutations/ConstantReplacementCheck.cpp | 8 ++++---- .../clang-mutations/LogicalConnectorReplacementCheck.cpp | 4 ++-- .../RelationalOperatorReplacementCheck.cpp | 4 ++-- .../clang-mutations/RemoveFunctionBodyCheck.cpp | 4 ++-- .../clang-mutations/UnaryOperatorInversionCheck.cpp | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp index e5cbad7960..18fa68b1c6 100644 --- a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp @@ -25,14 +25,14 @@ void ConstantReplacementCheck::check(const MatchFinder::MatchResult &Result) { SourceLocation Start = MatchedDecl->getBeginLoc(); auto Range = MatchedDecl->getSourceRange(); // Check if it is a Macro - std::string MacroInfo = ""; + std::string MacroInfo = " "; if (MatchedDecl->getLocation().isMacroID()) { std::string MacroId = Lexer::getImmediateMacroName(MatchedDecl->getLocation(), *Result.SourceManager, Result.Context->getLangOpts()).str(); - MacroInfo = "[MACRO][" + MacroId + "]"; + MacroInfo = "[MACRO][" + MacroId + "] "; } // Replace with 1 - std::string Replacement = "1 /* [MUTATION]" + MacroInfo + " Replaced Constant " + valueAsString + " */"; - diag(Start, (MacroInfo.empty() ? "" : (MacroInfo + " ")) + "Replaced Constant %0 with 1") + std::string Replacement = "1 /* [MUTATION][CR]" + MacroInfo + "Replaced Constant " + valueAsString + " */"; + diag(Start, "[MUTATION][CR]" + MacroInfo + "Replaced Constant %0 with 1") << valueAsString << FixItHint::CreateReplacement(Range, Replacement); } diff --git a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp index edee5399fa..21a04f1c6b 100644 --- a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp @@ -25,14 +25,14 @@ void LogicalConnectorReplacementCheck::check(const MatchFinder::MatchResult &Res } else { return; } - Replacement += " /* [MUTATION] Replaced Logical Connector */"; + Replacement += " /* [MUTATION][LCR] Replaced Logical Connector */"; // Get locations SourceLocation Start = MatchedDecl->getOperatorLoc(); SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(2); auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "Replaced Logical Connector %0") + diag(Start, "[MUTATION][LCR] Replaced Logical Connector %0") << MatchedDecl->getOpcodeStr() << FixItHint::CreateReplacement(Range, Replacement); } diff --git a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp index 9b675c87f6..489cb48018 100644 --- a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp @@ -36,14 +36,14 @@ void RelationalOperatorReplacementCheck::check(const MatchFinder::MatchResult &R } else { return; } - Replacement += " /* [MUTATION] Replaced Relational Operator */"; + Replacement += " /* [MUTATION][ROR] Replaced Relational Operator */"; // Get locations SourceLocation Start = MatchedDecl->getOperatorLoc(); SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(OperatorSize); auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "Replaced Relational Operator %0") + diag(Start, "[MUTATION][ROR] Replaced Relational Operator %0") << MatchedDecl->getOpcodeStr() << FixItHint::CreateReplacement(Range, Replacement); } diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp index b1c97fb642..5001a5a5de 100644 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp @@ -50,13 +50,13 @@ void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { } Replacement += "; "; } - Replacement += "/* [MUTATION] Stripped function of its body */ }"; + Replacement += "/* [MUTATION][RFB] Stripped function of its body */ }"; // Get locations SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); SourceLocation End = MatchedDecl->getBodyRBrace().getLocWithOffset(1); auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "Function %0 has been stripped of its body") + diag(Start, "[MUTATION][RFB] Function %0 has been stripped of its body") << MatchedDecl << FixItHint::CreateReplacement(Range, Replacement); } diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp index 24fa4a66e8..98b3b6f35c 100644 --- a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp @@ -24,8 +24,8 @@ void UnaryOperatorInversionCheck::check(const MatchFinder::MatchResult &Result) size_t Length = EndPtr - StartPtr; std::string Expression(StartPtr, Length); // Invert the expression with the unary operator - std::string Replacement = "!(" + Expression + ")" + " /* [MUTATION] Inverted if statement */"; - diag(Start, "If Statement %0 was inverted") + std::string Replacement = "!(" + Expression + ")" + " /* [MUTATION][UOI] Inverted if statement */"; + diag(Start, "[MUTATION][UOI] If Statement %0 was inverted") << Expression << FixItHint::CreateReplacement(Range, Replacement); } From 7875a58d863ed15129a1191c820f168c6d02df41 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 19 May 2023 15:24:18 +0200 Subject: [PATCH 023/131] New sample file for pthreads --- .../clang-mutations/sample-files/pthread.c | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c b/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c new file mode 100644 index 0000000000..02a114a971 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c @@ -0,0 +1,126 @@ +#include +#include + +void* thread_function(void* arg) { + printf("Thread function\n"); + return NULL; +} + +void cleanup_function(void* arg) { + printf("Cleanup function\n"); +} + +int main() { + pthread_t thread; + pthread_attr_t attr; + pthread_cond_t cond; + pthread_condattr_t cond_attr; + pthread_key_t key; + pthread_mutex_t mutex; + pthread_mutexattr_t mutex_attr; + pthread_once_t once_control; + pthread_rwlock_t rwlock; + pthread_rwlockattr_t rwlock_attr; + struct sched_param param; + + // Attribute functions + pthread_attr_destroy(&attr); + pthread_attr_getdetachstate(&attr, NULL); + pthread_attr_getguardsize(&attr, NULL); + pthread_attr_getinheritsched(&attr, NULL); + pthread_attr_getschedparam(&attr, ¶m); + pthread_attr_getschedpolicy(&attr, NULL); + pthread_attr_getscope(&attr, NULL); + pthread_attr_getstackaddr(&attr, NULL); + pthread_attr_getstacksize(&attr, NULL); + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, 0); + pthread_attr_setguardsize(&attr, 0); + pthread_attr_setinheritsched(&attr, 0); + pthread_attr_setschedparam(&attr, ¶m); + pthread_attr_setschedpolicy(&attr, 0); + pthread_attr_setscope(&attr, 0); + pthread_attr_setstackaddr(&attr, NULL); + pthread_attr_setstacksize(&attr, 0); + + // Cancellation functions + pthread_cancel(thread); + pthread_setcancelstate(0, NULL); + pthread_setcanceltype(0, NULL); + pthread_testcancel(); + + // Condition variable functions + pthread_cond_broadcast(&cond); + pthread_cond_destroy(&cond); + pthread_cond_init(&cond, &cond_attr); + pthread_cond_signal(&cond); + pthread_cond_timedwait(&cond, &mutex, NULL); + pthread_cond_wait(&cond, &mutex); + + // Condition attribute functions + pthread_condattr_destroy(&cond_attr); + pthread_condattr_getpshared(&cond_attr, NULL); + pthread_condattr_init(&cond_attr); + pthread_condattr_setpshared(&cond_attr, 0); + + // Creation and termination functions + pthread_create(&thread, &attr, thread_function, NULL); + pthread_detach(thread); + pthread_equal(thread, thread); + pthread_exit(NULL); + pthread_join(thread, NULL); + + // Thread-specific data functions + pthread_key_create(&key, cleanup_function); + pthread_key_delete(key); + pthread_getspecific(key); + pthread_setspecific(key, NULL); + + // Mutex functions + pthread_mutex_destroy(&mutex); + pthread_mutex_getprioceiling(&mutex, NULL); + pthread_mutex_init(&mutex, &mutex_attr); + pthread_mutex_lock(&mutex); + pthread_mutex_setprioceiling(&mutex, 0, NULL); + pthread_mutex_trylock(&mutex); + pthread_mutex_unlock(&mutex); + + // Mutex attribute functions + pthread_mutexattr_destroy(&mutex_attr); + pthread_mutexattr_getprioceiling(&mutex_attr, NULL); + pthread_mutexattr_getprotocol(&mutex_attr, NULL); + pthread_mutexattr_getpshared(&mutex_attr, NULL); + pthread_mutexattr_gettype(&mutex_attr, NULL); + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_setprioceiling(&mutex_attr, 0); + pthread_mutexattr_setprotocol(&mutex_attr, 0); + pthread_mutexattr_setpshared(&mutex_attr, 0); + pthread_mutexattr_settype(&mutex_attr, 0); + + // One-time initialization + pthread_once(&once_control, thread_function); + + // Read-write lock functions + pthread_rwlock_destroy(&rwlock); + pthread_rwlock_init(&rwlock, &rwlock_attr); + pthread_rwlock_rdlock(&rwlock); + pthread_rwlock_tryrdlock(&rwlock); + pthread_rwlock_trywrlock(&rwlock); + pthread_rwlock_unlock(&rwlock); + pthread_rwlock_wrlock(&rwlock); + + // Read-write lock attribute functions + pthread_rwlockattr_destroy(&rwlock_attr); + pthread_rwlockattr_getpshared(&rwlock_attr, NULL); + pthread_rwlockattr_init(&rwlock_attr); + pthread_rwlockattr_setpshared(&rwlock_attr, 0); + + // Other functions + pthread_self(); + pthread_getschedparam(thread, NULL, ¶m); + pthread_setschedparam(thread, 0, ¶m); + pthread_getconcurrency(); + pthread_setconcurrency(0); + + return 0; +} From 7d41ae42d281c6495dbd8d45caaa1de5c55a3e42 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 19 May 2023 15:35:48 +0200 Subject: [PATCH 024/131] Bugfix --- .../clang-mutations/sample-files/pthread.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c b/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c index 02a114a971..b47776dbf0 100644 --- a/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c @@ -98,7 +98,7 @@ int main() { pthread_mutexattr_settype(&mutex_attr, 0); // One-time initialization - pthread_once(&once_control, thread_function); + pthread_once(&once_control, (void (*)(void))thread_function); // Read-write lock functions pthread_rwlock_destroy(&rwlock); @@ -119,8 +119,6 @@ int main() { pthread_self(); pthread_getschedparam(thread, NULL, ¶m); pthread_setschedparam(thread, 0, ¶m); - pthread_getconcurrency(); - pthread_setconcurrency(0); return 0; } From a2d4bee984268a7ce1ee66fa27c543b1efb690f3 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 22 May 2023 10:42:37 +0200 Subject: [PATCH 025/131] Add Remove Thread Mutation --- .../clang-mutations/RemoveThreadCheck.cpp | 63 +++++++++++++++++++ .../clang-mutations/RemoveThreadCheck.h | 22 +++++++ .../clang-mutations/strip-body.py | 25 -------- 3 files changed, 85 insertions(+), 25 deletions(-) create mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/strip-body.py diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp new file mode 100644 index 0000000000..7e1748debd --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp @@ -0,0 +1,63 @@ +#include "RemoveThreadCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void RemoveThreadCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher(callExpr(callee(functionDecl(hasName("pthread_create")))).bind("remove-thread"), this); +} + +void RemoveThreadCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedDecl = Result.Nodes.getNodeAs("remove-thread"); + + // Get locations + SourceLocation Start = MatchedDecl->getBeginLoc(); + SourceLocation End = MatchedDecl->getEndLoc().getLocWithOffset(1); + auto Range = CharSourceRange::getCharRange(Start, End); + // Get the arguments + const SourceManager *SM = Result.SourceManager; + const char *StartPtr = SM->getCharacterData(Start); + const char *EndPtr = SM->getCharacterData(End); + size_t Length = EndPtr - StartPtr; + std::string S(StartPtr, Length); + std::vector Arguments; + std::istringstream iss(S); + std::string argument; + while (std::getline(iss, argument, ',')) { + Arguments.push_back(argument); + } + // Get the function name + std::string FunctionName = parseFunctionName(Arguments[2]); + // Get the argument + std::string Argument = parseArgument(Arguments[3]); + // Create the replacement + std::string Replacement = FunctionName + "(" + Argument + ") /* [MUTATION][RT] Thread creation was substituted with function call */"; + diag(Start, "[MUTATION][RT] Thread creation was substituted with function call %0") + << FunctionName + << FixItHint::CreateReplacement(Range, Replacement); +} + +std::string RemoveThreadCheck::parseFunctionName(std::string FunctionName) { + // Remove leading and trailing whitespaces + FunctionName.erase(0, FunctionName.find_first_not_of(" \t")); + FunctionName.erase(FunctionName.find_last_not_of(" \t") + 1); + // Remove leading pointers + FunctionName.erase(0, FunctionName.find_first_not_of("*")); + // Remove leading dereference + FunctionName.erase(0, FunctionName.find_first_not_of("&")); + return FunctionName; +} + +std::string RemoveThreadCheck::parseArgument(std::string Argument) { + // Remove leading and trailing whitespaces + Argument.erase(0, Argument.find_first_not_of(" \t")); + Argument.erase(Argument.find_last_not_of(" \t") + 1); + Argument.erase(Argument.find_last_of(')'), 1); + return Argument; +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h new file mode 100644 index 0000000000..71cf8d4ca7 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h @@ -0,0 +1,22 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Remove Thread +class RemoveThreadCheck : public ClangTidyCheck { +public: + RemoveThreadCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +private: + std::string parseFunctionName(std::string FunctionName); + std::string parseArgument(std::string Argument); +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/strip-body.py b/scripts/incremental-test-generation/clang-mutations/strip-body.py deleted file mode 100644 index 9f33432630..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/strip-body.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -# Change these variables to match your setup -CLANG_PATH = "~/BA/Clang/llvm-project/build/bin" -CHECKS_PATH = "~/BA/removeBody.so" - -# The name of the file to check -file_name = "test.c" - -# Build the clang command to run the removeBody check -clang_command = [ - "clang", - "-Xclang", - "-load", - "-Xclang", - CHECKS_PATH, - "-Xclang", - "-add-plugin", - "-Xclang", - "removeBody", - file_name, -] - -# Run the command -os.system(" ".join(clang_command)) \ No newline at end of file From e5483b0ca4d2fc29d6873f5c540f69597fff2204 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 22 May 2023 12:53:49 +0200 Subject: [PATCH 026/131] Documentation of Mutations --- .../clang-mutations/MUTATIONS.md | 94 +++++++++++++++++++ .../clang-mutations/README.md | 14 +-- .../clang-mutations/sample-files/constants.c | 6 ++ 3 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 scripts/incremental-test-generation/clang-mutations/MUTATIONS.md create mode 100644 scripts/incremental-test-generation/clang-mutations/sample-files/constants.c diff --git a/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md b/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md new file mode 100644 index 0000000000..83a493a3a7 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md @@ -0,0 +1,94 @@ +# Mutations +In this document is described what the different mutations do. In the [Readme](README.md) file it is described how to run the mutations. + +## Remove Function Body - RFB +**readability-remove-function-body**
+Removes the function body and adds when needed a generic return statement. +``` +int sum(int a, int b) { + return a + b; +} +``` +`clang-tidy -checks=-*,readability-remove-function-body -fix test.c --` +``` +int sum(int a, int b) { return 0; /* [MUTATION][RFB] Stripped function of its body */ } +``` +### Special Option +`-config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'sum, divide, ...'}}"` Special option for the **remove-function-body** check to only remove the function body of functions named foo1 and foo2. + +## Unary Operator Inversion - UOI +**readability-unary-operator-inversion**
+Flips if statements by adding a negation. +``` +if (a < b) { + printf("a is less than b\n"); +} +``` +`clang-tidy -checks=-*,readability-unary-operator-inversion -fix comparisons.c --` +``` +if (!(a < b) /* [MUTATION][UOI] Inverted if statement */) { + printf("a is less than b\n"); +} +``` + +## Relational Operator Replacement - ROR +**readability-relational-operator-replacement**
+Replaces `<=` with `<`, `<` with `<=`, `>=` with `>` and `>` with `>=`. +``` +if (a <= b) { + printf("a is less than or equal to b\n"); +} +``` +`clang-tidy -checks=-*,readability-relational-operator-replacement -fix comparisons.c --` +``` +if (a < /* [MUTATION][ROR] Replaced Relational Operator */ b) { + printf("a is less than or equal to b\n"); +} +``` + +## Constant Replacement - CR +**readability-constant-replacement**
+Replaces constants unequal 0 and unequal 1 with 1. The usage of macros is replaced, but not the definition. This is marked with `[MACRO][macro_name]`. +``` +#define MY_MACRO_5 5 +#define MY_MACRO_0 0 +int a = 42; +int b = 0; +int c = MY_MACRO_5; +int d = MY_MACRO_0; +``` +`clang-tidy -checks=-*,readability-constant-replacement -fix constants.c --` +``` +#define MY_MACRO_5 5 +#define MY_MACRO_0 0 +int a = 1 /* [MUTATION][CR] Replaced Constant 42 */; +int b = 0; +int c = 1 /* [MUTATION][CR][MACRO][MY_MACRO_5] Replaced Constant 5 */; +int d = MY_MACRO_0; +``` + +## Remove Thread - RT +**readability-remove-thread**
+Replaces a `pthread_create` call with the function call itself. The arguments of the function call are kept. Symbols like `*` or `&` in front of the function name are ignored. +``` +pthread_create(&thread, &attr, thread_function, NULL); +``` +`clang-tidy -checks=-*,readability-remove-thread -fix pthread.c --` +``` +thread_function(NULL) /* [MUTATION][RT] Thread creation was substituted with function call */; +``` + +## Logical Connector Replacement - LCR +**readability-logical-connector-replacement**
+Replaces the operator `&&` with `||` and vice versa. +``` +if (a && b) { + printf("Both a and b are non-zero\n"); +} +``` +`clang-tidy -checks=-*,readability-logical-connector-replacement -fix comparisons.c --` +``` +if (a || /* [MUTATION][LCR] Replaced Logical Connector */ b) { + printf("Both a and b are non-zero\n"); +} +``` \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/README.md b/scripts/incremental-test-generation/clang-mutations/README.md index 7dfb04be8b..8c1863fff1 100644 --- a/scripts/incremental-test-generation/clang-mutations/README.md +++ b/scripts/incremental-test-generation/clang-mutations/README.md @@ -1,10 +1,7 @@ # Create Clang-Tidy Checks for mutations - In this document is described how you can create all the Clang-Tidy checks needed for generating the code mutations. - ## Dependencies - For building Clang you need to install some dependencies: - [ ] `sudo apt install ninja-build` @@ -12,7 +9,6 @@ For building Clang you need to install some dependencies: - [ ] `sudo apt install lld` ## Cloning the repository - - [ ] There are two alternatives for getting the repository For creating all the checks by yourself clone the **Official Clang Repository**: @@ -21,7 +17,6 @@ Alternatively you can clone the **Fork** with all the additional checks ready: `git clone https://github.com/J2000A/llvm-project.git` ## Creating the checks - - [ ] When you cloned the Official Clang Repository you need to add the checks. Otherwise you can skip this part. - [ ] Move to the directory `llvm-project/clang-tools-extra` @@ -34,7 +29,6 @@ In this directory are the implementations of the checks with their corresponding Now you have added all the check we need for the mutations. ## Build - The first build can take a while (up to multiple hours). But you can increase the performance by changing the parallel compile and link jobs. For me using the value 5 for both of them got me the fastest results. When using too many jobs the memory becomes a bottleneck. You can check the memory status with `free -h --giga`. Additionally you may need to change the build target. Avaiable targets are: AMDGPU, ARM, AVR, BPF, Hexagon, Lanai, LoongArch, Mips, MSP430, NVPTX, PowerPC, RISCV, Sparc, SystemZ, VE, WebAssembly, X86, XCore @@ -44,20 +38,19 @@ Additionally you may need to change the build target. Avaiable targets are: AMDG - [ ] `sudo ninja install` ## Running Clang-Tidy - We will use the **>>check-name<<** again as defined in "Creating the checks". **Example:** Create the mutation "remove function body" on a file "test.c" in lines "4" and "14" when the function name is "foo": `clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'foo'}}" -line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]' test.c --` **The command consists of the following components:** - - [ ] Clang-Tidy `clang-tidy` The command itself. - [ ] General Options `-checks=-*,readability->>check-name<<` Deactivating all checks except >>check-name<<. -`-fix --fix-errors` Applying the mutations. +`-fix` Applying the mutations. +`--fix-errors` Apply also when errors where detected `-line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]'` Apply the mutations only on line 4 and 14. - [ ] Special Options @@ -66,5 +59,8 @@ We will use the **>>check-name<<** again as defined in "Creating the checks". - [ ] Filename `test.c --` The filename. +## Mutations +You find more details about the different Mutations in the [Mutations](MUTATIONS.md) file. + ## Workflow First run the check without `-fix --fix-errors` to see where mutations are possible without applying them. Remember the lines where you actually want to apply the mutation. Make a copy of the input file that you will mutate. The run the check again with `-fix --fix-errors` and `-line filter=...` on the copied file to apply only specific mutations and not all at ones. \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/constants.c b/scripts/incremental-test-generation/clang-mutations/sample-files/constants.c new file mode 100644 index 0000000000..a232571e92 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/constants.c @@ -0,0 +1,6 @@ +#define MY_MACRO_5 5 +#define MY_MACRO_0 0 +int a = 42; +int b = 0; +int c = MY_MACRO_5; +int d = MY_MACRO_0; \ No newline at end of file From 507fe1260191667929cf82d3688c45dd033c5148 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 22 May 2023 15:03:53 +0200 Subject: [PATCH 027/131] Bugfix --- .../clang-mutations/sample-files/threads.c | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c b/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c index 13b3117c59..0148d12a0d 100644 --- a/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c +++ b/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c @@ -15,7 +15,6 @@ int main() { pthread_t threads[NUM_THREADS]; int threadArgs[NUM_THREADS]; int i, result; - int j = 1; // Create threads for (i = 0; i < NUM_THREADS; i++) { From 31d39cf840f85bd683a2f6882c0e89f004716dcb Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 22 May 2023 15:04:07 +0200 Subject: [PATCH 028/131] Add ML Heuristic --- .../openai-text-completion/.gitignore | 1 + .../openai-text-completion/APIKEY.yaml | 3 + .../openai-text-completion/ml.py | 77 ++++++ .../outputs/20230522-01.txt | 233 ++++++++++++++++++ 4 files changed, 314 insertions(+) create mode 100644 scripts/incremental-test-generation/openai-text-completion/.gitignore create mode 100644 scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml create mode 100644 scripts/incremental-test-generation/openai-text-completion/ml.py create mode 100644 scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt diff --git a/scripts/incremental-test-generation/openai-text-completion/.gitignore b/scripts/incremental-test-generation/openai-text-completion/.gitignore new file mode 100644 index 0000000000..ed91ac7eef --- /dev/null +++ b/scripts/incremental-test-generation/openai-text-completion/.gitignore @@ -0,0 +1 @@ +APIKEY.txt diff --git a/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml b/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml new file mode 100644 index 0000000000..11bfadb275 --- /dev/null +++ b/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml @@ -0,0 +1,3 @@ +organisation: org-J3GRsachQEfeSZXv7KpFj3vD +api-key: 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 + diff --git a/scripts/incremental-test-generation/openai-text-completion/ml.py b/scripts/incremental-test-generation/openai-text-completion/ml.py new file mode 100644 index 0000000000..d8b10ffc8d --- /dev/null +++ b/scripts/incremental-test-generation/openai-text-completion/ml.py @@ -0,0 +1,77 @@ +import os +import yaml +import openai + +# [TODO] run "sudo pip install openai" +# +# [TODO] get api key and store it in a yaml file: +# organisation: .... +# api-key: ... +# +api_key_path = os.path.expanduser("~/BA/Goblint-Repo/analyzer/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml") + +# Read the api key and organisation +with open(api_key_path, 'r') as file: + data = yaml.safe_load(file) +organisation = data.get('organisation') +api_key = data.get('api-key') + +# Authenticate +openai.organization = organisation +openai.api_key = api_key + +# Select Model +model = "text-davinci-edit-001" + +input = r''' +#include + +// Function to calculate the sum of two numbers +int sum(int a, int b) { + return a + b; +} + +// Function to calculate the difference of two numbers +int difference(int a, int b) { + return a - b; +} + +// Function to calculate the product of two numbers +int product(int a, int b) { + return a * b; +} + +// Function to calculate the quotient of two numbers +float divide(int a, int b) { + if (b != 0) { + return (float)a / b; + } else { + printf("Error: Division by zero is not allowed.\n"); + return 0; + } +} + +int main() { + int num1, num2; + + printf("Enter two numbers: "); + scanf("%d %d", &num1, &num2); + + printf("Sum: %d\n", sum(num1, num2)); + printf("Difference: %d\n", difference(num1, num2)); + printf("Product: %d\n", product(num1, num2)); + printf("Quotient: %.2f\n", divide(num1, num2)); + + return 0; +} +''' + +completion = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + n = 3, + messages=[ + {"role": "user", "content": "Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code and introduce bugs. But the code should still compile. Here the C code: " + input}, + ] +) + +print(completion.choices[0].message) \ No newline at end of file diff --git a/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt b/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt new file mode 100644 index 0000000000..0eef825b54 --- /dev/null +++ b/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt @@ -0,0 +1,233 @@ +Promt: Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code and introduce bugs. But the code should still compile. Here the C code: +############################################################################################### +#include +#include + +#define NUM_THREADS 5 + +void *threadFunction(void *arg) { + int threadID = *(int *)arg; + printf("Thread %d is running.\n", threadID); + + // Perform some task in the thread + + printf("Thread %d completed.\n", threadID); + pthread_exit(NULL); +} + +int main() { + pthread_t threads[NUM_THREADS]; + int threadArgs[NUM_THREADS]; + int i, result; + + // Create threads + for (i = 0; i < NUM_THREADS; i++) { + threadArgs[i] = i; + result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); + if (result != 0) { + printf("Error creating thread %d. Exiting program.\n", i); + return -1; + } + } + + // Wait for threads to finish + for (i = 0; i < NUM_THREADS; i++) { + result = pthread_join(threads[i], NULL); + if (result != 0) { + printf("Error joining thread %d. Exiting program.\n", i); + return -1; + } + } + + printf("All threads have completed. Exiting program.\n"); + + return 0; +} + +############################################################################################################ + +#include + +void *threadFunction(void *arg) { + int threadID = *(int *)arg; + printf("Thread %d is running.\n", threadID); + printf("Thread %d completed.\n", threadID); + pthread_exit(NULL); +} + +int main() { + pthread_t threads[NUM_THREADS]; + int threadArgs[NUM_THREADS]; + int i, result; + + for (i = 0; i < NUM_THREADS; i++) { + threadArgs[i] = i; + result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); + } + + for (i = 0; i < NUM_THREADS; i++) { + result = pthread_join(threads[i], NULL); + } + + printf("Exiting program.\n"); + + return 0; +} + +############################################################################################################ +#include + +#define NUM_THREADS 5 + +void *threadFunction(void *arg) { + int threadID = *(int *)arg; + printf("Thread %d is running.\n", threadID); + // Perform some task in the thread + printf("Thread %d completed.\n", threadID); + return NULL; +} + +int main() { + int threads[NUM_THREADS]; + int threadArgs[NUM_THREADS]; + int i, result; + + // Create threads + for (i = 0; i < NUM_THREADS; i++) { + threadArgs[i] = i; + result = threads[i]; // Introduce bug + if (result != 0) { + printf("Error creating thread %d. Exiting program.\n", i); + return -1; + } + } + + // Wait for threads to finish + for (i = 0; i < NUM_THREADS; i++) { + result = threads[i]; // Introduce bug + if (result != 0) { + printf("Error joining thread %d. Exiting program.\n", i); + return -1; + } + } + + printf("All threads have completed. Exiting program.\n"); + + return 0; +} + +############################################################################################################ +#include + +int sum(int a, int b) { + return a + b; +} + +int product(int a, int b) { + return a * b; +} + +float divide(int a, int b) { + if (b != 0) { + return (float) a / b; + } else { + printf("Error: Division by zero is not allowed.\n"); + return 0; + } +} + +int main() { + int num1, num2; + + printf("Enter two numbers: "); + scanf("%d", &num1); + scanf("%d", &num2); + + printf("Sum: %d\n", sum(num1, num2)); + printf("Product: %d\n", product(num1, num2)); + printf("Quotient: %.2f\n", divide(num1, num2)); + + return 0; +} + +############################################################################################################ +#include + +int sum(int a, int b) { + return a + b; +} + +int product(int a, int b) { + return a * b; +} + +float divide(int a, int b) { + if (b != 0) { + return (float) a / b; + } else { + printf("Error: Division by zero is not allowed.\n"); + return 0; + } +} + +int main() { + int num1, num2; + + printf("Enter two numbers: "); + scanf("%d", &num1); + scanf("%d", &num2); + + printf("Sum: %d\n", sum(num1, num2)); + printf("Product: %d\n", product(num1, num2)); + printf("Quotient: %.2f\n", divide(num1, num2)); + + return 0; +} + +############################################################################################################ +#include + +// Function to calculate the sum of two numbers +int sum(int a, int b) { + return a + b; // function remains unchanged +} + +// Function to calculate the difference of two numbers +int difference(int a, int b) { + return a - b; // function remains unchanged +} + +// Function to calculate the product of two numbers +int product(int a, int b) { + return a; // introducing bug by returning only the first number +} + +// Function to calculate the quotient of two numbers +float divide(int a, int b) { + if (b != 0) { + return a / b; // introducing bug by not casting to float + } else { + printf("Error: Division by zero is not allowed.\n"); + return a; // introducing bug by returning the first number instead of 0 + } +} + +int main() { + int num1, num2; + + printf("Enter two numbers: "); + scanf("%d %d", &num1, &num2); + + printf("Sum: %d\n", sum(num1, num2)); + printf("Difference: %d\n", difference(num1, num2)); + printf("Product: %d\n", product(num1, num2)); + printf("Quotient: %.2f\n", divide(num1, num2)); + + return 0; +} + +############################################################################################################ +############################################################################################################ +############################################################################################################ +############################################################################################################ +############################################################################################################ \ No newline at end of file From f55aebec749085f232166b4104f146b0c03f774c Mon Sep 17 00:00:00 2001 From: Jonas August <84895687+J2000A@users.noreply.github.com> Date: Mon, 22 May 2023 15:37:48 +0200 Subject: [PATCH 029/131] Delete APIKEY.yaml --- .../openai-text-completion/APIKEY.yaml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml diff --git a/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml b/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml deleted file mode 100644 index 11bfadb275..0000000000 --- a/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml +++ /dev/null @@ -1,3 +0,0 @@ -organisation: org-J3GRsachQEfeSZXv7KpFj3vD -api-key: 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 - From d35ef36e0ccaec57a6532c03e140dcb260be0d98 Mon Sep 17 00:00:00 2001 From: Jonas August <84895687+J2000A@users.noreply.github.com> Date: Mon, 22 May 2023 15:38:05 +0200 Subject: [PATCH 030/131] Update .gitignore --- .../openai-text-completion/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/openai-text-completion/.gitignore b/scripts/incremental-test-generation/openai-text-completion/.gitignore index ed91ac7eef..a3ba8c66d2 100644 --- a/scripts/incremental-test-generation/openai-text-completion/.gitignore +++ b/scripts/incremental-test-generation/openai-text-completion/.gitignore @@ -1 +1 @@ -APIKEY.txt +APIKEY.yaml From 28f5976b3320c59230c20e1e2d9f93ec162d227b Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 23 May 2023 20:48:36 +0200 Subject: [PATCH 031/131] Bugfix --- src/transform/evalAssert.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index 28d400a0b6..8265c72a5f 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -29,7 +29,7 @@ module EvalAssert = struct let goblintCheck () = GobConfig.get_bool "trans.goblint-check" (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) - let ass () = if goblintCheck then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) in + let ass () = if goblintCheck () then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) let atomicBegin = makeVarinfo true "__VERIFIER_atomic_begin" (TVoid []) let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) @@ -61,8 +61,8 @@ module EvalAssert = struct match (ask ~node loc).f (Queries.Invariant context) with | `Lifted e -> let es = WitnessUtil.InvariantExp.process_exp e in - let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv ass); ("exp", Fe e)]) es in - if surroundByAtomic && not goblintCheck then + let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in + if surroundByAtomic && not (goblintCheck ()) then let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in abegin :: (asserts @ [aend]) From 453da2cb992c5e8e1869f5d57aceb44bea05e2ec Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 23 May 2023 20:49:01 +0200 Subject: [PATCH 032/131] Testing ML --- .../openai-text-completion/ml.py | 4 +-- .../outputs/20230523-01.txt | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt diff --git a/scripts/incremental-test-generation/openai-text-completion/ml.py b/scripts/incremental-test-generation/openai-text-completion/ml.py index d8b10ffc8d..a5156bb862 100644 --- a/scripts/incremental-test-generation/openai-text-completion/ml.py +++ b/scripts/incremental-test-generation/openai-text-completion/ml.py @@ -5,7 +5,7 @@ # [TODO] run "sudo pip install openai" # # [TODO] get api key and store it in a yaml file: -# organisation: .... +# organisation: ... # api-key: ... # api_key_path = os.path.expanduser("~/BA/Goblint-Repo/analyzer/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml") @@ -70,7 +70,7 @@ model="gpt-3.5-turbo", n = 3, messages=[ - {"role": "user", "content": "Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code and introduce bugs. But the code should still compile. Here the C code: " + input}, + {"role": "user", "content": "Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code, introduce bugs or do any other modifications that might happen in a development workflow. But the code should still compile. Explain shortly what you have changed. Here the C code: " + input}, ] ) diff --git a/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt b/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt new file mode 100644 index 0000000000..cf4b576f82 --- /dev/null +++ b/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt @@ -0,0 +1,25 @@ +Promt: Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code, introduce bugs or do any other modifications that might happen in a development workflow. But the code should still compile. Explain shortly what you have changed. Here the C code: +################################################################################## +#include + +int main() { + int num1, num2; + + // User input + printf("Enter two numbers: "); + scanf("%d %d", &num1, &num2); + + // Calculation and output + printf("Sum: %d\n", num1 + num2); + printf("Difference: %d\n", num1 - num2); + printf("Product: %d\n", num1 * num2); + + if (num2 != 0) { + printf("Quotient: %.2f\n", (float)num1 / num2); + } else { + printf("Error: Division by zero is not allowed.\n"); + } + + return 0; +} +################################################################################## From f99fc765747b4aa728932cecb2160a3124e7b8ab Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 23 May 2023 20:50:07 +0200 Subject: [PATCH 033/131] Script for adding SUCCESS or UNDEFINED to checks --- .../test-generation/add-check-comments.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/add-check-comments.py diff --git a/scripts/incremental-test-generation/test-generation/add-check-comments.py b/scripts/incremental-test-generation/test-generation/add-check-comments.py new file mode 100644 index 0000000000..f2b166dcbb --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/add-check-comments.py @@ -0,0 +1,48 @@ +# Adds "//UNDEFINED" or "//SUCCESS" to the Goblint checks __goblint_check(exp); +# Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´ + +import sys +import re + +def process_file(file_name, option): + if option == '-u': + print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") + elif option == '-s': + print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") + + try: + with open(file_name, 'r+') as file: + lines = file.readlines() + modified_lines = [] + + for line in lines: + if '__goblint_check(' in line: + match = re.search(r'(__goblint_check\(.*?\);)', line) + if match: + modified_line = match.group(1) + if option == '-u': + modified_line += ' //UNDEFINED' + elif option == '-s': + modified_line += ' //SUCCESS' + line = line.replace(match.group(1), modified_line) + modified_lines.append(line) + + file.seek(0) + file.writelines(modified_lines) + file.truncate() + print("Processing complete. Modified lines have been added.") + except FileNotFoundError: + print("Error: File not found.") + except: + print("An error occurred while processing the file.") + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Error: Invalid number of arguments. Provide first a file and then -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") + else: + file_name = sys.argv[1] + option = sys.argv[2] + if option not in ['-u', '-s']: + print("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") + else: + process_file(file_name, option) From 36a2b21033baf1f0e9d5540f9eab809c64604585 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 24 May 2023 20:05:55 +0200 Subject: [PATCH 034/131] Use argparser --- .../test-generation/add-check-comments.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/add-check-comments.py b/scripts/incremental-test-generation/test-generation/add-check-comments.py index f2b166dcbb..af5b29aeb1 100644 --- a/scripts/incremental-test-generation/test-generation/add-check-comments.py +++ b/scripts/incremental-test-generation/test-generation/add-check-comments.py @@ -1,17 +1,20 @@ -# Adds "//UNDEFINED" or "//SUCCESS" to the Goblint checks __goblint_check(exp); -# Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´ +# Adds "//UNDEFINED" or "//SUCCESS" to the Goblint checks "__goblint_check(exp);". +# Stores the file with the same name when adding "//SUCCESS". +# Stores the file with the appendix _u when adding "//UNDEFINED". +# Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´. +import argparse import sys import re def process_file(file_name, option): - if option == '-u': + if option == 'u': print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") - elif option == '-s': + elif option == 's': print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") try: - with open(file_name, 'r+') as file: + with open(file_name, 'r') as file: lines = file.readlines() modified_lines = [] @@ -27,22 +30,28 @@ def process_file(file_name, option): line = line.replace(match.group(1), modified_line) modified_lines.append(line) - file.seek(0) - file.writelines(modified_lines) - file.truncate() - print("Processing complete. Modified lines have been added.") + if option == '-u': + new_file_name = file_name.rsplit('.', 1)[0] + '_u.c' + with open(new_file_name, 'w') as new_file: + new_file.writelines(modified_lines) + print("Processing complete. Modified lines have been added to the new file:", new_file_name) + elif option == '-s': + with open(file_name, 'w') as file: + file.writelines(modified_lines) + print("Processing complete. Modified lines have been added to the existing file:", file_name) except FileNotFoundError: print("Error: File not found.") except: print("An error occurred while processing the file.") if __name__ == "__main__": - if len(sys.argv) != 3: - print("Error: Invalid number of arguments. Provide first a file and then -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - else: - file_name = sys.argv[1] - option = sys.argv[2] - if option not in ['-u', '-s']: - print("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - else: - process_file(file_name, option) + parser = argparse.ArgumentParser() + parser.add_argument("file", help="file name") + parser.add_argument("-u", "--undefined", action="store_true", help="option for //UNDEFINED") + parser.add_argument("-s", "--success", action="store_true", help="option for //SUCCESS") + args = parser.parse_args() + + if not (args.undefined or args.success): + parser.error("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") + + process_file(args.file, "u" if args.undefined else "s") From 6c1ae58e6e50a97b5c79aa6e477a5c0e9fb61a22 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 24 May 2023 21:19:35 +0200 Subject: [PATCH 035/131] Add gitignore --- scripts/incremental-test-generation/test-generation/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/.gitignore diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore new file mode 100644 index 0000000000..8172810096 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/.gitignore @@ -0,0 +1,2 @@ +test* +meta* From 09d91ca45459adfffece3fa00ef8760b35adf308 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 24 May 2023 21:20:52 +0200 Subject: [PATCH 036/131] Refactoring --- .../test-generation/add-check-comments.py | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/add-check-comments.py b/scripts/incremental-test-generation/test-generation/add-check-comments.py index af5b29aeb1..e4875987fa 100644 --- a/scripts/incremental-test-generation/test-generation/add-check-comments.py +++ b/scripts/incremental-test-generation/test-generation/add-check-comments.py @@ -7,42 +7,37 @@ import sys import re -def process_file(file_name, option): +def process_file(file_path, option): if option == 'u': print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") elif option == 's': print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") - try: - with open(file_name, 'r') as file: - lines = file.readlines() - modified_lines = [] - - for line in lines: - if '__goblint_check(' in line: - match = re.search(r'(__goblint_check\(.*?\);)', line) - if match: - modified_line = match.group(1) - if option == '-u': - modified_line += ' //UNDEFINED' - elif option == '-s': - modified_line += ' //SUCCESS' - line = line.replace(match.group(1), modified_line) - modified_lines.append(line) - - if option == '-u': - new_file_name = file_name.rsplit('.', 1)[0] + '_u.c' - with open(new_file_name, 'w') as new_file: - new_file.writelines(modified_lines) - print("Processing complete. Modified lines have been added to the new file:", new_file_name) - elif option == '-s': - with open(file_name, 'w') as file: - file.writelines(modified_lines) - print("Processing complete. Modified lines have been added to the existing file:", file_name) - except FileNotFoundError: - print("Error: File not found.") - except: - print("An error occurred while processing the file.") + with open(file_path, 'r') as file: + lines = file.readlines() + modified_lines = [] + + for line in lines: + if '__goblint_check(' in line: + match = re.search(r'(__goblint_check\(.*?\);)', line) + if match: + modified_line = match.group(1) + if option == '-u': + modified_line += ' //UNDEFINED' + elif option == '-s': + modified_line += ' //SUCCESS' + line = line.replace(match.group(1), modified_line) + modified_lines.append(line) + + if option == '-u': + new_file_name = file_path.rsplit('.', 1)[0] + '_u.c' + with open(new_file_name, 'w') as new_file: + new_file.writelines(modified_lines) + print("Processing complete. Modified lines have been added to the new file:", new_file_name) + elif option == '-s': + with open(file_path, 'w') as file: + file.writelines(modified_lines) + print("Processing complete. Modified lines have been added to the existing file:", file_path) if __name__ == "__main__": parser = argparse.ArgumentParser() From aea65aaacf3f9d54271a1b6d6eda71fb1bb1fdca Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 24 May 2023 21:21:03 +0200 Subject: [PATCH 037/131] Implementing add-check --- .../test-generation/add-check.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/add-check.py diff --git a/scripts/incremental-test-generation/test-generation/add-check.py b/scripts/incremental-test-generation/test-generation/add-check.py new file mode 100644 index 0000000000..55e332f383 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/add-check.py @@ -0,0 +1,52 @@ +# Takes a file and generates the goblint checks +# Stores the file with an additional "_c" +# When there is a compilation error the process writes [COMPILE_FAIL] into the meta data file for the given index + +import argparse +import subprocess +import sys + +import yaml + +def process_file(file_path, index, goblint_path, meta_path): + command = [ + goblint_path, + "--enable", + "trans.goblint-check", + "--set", + "trans.activated", + '["assert"]', + "--set", + "trans.output", + file_path.rsplit('.', 1)[0] + '_c.c', + file_path + ] + + result = subprocess.run(command) + + compiling = result.returncode == 0 + + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data["p_" + str(index)]['compilation'] = compiling + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file) + + if not compiling : + print(f"Error compiling program with index {index}.") + sys.exit(-1) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate __goblint_check() for a C file. When not compiling write [NOT COMPILING] in the meta file') + + # Add the required arguments + parser.add_argument('file', help='Path to the C file') + parser.add_argument('index', help='Index of the file (needed for meta data)') + parser.add_argument('goblint', help='Path to the Goblint executable') + parser.add_argument('meta', help='Path to the meta data file') + + # Parse the command-line arguments + args = parser.parse_args() + + # Call the process_file function with the provided arguments + process_file(args.file, args.index, args.goblint, args.meta) From 11fb212eddd9056c1301d20a89c8cd1da62a52b5 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 12:56:05 +0200 Subject: [PATCH 038/131] Refactoring --- .../{ => util}/add-check-comments.py | 23 +++++++++---------- .../test-generation/{ => util}/add-check.py | 4 ++-- 2 files changed, 13 insertions(+), 14 deletions(-) rename scripts/incremental-test-generation/test-generation/{ => util}/add-check-comments.py (81%) rename scripts/incremental-test-generation/test-generation/{ => util}/add-check.py (91%) diff --git a/scripts/incremental-test-generation/test-generation/add-check-comments.py b/scripts/incremental-test-generation/test-generation/util/add-check-comments.py similarity index 81% rename from scripts/incremental-test-generation/test-generation/add-check-comments.py rename to scripts/incremental-test-generation/test-generation/util/add-check-comments.py index e4875987fa..23260bbd00 100644 --- a/scripts/incremental-test-generation/test-generation/add-check-comments.py +++ b/scripts/incremental-test-generation/test-generation/util/add-check-comments.py @@ -4,13 +4,12 @@ # Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´. import argparse -import sys import re -def process_file(file_path, option): - if option == 'u': +def add_check_comments(file_path: str, undefined_instead_of_success: bool): + if undefined_instead_of_success: print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") - elif option == 's': + else: print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") with open(file_path, 'r') as file: @@ -22,31 +21,31 @@ def process_file(file_path, option): match = re.search(r'(__goblint_check\(.*?\);)', line) if match: modified_line = match.group(1) - if option == '-u': + if undefined_instead_of_success: modified_line += ' //UNDEFINED' - elif option == '-s': + else: modified_line += ' //SUCCESS' line = line.replace(match.group(1), modified_line) modified_lines.append(line) - if option == '-u': + if undefined_instead_of_success: new_file_name = file_path.rsplit('.', 1)[0] + '_u.c' with open(new_file_name, 'w') as new_file: new_file.writelines(modified_lines) print("Processing complete. Modified lines have been added to the new file:", new_file_name) - elif option == '-s': + else: with open(file_path, 'w') as file: file.writelines(modified_lines) print("Processing complete. Modified lines have been added to the existing file:", file_path) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("file", help="file name") - parser.add_argument("-u", "--undefined", action="store_true", help="option for //UNDEFINED") - parser.add_argument("-s", "--success", action="store_true", help="option for //SUCCESS") + parser.add_argument("file", help="Path to the C file") + parser.add_argument("-u", "--undefined", action="store_true", help="Option for //UNDEFINED") + parser.add_argument("-s", "--success", action="store_true", help="ption for //SUCCESS") args = parser.parse_args() if not (args.undefined or args.success): parser.error("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - process_file(args.file, "u" if args.undefined else "s") + add_check_comments(args.file, args.undefined) diff --git a/scripts/incremental-test-generation/test-generation/add-check.py b/scripts/incremental-test-generation/test-generation/util/add-check.py similarity index 91% rename from scripts/incremental-test-generation/test-generation/add-check.py rename to scripts/incremental-test-generation/test-generation/util/add-check.py index 55e332f383..3dda57cd82 100644 --- a/scripts/incremental-test-generation/test-generation/add-check.py +++ b/scripts/incremental-test-generation/test-generation/util/add-check.py @@ -8,7 +8,7 @@ import yaml -def process_file(file_path, index, goblint_path, meta_path): +def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): command = [ goblint_path, "--enable", @@ -49,4 +49,4 @@ def process_file(file_path, index, goblint_path, meta_path): args = parser.parse_args() # Call the process_file function with the provided arguments - process_file(args.file, args.index, args.goblint, args.meta) + add_check(args.file, args.index, args.goblint, args.meta) From 213ee8d440a38a5a610b4d911d6c7aa3dc517c13 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 13:38:50 +0200 Subject: [PATCH 039/131] Create generate mutations --- .../generators/generate_mutations.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/generators/generate_mutations.py diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py new file mode 100644 index 0000000000..be40e489cf --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -0,0 +1,148 @@ +# Generate all possible mutations of a program and +# Assume there is a meta.yaml file with content "n: >=0" + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import yaml + +generate_type_mutation = "MUTATION" +seperator = "--------------------" + +class Mutations: + def __init__(self, rfb=False, uoi=False, ror=False, cr=False, rt=False, lcr=False): + self.rfb = rfb + self.rfb_s = "remove-function-body" + self.uoi = uoi + self.uoi_s = "unary-operator-inversion" + self.ror = ror + self.ror_s = "relational-operator-replacement" + self.cr = cr + self.cr_s = "constant-replacement" + self.rt = rt + self.rt_s = "remove-thread" + self.lcr = lcr + self.lcr_s = "logical-connector-replacement" + +def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + index: int = yaml_data["n"] + + if mutations.rfb: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rfb_s, index) + if mutations.uoi: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.uoi_s, index) + if mutations.ror: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.ror_s, index) + if mutations.cr: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.cr_s, index) + if mutations.rt: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) + if mutations.lcr: + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) + +def _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): + print(seperator) + print(f"[{generate_type_mutation}] {mutation_name}") + lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path) + for lines in lineGroups: + index += 1 + new_path = _make_copy(program_path, index) + _apply_mutation(clang_tidy_path, mutation_name, lines, new_path, index) + _write_meta_data(meta_path, new_path, index, mutation_name, lines) + return index + +def _make_copy(program_path, index): + new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' + shutil.copy2(program_path, new_path) + return new_path + +def _get_line_groups(clang_tidy_path, mutation_name, program_path): + #TODO Handle [MACRO] tags + command = [ + clang_tidy_path, + "-checks=-*,readability-" + mutation_name, + program_path, + "--" + ] + + result = subprocess.run(command, text=True, capture_output=True) + print(f"[CHECK] Check mutation {mutation_name} with return code {result.returncode}") + if result.returncode != 0: + print(result.stdout) + print("ERROR Running Clang") + sys.exit(-1) + + line_groups = [] + pattern = r":(\d+):.*\[readability-" + mutation_name + r"\]" + + for line in result.stdout.splitlines(): + match = re.search(pattern, line) + if match: + line_groups.append([int(match.group(1))]) + + print(f"[CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") + return line_groups + +def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): + lines_mapped = [[x,x] for x in lines] + line_filter = [{"name": program_path, "lines": lines_mapped}] + line_filter_json = json.dumps(line_filter) + command = [ + clang_tidy_path, + "-checks=-*,readability-" + mutation_name, + "-fix", + "--fix-errors", + "-line-filter=" + line_filter_json, + program_path, + "--" + ] + + result = subprocess.run(command, text=True, capture_output=True) + print(f"[{index}] Run mutation {mutation_name} on lines {lines} with return code {result.returncode}") + if result.returncode != 0: + print(result.stdout) + print("ERROR Running Clang") + sys.exit(-1) + +def _write_meta_data(meta_path, new_path, index, mutation_name, lines): + name = os.path.basename(new_path) + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data["n"] = index + yaml_data[name] = { + "type": generate_type_mutation, + "sub_type": mutation_name, + "lines": lines + } + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file, sort_keys=False) + +def add_mutation_options(parser): + parser.add_argument("-rfb", "--remove-function-body", action="store_true", help="Option for \"remove function body\" mutation") + parser.add_argument("-uoi", "--unary-operator-inversion", action="store_true", help="Option for \"unary operator inversion\" mutation") + parser.add_argument("-ror", "--relational-operator-replacement", action="store_true", help="Option for \"relational operator replacement\" mutation") + parser.add_argument("-cr", "--constant-replacement", action="store_true", help="Option for \"constant replacement\" mutation") + parser.add_argument("-rt", "--remove-thread", action="store_true", help="Option for \"remove thread\" mutation") + parser.add_argument("-lcr", "--logical-connector-replacement", action="store_true", help="Option for \"logical connector replacement\" mutation") + +def get_mutations_from_args(args): + return Mutations(args.remove_function_body, args.unary_operator_inversion, + args.relational_operator_replacement, args.constant_replacement, + args.remove_thread, args.logical_connector_replacement) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate all possible mutations of a program.") + parser.add_argument("program", help="Path to the C program") + parser.add_argument("clang_tidy", help="Path to the modified clang-tidy executable") + parser.add_argument("meta", help="Path to the meta data file") + add_mutation_options(parser) + + args = parser.parse_args() + mutations = get_mutations_from_args(args) + generate_mutations(args.program, args.clang_tidy, args.meta, mutations) From d3cf3effd1cb6f239807c030b08b0881164e0d05 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 16:06:56 +0200 Subject: [PATCH 040/131] Add new check to create wrapper for remove thread --- .../clang-mutations/MUTATIONS.md | 26 +++++++++++-- .../clang-mutations/RemoveThreadCheck.cpp | 4 +- .../RemoveThreadWrapperCheck.cpp | 38 +++++++++++++++++++ .../RemoveThreadWrapperCheck.h | 23 +++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp create mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h diff --git a/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md b/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md index 83a493a3a7..84d1a46310 100644 --- a/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md +++ b/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md @@ -69,13 +69,33 @@ int d = MY_MACRO_0; ## Remove Thread - RT **readability-remove-thread**
-Replaces a `pthread_create` call with the function call itself. The arguments of the function call are kept. Symbols like `*` or `&` in front of the function name are ignored. +Replaces a `pthread_create` call with the function call itself. Additionally `0;` is added for the case that the result was checked in the form `result = pthread_create()` The arguments of the function call are kept. Symbols like `*` or `&` in front of the function name are ignored. ``` -pthread_create(&thread, &attr, thread_function, NULL); +result = pthread_create(&thread, &attr, thread_function, NULL); ``` `clang-tidy -checks=-*,readability-remove-thread -fix pthread.c --` ``` -thread_function(NULL) /* [MUTATION][RT] Thread creation was substituted with function call */; +result = 0; thread_function(NULL) /* [MUTATION][RT][FUNCTION_NAME][thread_function] Thread creation was substituted with function call */; +``` + +### Remove Thread Wrapper - RTW +**readability-remove-thread-wrapper**
+Wraps the given Function Name for a pthread_create call. This should be run before `remove-thread`. The function name has to be passed. +``` +void *threadFunction(void *arg) { + //... +} +``` +`clang-tidy -checks=-*,readability-remove-thread-wrapper -config="{CheckOptions: {readability-remove-thread-wrapper.WrapFunctionName: 'threadFunction'}}" -fix pthread.c --` +``` +void *threadFunction(void *arg) { + //... +} +int threadFunction_wrap(void *arg) { + /*[MUTATION][RTW] Wrapped function for remove-thread */ + threadFunction(arg); + return 0; +} ``` ## Logical Connector Replacement - LCR diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp index 7e1748debd..ec76b96fff 100644 --- a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp +++ b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp @@ -35,8 +35,8 @@ void RemoveThreadCheck::check(const MatchFinder::MatchResult &Result) { // Get the argument std::string Argument = parseArgument(Arguments[3]); // Create the replacement - std::string Replacement = FunctionName + "(" + Argument + ") /* [MUTATION][RT] Thread creation was substituted with function call */"; - diag(Start, "[MUTATION][RT] Thread creation was substituted with function call %0") + std::string Replacement = FunctionName + "_wrap(" + Argument + ") /* [MUTATION][RT][FUNCTION_NAME][" + FunctionName + "] Thread creation was substituted with function call */"; + diag(Start, "[MUTATION][RT][FUNCTION_NAME][%0] Thread creation was substituted with function call %0_wrapper") << FunctionName << FixItHint::CreateReplacement(Range, Replacement); } diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp new file mode 100644 index 0000000000..f900586be6 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp @@ -0,0 +1,38 @@ +#include "RemoveThreadWrapperCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void RemoveThreadWrapperCheck::registerMatchers(MatchFinder *Finder) { + if (WrapFunctionName.empty()) { + exit(-1); + } else { + Finder->addMatcher(functionDecl(hasName(WrapFunctionName), isDefinition()).bind("wrap_function"), this); + } +} + +void RemoveThreadWrapperCheck::check(const MatchFinder::MatchResult &Result) { + auto *MatchedDecl = Result.Nodes.getNodeAs("wrap_function"); + + std::string Replacement = "\nint " + WrapFunctionName + "_wrap(void *arg) {\n" + + "\t/*[MUTATION][RTW] Wrapped function for remove-thread */\n" + + "\t" + WrapFunctionName + "(arg);\n" + + "\treturn 0;\n}"; + + // Get locations + SourceLocation Start = MatchedDecl->getEndLoc().getLocWithOffset(1); + SourceLocation End = MatchedDecl->getEndLoc().getLocWithOffset(1); + auto Range = CharSourceRange::getCharRange(Start, End); + diag(Start, "[MUTATION][RTW] Function %0 has been wrapped") + << MatchedDecl + << FixItHint::CreateReplacement(Range, Replacement); +} + +void RemoveThreadWrapperCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "WrapFunctionName", WrapFunctionName); +} + +} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h new file mode 100644 index 0000000000..91e1989482 --- /dev/null +++ b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h @@ -0,0 +1,23 @@ +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Create wrapper for remove thread check +class RemoveThreadWrapperCheck : public ClangTidyCheck { + const std::string WrapFunctionName; + +public: + RemoveThreadWrapperCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + WrapFunctionName(Options.get("WrapFunctionName", "")) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H \ No newline at end of file From ab3b97332e15d41dd0464132c94161272418d907 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 16:07:04 +0200 Subject: [PATCH 041/131] Bugfix --- .../generators/generate_mutations.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index be40e489cf..2b7c3190ba 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -44,7 +44,7 @@ def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): if mutations.rt: index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) if mutations.lcr: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) + index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.lcr_s, index) def _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): print(seperator) @@ -80,11 +80,28 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path): line_groups = [] pattern = r":(\d+):.*\[readability-" + mutation_name + r"\]" + macro_pattern = r"\[MACRO\]\[(.*?)\]" + macro_lines = {} for line in result.stdout.splitlines(): match = re.search(pattern, line) if match: - line_groups.append([int(match.group(1))]) + macro_match = re.search(macro_pattern, line) + if macro_match: + macro_name = macro_match.group(1) + line_number = int(match.group(1)) + if macro_name not in macro_lines: + macro_lines[macro_name] = [line_number] + else: + macro_lines[macro_name].append(line_number) + else: + line_groups.append([int(match.group(1))]) + + for macro_name, lines in macro_lines.items(): + line_groups.append(lines) + + # Remove duplicate line groups + line_groups = [list(x) for x in set(tuple(x) for x in line_groups)] print(f"[CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") return line_groups From 5a78a0526f57264bf4c3596dcbb6e65c099d8875 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 17:04:54 +0200 Subject: [PATCH 042/131] Add handling of thread function wrapping --- .../generators/generate_mutations.py | 79 +++++++++++++++++-- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 2b7c3190ba..3b46ccf650 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -34,25 +34,32 @@ def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): index: int = yaml_data["n"] if mutations.rfb: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rfb_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rfb_s, index) if mutations.uoi: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.uoi_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.uoi_s, index) if mutations.ror: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.ror_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.ror_s, index) if mutations.cr: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.cr_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.cr_s, index) if mutations.rt: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) if mutations.lcr: - index = _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.lcr_s, index) + index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.lcr_s, index) -def _basic_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): +def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): print(seperator) print(f"[{generate_type_mutation}] {mutation_name}") lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path) for lines in lineGroups: index += 1 new_path = _make_copy(program_path, index) + if mutation_name == mutations.rt_s: + # When Remove Thread create wrapper an then apply the mutations + if len(lines) != 1: + # Needed to prevent conflicts on generating wrappers + print("ERROR When applying remove_thread there always should be exactly one line") + function_name = _get_thread_function_name(clang_tidy_path, lines, new_path, index) + _wrap_thread_function(clang_tidy_path, new_path, function_name, index) _apply_mutation(clang_tidy_path, mutation_name, lines, new_path, index) _write_meta_data(meta_path, new_path, index, mutation_name, lines) return index @@ -75,6 +82,7 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path): print(f"[CHECK] Check mutation {mutation_name} with return code {result.returncode}") if result.returncode != 0: print(result.stdout) + print(result.stderr) print("ERROR Running Clang") sys.exit(-1) @@ -124,6 +132,63 @@ def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): print(f"[{index}] Run mutation {mutation_name} on lines {lines} with return code {result.returncode}") if result.returncode != 0: print(result.stdout) + print(result.stderr) + print("ERROR Running Clang") + sys.exit(-1) + +def _get_thread_function_name(clang_tidy_path, lines, program_path, index): + lines_mapped = [[x,x] for x in lines] + line_filter = [{"name": program_path, "lines": lines_mapped}] + line_filter_json = json.dumps(line_filter) + command = [ + clang_tidy_path, + "-checks=-*,readability-" + mutations.rt_s, + "-line-filter=" + line_filter_json, + program_path, + "--" + ] + result = subprocess.run(command, text=True, capture_output=True) + print(f"[{index}][WRAP] Check function name for wrapping thread function with return code {result.returncode}") + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + print("ERROR Running Clang") + sys.exit(-1) + + print(result.stdout) + + function_name_pattern = r"\[FUNCTION_NAME\]\[(.*?)\]" + function_name = None + + for line in result.stdout.splitlines(): + function_name_match = re.search(function_name_pattern, line) + if function_name_match: + function_name = function_name_match.group(1) + break + + print(f"[{index}][WRAP RESULT] Found the thread function name {function_name}") + return function_name + +def _wrap_thread_function(clang_tidy_path, program_path, function_name, index): + if function_name == None: + print(f"[{index}][WRAP FIX] No function name was provided. Hope the program will compile without wrapping") + return + + check_options = {"CheckOptions": {"readability-remove-thread-wrapper.WrapFunctionName": function_name}} + check_options_json = json.dumps(check_options) + command = [ + clang_tidy_path, + "-checks=-*,readability-remove-thread-wrapper", + "-config=" + check_options_json, + "-fix", + program_path, + "--" + ] + result = subprocess.run(command, text=True, capture_output=True) + print(f"[{index}][WRAP FIX] Apply the wrapping of {function_name} with return code {result.returncode}") + if result.returncode != 0: + print(result.stdout) + print(result.stderr) print("ERROR Running Clang") sys.exit(-1) From ca13c9e42344039789f40164c713c2be34d46f60 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 17:20:47 +0200 Subject: [PATCH 043/131] Refactor file name --- .../test-generation/util/{add-check.py => add_check.py} | 0 .../util/{add-check-comments.py => add_check_comments.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/incremental-test-generation/test-generation/util/{add-check.py => add_check.py} (100%) rename scripts/incremental-test-generation/test-generation/util/{add-check-comments.py => add_check_comments.py} (100%) diff --git a/scripts/incremental-test-generation/test-generation/util/add-check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/add-check.py rename to scripts/incremental-test-generation/test-generation/util/add_check.py diff --git a/scripts/incremental-test-generation/test-generation/util/add-check-comments.py b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/add-check-comments.py rename to scripts/incremental-test-generation/test-generation/util/add_check_comments.py From 540565bfe4884108646e7952af88cfe1b0f4119a Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 19:11:16 +0200 Subject: [PATCH 044/131] Added gitignore --- .../test-generation/generators/.gitignore | 1 + .../incremental-test-generation/test-generation/util/.gitignore | 1 + 2 files changed, 2 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/generators/.gitignore create mode 100644 scripts/incremental-test-generation/test-generation/util/.gitignore diff --git a/scripts/incremental-test-generation/test-generation/generators/.gitignore b/scripts/incremental-test-generation/test-generation/generators/.gitignore new file mode 100644 index 0000000000..ed8ebf583f --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/util/.gitignore b/scripts/incremental-test-generation/test-generation/util/.gitignore new file mode 100644 index 0000000000..ed8ebf583f --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file From 13875a11e329f0cf7dbd38b590e7e42c06803633 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 19:12:10 +0200 Subject: [PATCH 045/131] Bugfixes --- .../test-generation/generators/generate_mutations.py | 10 +++++----- .../test-generation/util/add_check.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 3b46ccf650..0725baf3d7 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -46,6 +46,8 @@ def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): if mutations.lcr: index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.lcr_s, index) + return index + def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): print(seperator) print(f"[{generate_type_mutation}] {mutation_name}") @@ -53,7 +55,7 @@ def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mut for lines in lineGroups: index += 1 new_path = _make_copy(program_path, index) - if mutation_name == mutations.rt_s: + if mutation_name == Mutations().rt_s: # When Remove Thread create wrapper an then apply the mutations if len(lines) != 1: # Needed to prevent conflicts on generating wrappers @@ -142,7 +144,7 @@ def _get_thread_function_name(clang_tidy_path, lines, program_path, index): line_filter_json = json.dumps(line_filter) command = [ clang_tidy_path, - "-checks=-*,readability-" + mutations.rt_s, + "-checks=-*,readability-" + Mutations().rt_s, "-line-filter=" + line_filter_json, program_path, "--" @@ -155,8 +157,6 @@ def _get_thread_function_name(clang_tidy_path, lines, program_path, index): print("ERROR Running Clang") sys.exit(-1) - print(result.stdout) - function_name_pattern = r"\[FUNCTION_NAME\]\[(.*?)\]" function_name = None @@ -197,7 +197,7 @@ def _write_meta_data(meta_path, new_path, index, mutation_name, lines): with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) yaml_data["n"] = index - yaml_data[name] = { + yaml_data[f"p_{index}"] = { "type": generate_type_mutation, "sub_type": mutation_name, "lines": lines diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index 3dda57cd82..ee59604d33 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -22,7 +22,7 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): file_path ] - result = subprocess.run(command) + result = subprocess.run(command, text=True, capture_output=True) compiling = result.returncode == 0 @@ -33,6 +33,8 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): yaml.safe_dump(yaml_data, file) if not compiling : + print(result.stdout) + print(result.stderr) print(f"Error compiling program with index {index}.") sys.exit(-1) From d0760eb54452055b229242165fa194ccd1cd8406 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 19:12:22 +0200 Subject: [PATCH 046/131] Add vervose option --- .../util/add_check_comments.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py index 23260bbd00..bc276baf89 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py @@ -6,11 +6,12 @@ import argparse import re -def add_check_comments(file_path: str, undefined_instead_of_success: bool): - if undefined_instead_of_success: - print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") - else: - print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") +def add_check_comments(file_path: str, undefined_instead_of_success: bool, verbose = False): + if verbose: + if undefined_instead_of_success: + print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") + else: + print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") with open(file_path, 'r') as file: lines = file.readlines() @@ -32,11 +33,13 @@ def add_check_comments(file_path: str, undefined_instead_of_success: bool): new_file_name = file_path.rsplit('.', 1)[0] + '_u.c' with open(new_file_name, 'w') as new_file: new_file.writelines(modified_lines) - print("Processing complete. Modified lines have been added to the new file:", new_file_name) + if verbose: + print("Processing complete. Modified lines have been added to the new file:", new_file_name) else: with open(file_path, 'w') as file: file.writelines(modified_lines) - print("Processing complete. Modified lines have been added to the existing file:", file_path) + if verbose: + print("Processing complete. Modified lines have been added to the existing file:", file_path) if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -48,4 +51,4 @@ def add_check_comments(file_path: str, undefined_instead_of_success: bool): if not (args.undefined or args.success): parser.error("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - add_check_comments(args.file, args.undefined) + add_check_comments(args.file, args.undefined, True) From 5d06a6d87529dffb8e36ad73a473dbc4de3f5d77 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 25 May 2023 19:12:38 +0200 Subject: [PATCH 047/131] Add Script for generating programs --- .../test-generation/util/generate_programs.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/util/generate_programs.py diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py new file mode 100644 index 0000000000..502e040dc8 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -0,0 +1,108 @@ +import argparse +import os +import shutil +import sys +sys.path.append("..") +from generators.generate_mutations import * +from util.add_check import * +from util.add_check_comments import * + +generate_type_source = "SOURCE" + +def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git): + # Clean working directory + if os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + os.makedirs(temp_dir) + # Create Meta file + meta_path = os.path.join(temp_dir,'meta.yaml') + with open(meta_path, 'w') as outfile: + yaml.dump({'n': 0, 'p_0': {'type': generate_type_source}}, outfile) + # Copy the source program into the temp dir + program_path = os.path.join(temp_dir, 'p.c') + shutil.copy2(source_path, program_path) + program_0_path = os.path.join(temp_dir, 'p_0.c') + shutil.copy2(source_path, program_0_path) + + index = 0 + if enable_mutations: + index = generate_mutations(program_path, clang_tidy_path, meta_path, mutations) + + if enable_ml: + pass + + if enable_git: + pass + + # Add checks with comments + print(seperator) + index += 1 + for i in range(index): + if i % 9 == 0: + print(f"Generating goblint checks [{i+1}/{index}]") + file_path = os.path.join(temp_dir, f"p_{i}.c") + add_check(file_path, i, goblint_path, meta_path) + file_path = os.path.join(temp_dir, f"p_{i}_c.c") + if i == 0: + add_check_comments(file_path, True) + add_check_comments(file_path, False) + print(f"Generating goblint checks [DONE]") + + # Check how many and which files were not compiling + print(seperator) + print("Check if the files compiled...") + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + failed_count = 0 + failed_compilation_keys = [] + for key, value in yaml_data.items(): + if isinstance(value, dict) and 'compilation' in value and value['compilation'] is False: + failed_count += 1 + failed_compilation_keys.append(key) + if failed_count == 0: + print("All files compiled succesfully") + else: + print(f"There where {failed_count} files not compiling: {failed_compilation_keys}") + + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate programs in the working directory') + parser.add_argument('source_path', help='Path to the original program provided by the user') + parser.add_argument('temp_dir', help='Path to the working directory') + parser.add_argument('clang_tidy_path', help='Path to the modified clang-tidy executable') + parser.add_argument('goblint_path', help='Path to the goblint executable') + parser.add_argument('--apikey-path', help='Path to the API') + parser.add_argument('--git-url', help='Git URL') + parser.add_argument('--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') + parser.add_argument('--enable-ml', action='store_true', help='Enable ML') + parser.add_argument('--enable-git', action='store_true', help='Enable Git') + + # Add mutation options + add_mutation_options(parser) + + args = parser.parse_args() + + # At least one generator has to be enabled + if not args.enable_mutations and not args.enable_ml and not args.enable_git: + parser.error("At least one generator has to be enabled (--enable_mutations, --enable-ml, --enable-git)") + + # If using git, only git can be used + if args.enable_git and (args.enable_ml or args.enable_mutations): + parser.error("--enable-git cannot be used with --enable-ml or --enable-mutations") + + # If all mutation options are false, set all to true + mutations = get_mutations_from_args(args) + non_str_attributes = [attr for attr in vars(mutations) if not attr.endswith('_s')] + if all(getattr(mutations, attr) is False for attr in non_str_attributes): + mutations = Mutations(True, True, True, True, True, True) + + # Check required parameters for optional features + if args.enable_ml and not args.apikey_path: + parser.error("--enable-ml requires --apikey-path") + + if args.enable_git and not args.git_url: + parser.error("--enable-git requires --git-url") + + gernerate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, args.git_url, mutations, args.enable_mutations, args.enable_ml, args.enable_git) From 539ae63d853d246a70197e590fd78e983286d9c1 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 26 May 2023 13:53:03 +0200 Subject: [PATCH 048/131] Move sample files --- .../{clang-mutations => }/sample-files/comparisons.c | 0 .../{clang-mutations => }/sample-files/constants.c | 0 .../{clang-mutations => }/sample-files/functions.c | 0 .../{clang-mutations => }/sample-files/pthread.c | 0 .../{clang-mutations => }/sample-files/threads.c | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename scripts/incremental-test-generation/{clang-mutations => }/sample-files/comparisons.c (100%) rename scripts/incremental-test-generation/{clang-mutations => }/sample-files/constants.c (100%) rename scripts/incremental-test-generation/{clang-mutations => }/sample-files/functions.c (100%) rename scripts/incremental-test-generation/{clang-mutations => }/sample-files/pthread.c (100%) rename scripts/incremental-test-generation/{clang-mutations => }/sample-files/threads.c (100%) diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c b/scripts/incremental-test-generation/sample-files/comparisons.c similarity index 100% rename from scripts/incremental-test-generation/clang-mutations/sample-files/comparisons.c rename to scripts/incremental-test-generation/sample-files/comparisons.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/constants.c b/scripts/incremental-test-generation/sample-files/constants.c similarity index 100% rename from scripts/incremental-test-generation/clang-mutations/sample-files/constants.c rename to scripts/incremental-test-generation/sample-files/constants.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/functions.c b/scripts/incremental-test-generation/sample-files/functions.c similarity index 100% rename from scripts/incremental-test-generation/clang-mutations/sample-files/functions.c rename to scripts/incremental-test-generation/sample-files/functions.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c b/scripts/incremental-test-generation/sample-files/pthread.c similarity index 100% rename from scripts/incremental-test-generation/clang-mutations/sample-files/pthread.c rename to scripts/incremental-test-generation/sample-files/pthread.c diff --git a/scripts/incremental-test-generation/clang-mutations/sample-files/threads.c b/scripts/incremental-test-generation/sample-files/threads.c similarity index 100% rename from scripts/incremental-test-generation/clang-mutations/sample-files/threads.c rename to scripts/incremental-test-generation/sample-files/threads.c From 488519fcb2ade21292f264bfca3383ab5aee9569 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 26 May 2023 14:19:50 +0200 Subject: [PATCH 049/131] Add gitignore --- .../test-generation/util/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/test-generation/util/.gitignore b/scripts/incremental-test-generation/test-generation/util/.gitignore index ed8ebf583f..dc185efb64 100644 --- a/scripts/incremental-test-generation/test-generation/util/.gitignore +++ b/scripts/incremental-test-generation/test-generation/util/.gitignore @@ -1 +1,2 @@ -__pycache__ \ No newline at end of file +__pycache__ +test* From 0742b4f395c3f8fe835086aa518a6ed2ee5c21ef Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 26 May 2023 14:20:04 +0200 Subject: [PATCH 050/131] Add comment with run command --- .../test-generation/util/generate_programs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 502e040dc8..7087f2a289 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -7,6 +7,9 @@ from util.add_check import * from util.add_check_comments import * +# Run for example with: +# python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations + generate_type_source = "SOURCE" def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git): From 0e2dd7e2002d9be9954345008b36365e49517228 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 16:27:22 +0200 Subject: [PATCH 051/131] Writing of generate tests script --- .../test-generation/util/.gitignore | 1 + .../test-generation/util/generate_tests.py | 109 ++++++++++++++++++ .../test-generation/util/util.py | 34 ++++++ tests/incremental/.gitignore | 1 + 4 files changed, 145 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/util/generate_tests.py create mode 100644 scripts/incremental-test-generation/test-generation/util/util.py create mode 100644 tests/incremental/.gitignore diff --git a/scripts/incremental-test-generation/test-generation/util/.gitignore b/scripts/incremental-test-generation/test-generation/util/.gitignore index dc185efb64..9d0dd38eed 100644 --- a/scripts/incremental-test-generation/test-generation/util/.gitignore +++ b/scripts/incremental-test-generation/test-generation/util/.gitignore @@ -1,2 +1,3 @@ __pycache__ test* +99-* diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py new file mode 100644 index 0000000000..10ef3ba936 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -0,0 +1,109 @@ +import argparse +import json +import os +import re +import shutil +import subprocess +import yaml +import sys +import datetime +sys.path.append("..") +from util import * + +def generate_tests(temp_dir, target_dir, precision_test): + # Check the name of the target_dir + directoryName = os.path.basename(target_dir) + pattern = r"\d{2}-\w+" + if not re.match(pattern, directoryName): + print("[ERROR] Target Directory name is not of the format 01-Name (\d{2}-\w+)") + return + + # Clear and create target_dir + if os.path.exists(target_dir): + shutil.rmtree(target_dir) + os.mkdir(target_dir) + + # Read the meta.yaml + meta_path = os.path.join(temp_dir,META_FILENAME) + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + n = yaml_data[META_N] + + source_program_id = 'p_0' + source_program = os.path.join(temp_dir, source_program_id + '_c_u.c') + source_program_precison = os.path.join(temp_dir, source_program_id + '_c.c') + unchanged_count = 0 + for i in range(n + 1): + generated_id = 'p_' + str(i) + generated_program = os.path.join(temp_dir, generated_id + '_c.c') + compilation_success = yaml_data[generated_id][META_COMPILING] + if not compilation_success: + print(f"Generating test files [{i}/{n}] Skipped {i} (Not compiling)") + continue + type = yaml_data[generated_id][META_TYPE] + if type == Generate_Type.SOURCE.value: + continue + if (i-1) % 9 == 0: + print(f"Generating test files [{i}/{n}]") + if type == Generate_Type.MUTATION.value: + sub_type = yaml_data[generated_id][META_SUB_TYPE] + test_name = _format_number(i-1) + '-' + type + '_' + sub_type + # Copy mutated code as the original code + shutil.copy2(generated_program, os.path.join(target_dir, test_name + '.c')) + # Create a patch file + patch_path = os.path.join(target_dir, test_name + '.patch') + command = 'diff -u {} {} > {}'.format( + os.path.join(target_dir, test_name + '.c'), + source_program_precison if precision_test else source_program, + patch_path + ) + result = subprocess.run(command, shell=True) + _fix_patch_file(patch_path, directoryName, test_name + '.c') + if result.returncode in [0, 1]: + if result.returncode == 0: + print(f"[WARNING] There were no changes in the patch for test {i}") + unchanged_count += 1 + yaml_data[generated_id][META_DIFF] = False + else: + yaml_data[generated_id][META_DIFF] = True + else: + raise Exception("Command failed with return code: {}".format(result.returncode)) + # Create a empty config file + #TODO Support other config files + data = {} + with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: + json.dump(data, f) + if type == Generate_Type.ML.value: + #TODO + print ('[ERROR] Generating ML Tests not implemented') + if type == Generate_Type.GIT.value: + #TODO + print ('[ERROR] Generating GIT Tests not implemented') + print(f"Generating test files [DONE]") + + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file) + +def _fix_patch_file(patch_file, folder_name, file_name): + with open(patch_file, 'r') as file: + lines = file.readlines() + + with open(patch_file, 'w') as file: + for line in lines: + if line.startswith('---') or line.startswith('+++'): + line = line.split(' ')[0] + " " + "tests/incremental/" + folder_name + "/" + file_name + "\n" + file.write(line) + +def _format_number(n): + return str(n).zfill(2) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate Test files in the target directory based on the working directory') + parser.add_argument('temp_dir', help='Path to the working directory') + parser.add_argument('target_dir', help='Path to the target directory') + parser.add_argument('-p', '--precision-test', action='store_true', help='Generate tests for precision') + + args = parser.parse_args() + + generate_tests(args.temp_dir, args.target_dir, args.precision_test) diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py new file mode 100644 index 0000000000..4bfaaeea6b --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -0,0 +1,34 @@ +from enum import Enum + + +class Mutations: + def __init__(self, rfb=False, uoi=False, ror=False, cr=False, rt=False, lcr=False): + self.rfb = rfb + self.rfb_s = "remove-function-body" + self.uoi = uoi + self.uoi_s = "unary-operator-inversion" + self.ror = ror + self.ror_s = "relational-operator-replacement" + self.cr = cr + self.cr_s = "constant-replacement" + self.rt = rt + self.rt_s = "remove-thread" + self.lcr = lcr + self.lcr_s = "logical-connector-replacement" + +class Generate_Type(Enum): + SOURCE = 'SOURCE' + MUTATION = 'MUTATION' + ML = 'ML' + GIT = 'GIT' + +SEPERATOR = "--------------------" + +META_FILENAME = 'meta.yaml' +META_N = 'n' +META_COMPILING = 'compilation' +META_DIFF = 'diff' +META_TYPE = 'type' +META_SUB_TYPE = 'sub_type' +META_LINES = 'lines' + diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore new file mode 100644 index 0000000000..1a9f241854 --- /dev/null +++ b/tests/incremental/.gitignore @@ -0,0 +1 @@ +99-* \ No newline at end of file From d9e25e7e7a2e9d452f0791d78b448040289996e9 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 16:31:44 +0200 Subject: [PATCH 052/131] Use common util constants --- .../generators/generate_mutations.py | 34 +++++-------------- .../test-generation/util/add_check.py | 5 +-- .../test-generation/util/generate_programs.py | 13 ++++--- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 0725baf3d7..4599130207 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -9,29 +9,13 @@ import subprocess import sys import yaml - -generate_type_mutation = "MUTATION" -seperator = "--------------------" - -class Mutations: - def __init__(self, rfb=False, uoi=False, ror=False, cr=False, rt=False, lcr=False): - self.rfb = rfb - self.rfb_s = "remove-function-body" - self.uoi = uoi - self.uoi_s = "unary-operator-inversion" - self.ror = ror - self.ror_s = "relational-operator-replacement" - self.cr = cr - self.cr_s = "constant-replacement" - self.rt = rt - self.rt_s = "remove-thread" - self.lcr = lcr - self.lcr_s = "logical-connector-replacement" +sys.path.append("..") +from util import * def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) - index: int = yaml_data["n"] + index: int = yaml_data[META_N] if mutations.rfb: index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rfb_s, index) @@ -49,8 +33,8 @@ def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): return index def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): - print(seperator) - print(f"[{generate_type_mutation}] {mutation_name}") + print(SEPERATOR) + print(f"[{Generate_Type.MUTATION.value}] {mutation_name}") lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path) for lines in lineGroups: index += 1 @@ -196,11 +180,11 @@ def _write_meta_data(meta_path, new_path, index, mutation_name, lines): name = os.path.basename(new_path) with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) - yaml_data["n"] = index + yaml_data[META_N] = index yaml_data[f"p_{index}"] = { - "type": generate_type_mutation, - "sub_type": mutation_name, - "lines": lines + META_TYPE: Generate_Type.MUTATION.value, + META_SUB_TYPE: mutation_name, + META_LINES: lines } with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file, sort_keys=False) diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index ee59604d33..aa9e5ed487 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -5,8 +5,9 @@ import argparse import subprocess import sys - import yaml +sys.path.append("..") +from util import * def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): command = [ @@ -28,7 +29,7 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) - yaml_data["p_" + str(index)]['compilation'] = compiling + yaml_data["p_" + str(index)][META_COMPILING] = compiling with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 7087f2a289..bfc6da14d8 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -4,8 +4,7 @@ import sys sys.path.append("..") from generators.generate_mutations import * -from util.add_check import * -from util.add_check_comments import * +from util import * # Run for example with: # python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations @@ -18,9 +17,9 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api shutil.rmtree(temp_dir) os.makedirs(temp_dir) # Create Meta file - meta_path = os.path.join(temp_dir,'meta.yaml') + meta_path = os.path.join(temp_dir, META_FILENAME) with open(meta_path, 'w') as outfile: - yaml.dump({'n': 0, 'p_0': {'type': generate_type_source}}, outfile) + yaml.dump({'n': 0, 'p_0': {META_TYPE: generate_type_source}}, outfile) # Copy the source program into the temp dir program_path = os.path.join(temp_dir, 'p.c') shutil.copy2(source_path, program_path) @@ -38,7 +37,7 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api pass # Add checks with comments - print(seperator) + print(SEPERATOR) index += 1 for i in range(index): if i % 9 == 0: @@ -52,14 +51,14 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api print(f"Generating goblint checks [DONE]") # Check how many and which files were not compiling - print(seperator) + print(SEPERATOR) print("Check if the files compiled...") with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) failed_count = 0 failed_compilation_keys = [] for key, value in yaml_data.items(): - if isinstance(value, dict) and 'compilation' in value and value['compilation'] is False: + if isinstance(value, dict) and META_COMPILING in value and value[META_COMPILING] is False: failed_count += 1 failed_compilation_keys.append(key) if failed_count == 0: From 9cc1e8c83d7737ac657ad39ad924a6f16214fecb Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 21:11:04 +0200 Subject: [PATCH 053/131] Add main to test programs --- .../sample-files/constants.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/incremental-test-generation/sample-files/constants.c b/scripts/incremental-test-generation/sample-files/constants.c index a232571e92..c092c1ebeb 100644 --- a/scripts/incremental-test-generation/sample-files/constants.c +++ b/scripts/incremental-test-generation/sample-files/constants.c @@ -1,6 +1,11 @@ #define MY_MACRO_5 5 #define MY_MACRO_0 0 -int a = 42; -int b = 0; -int c = MY_MACRO_5; -int d = MY_MACRO_0; \ No newline at end of file + +int main() { + int a = 42; + int b = 0; + int c = MY_MACRO_5; + int d = MY_MACRO_0; + + return 0; +} \ No newline at end of file From c9f5e6e02b16d5fca5b945bba1b0ca7858e64a29 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 21:11:47 +0200 Subject: [PATCH 054/131] Fix imports --- .../test-generation/generators/generate_mutations.py | 2 +- .../test-generation/util/add_check.py | 2 +- .../test-generation/util/generate_programs.py | 4 +++- .../test-generation/util/generate_tests.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 4599130207..ad19735ba1 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -10,7 +10,7 @@ import sys import yaml sys.path.append("..") -from util import * +from util.util import * def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): with open(meta_path, 'r') as file: diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index aa9e5ed487..64a735e720 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -7,7 +7,7 @@ import sys import yaml sys.path.append("..") -from util import * +from util.util import * def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): command = [ diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index bfc6da14d8..c0987cd5e5 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -4,7 +4,9 @@ import sys sys.path.append("..") from generators.generate_mutations import * -from util import * +from util.util import * +from util.add_check import add_check +from util.add_check_comments import add_check_comments # Run for example with: # python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index 10ef3ba936..e89ca6431f 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -8,7 +8,7 @@ import sys import datetime sys.path.append("..") -from util import * +from util.util import * def generate_tests(temp_dir, target_dir, precision_test): # Check the name of the target_dir From 1821e936df21b953f79958d2e32aa8221e53f06b Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 21:12:19 +0200 Subject: [PATCH 055/131] Implement script for running tests --- .../test-generation/util/run_tests.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/util/run_tests.py diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py new file mode 100644 index 0000000000..fef7fd40f3 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -0,0 +1,37 @@ +import argparse +import os +import shutil +import subprocess + +def run_tests(test_dir, goblint_repo_dir, cfg): + # Check the name of the test_dir + test_dir_name = os.path.basename(test_dir) + if test_dir_name != "99-temp": + print("[ERROR] The test directory name has to be \'99-temp\'") + + incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) + if os.path.exists(incremental_tests_dir_abs): + shutil.rmtree(incremental_tests_dir_abs) + shutil.copytree(test_dir, incremental_tests_dir_abs) + + ruby_path_abs = os.path.abspath(os.path.join(goblint_repo_dir, "scripts", "update_suite.rb")) + os.chdir(goblint_repo_dir) + command = f"{ruby_path_abs} group temp -i" + if cfg: + command += " -c" + process = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True, shell=True) + for line in process.stdout: + print(line, end='') + process.wait() + + shutil.rmtree(incremental_tests_dir_abs) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') + parser.add_argument('test_dir', help='Path to the directory with the tests') + parser.add_argument('goblint_repo_dir', help='Path to the Goblint repository') + parser.add_argument('-c', '--cfg', action='store_true', help='Run with fine-grained cfg-based change detection') + + args = parser.parse_args() + + run_tests(args.test_dir, args.goblint_repo_dir, args.cfg) \ No newline at end of file From 112850dc644442ce6591c892f8e5f03fd3dfbf88 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 30 May 2023 21:12:29 +0200 Subject: [PATCH 056/131] Implemente base cli --- .../test-generation/.gitignore | 6 + .../test-generation/RUN_CLI.py | 190 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 scripts/incremental-test-generation/test-generation/RUN_CLI.py diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore index 8172810096..343577906c 100644 --- a/scripts/incremental-test-generation/test-generation/.gitignore +++ b/scripts/incremental-test-generation/test-generation/.gitignore @@ -1,2 +1,8 @@ test* meta* +input.c +repo +99-* +98-* +temp* +config.yaml \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py new file mode 100644 index 0000000000..7013077c93 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -0,0 +1,190 @@ +import argparse +import os +import shutil +import questionary +import yaml +from pathlib import Path +from util.util import * +from util.generate_programs import gernerate_programs +from util.generate_tests import generate_tests +from util.run_tests import run_tests +from generators.generate_mutations import add_mutation_options, get_mutations_from_args + +logo = '''Use -h to see the command line options + + __ __ _ _ _ + | \/ | | | | | (_) + | \ / |_ _| |_ __ _| |_ _ ___ _ __ + | |\/| | | | | __/ _` | __| |/ _ \| '_ \ + | | | | |_| | || (_| | |_| | (_) | | | | + |_| |_|\__,_|\__\__,_|\__|_|\___/|_| |_| + _____ _ + / ____| | | + | | __ ___ _ __ ___ _ __ __ _| |_ ___ _ __ + | | |_ |/ _ \ '_ \ / _ \ '__/ _` | __/ _ \| '__| + | |__| | __/ | | | __/ | | (_| | || (_) | | + \_____|\___|_| |_|\___|_| \__,_|\__\___/|_| + + + + ''' + +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests): + # Make paths absolute + goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) + llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) + input_path = os.path.abspath(os.path.expanduser(input_path)) #TODO Handle git url + + # Generate the programs + goblint_executable_path = os.path.join(goblint_path, 'goblint') + clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') + temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) + gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, 'TODO API KEY PATH', input_path, mutations, is_mutation, is_ml, is_git) + + #Write out custom test files + print(SEPERATOR) + print('Writing out custom test files:') + generate_tests(temp_path, os.path.join(os.path.curdir, '99-test'), precision_test=False) #TODO Custom name + if create_precision: + print(SEPERATOR) + print('Writing out custom precision files:') + generate_tests(temp_path, os.path.join(os.path.curdir, '98-precision'), precision_test=False) #TODO Custom name + + if is_run_tests: + test_path = os.path.abspath(os.path.join(os.path.curdir, '99-temp')) + if create_precision: + print(SEPERATOR) + print('Writing out precision test files for running:') + generate_tests(temp_path, test_path, precision_test=True) + run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg + print(SEPERATOR) + print('Writing out test files for running:') + generate_tests(temp_path, test_path, precision_test=False) + run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg + if os.path.exists(test_path): + shutil.rmtree(test_path) + + #TODO Print link to html result and give summary + +def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input): + # Check config file + config_path = Path("config.yaml") + config = {} + if not config_path.is_file(): + print(f'Config file "{config_path}" not found. Please provide the paths:') + goblint_path = questionary.text('Enter the path to the goblint repository: ', default="~/Goblint-Repo/analyzer").ask() + llvm_path = questionary.text('Enter the path to the llvm repository with the modified clang-tidy: ', default="~/Clang-Repo/llvm-project").ask() + config.update({"goblint-path": goblint_path, "llvm-path": llvm_path}) + with open(config_path, 'w') as outfile: + yaml.dump(config, outfile) + else: + with open(config_path, 'r') as stream: + config = yaml.safe_load(stream) + goblint_path = config["goblint-path"] + llvm_path = config["llvm-path"] + print(f'Using goblint-path (change in ./config.yaml): {goblint_path}') + print(f'Using llvm-path (change in ./config.yaml): {llvm_path}') + + # Handle Questions + if not (enable_mutations or enable_ml or enable_git): + while True: + generators = questionary.checkbox( + 'Select one or more generator types (When git is checked no other can be checked!):', + choices=[ + questionary.Choice('Mutations', checked=True), + questionary.Choice('ML', checked=True), + 'Git' + ]).ask() + + # check if 'Git' is selected along with other options + if 'Git' in generators and len(generators) > 1: + print("If 'Git' is selected, no other options should be selected. Please select again.") + continue + else: + break + if 'Mutations' in generators: + selected_mutations = questionary.checkbox( + 'Select one or more mutation types:', + choices=[ + questionary.Choice('remove-function-body (RFB)', checked=True), + questionary.Choice('unary-operator-inversion (UOI)', checked=True), + questionary.Choice('relational-operator-replacement (ROR)', checked=True), + questionary.Choice('constant-replacement (CR)', checked=True), + questionary.Choice('remove-thread (RT)', checked=True), + questionary.Choice('logical-connector-replacement (LCR)', checked=True), + ]).ask() + mutations = Mutations( + rfb='remove-function-body (RFB)' in selected_mutations, + uoi='unary-operator-inversion (UOI)' in selected_mutations, + ror='relational-operator-replacement (ROR)' in selected_mutations, + cr='constant-replacement (CR)' in selected_mutations, + rt='remove-thread (RT)' in selected_mutations, + lcr='logical-connector-replacement (LCR)' in selected_mutations + ) + enable_mutations = 'Mutations' in generators + enable_ml = 'ML' in generators + enable_git = 'Git' in generators + if precision == None: + precision = questionary.confirm('Create precision test files?', default=False).ask() + if running == None: + running = questionary.confirm('Run the tests?').ask() + if input == None: + if enable_mutations or enable_ml: + input = questionary.text('Enter the path to the c program for the mutations: ', default="input.c").ask() + else: + input = questionary.text('Enter the path or URL to the git repository for the mutations: ', default="repo").ask() + + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running) + + +if __name__ == "__main__": + print(logo) + + parser = argparse.ArgumentParser(description='Generates mutations for creating incremental tests') + parser.add_argument('-m', '--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') + parser.add_argument('-o', '--enable-ml', action='store_true', help='Enable ML') + parser.add_argument('-g', '--enable-git', action='store_true', help='Enable Git') + parser.add_argument('-ep', '--enable-precision', action='store_true', help='Enable Precision Tests') + parser.add_argument('-dp', '--disable-precision', action='store_true', help='Disable Precision Tests') + parser.add_argument('-er', '--enable-running', action='store_true', help='Enable Running Tests') + parser.add_argument('-dr', '--disable-running', action='store_true', help='Disable Running Tests') + parser.add_argument('-i', '--input', help='Input File') + + # Add mutation options + add_mutation_options(parser) + + args = parser.parse_args() + + if args.enable_mutations or args.enable_ml or args.enable_git: + # If using git, only git can be used + if args.enable_git and (args.enable_ml or args.enable_mutations): + parser.error("--enable-git cannot be used with --enable-ml or --enable-mutations") + + # If all mutation options are false, set all to true + mutations = get_mutations_from_args(args) + non_str_attributes = [attr for attr in vars(mutations) if not attr.endswith('_s')] + if all(getattr(mutations, attr) is False for attr in non_str_attributes): + mutations = Mutations(True, True, True, True, True, True) + else: + args.enable_mutations = None + args.enable_ml = None + args.enable_git = None + mutations = None + + if args.enable_precision or args.disable_precision: + # Only one can be selected + if args.enable_precision and args.disable_precision: + parser.error('Precision can not be enabled AND diabled') + precision = args.enable_precision + else: + precision = None + + if args.enable_running or args.disable_running: + # Only one can be selected + if args.enable_running and args.disable_running: + parser.error('Running can not be enabled AND diabled') + running = args.enable_running + else: + running = None + + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input) \ No newline at end of file From b7252e109c8daa3d1652b01f7e3ffbb98a2ff482 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 31 May 2023 10:29:41 +0200 Subject: [PATCH 057/131] Make name of tests unique --- .../test-generation/util/generate_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index e89ca6431f..4c92bf0b83 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -47,7 +47,7 @@ def generate_tests(temp_dir, target_dir, precision_test): print(f"Generating test files [{i}/{n}]") if type == Generate_Type.MUTATION.value: sub_type = yaml_data[generated_id][META_SUB_TYPE] - test_name = _format_number(i-1) + '-' + type + '_' + sub_type + test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) # Copy mutated code as the original code shutil.copy2(generated_program, os.path.join(target_dir, test_name + '.c')) # Create a patch file From d7a3dd857ad42d89adb08c3dbe84e836d3a5f97d Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 31 May 2023 10:31:19 +0200 Subject: [PATCH 058/131] Fix Logo --- scripts/incremental-test-generation/test-generation/RUN_CLI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 7013077c93..9e8ff9db5b 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -23,7 +23,7 @@ | | __ ___ _ __ ___ _ __ __ _| |_ ___ _ __ | | |_ |/ _ \ '_ \ / _ \ '__/ _` | __/ _ \| '__| | |__| | __/ | | | __/ | | (_| | || (_) | | - \_____|\___|_| |_|\___|_| \__,_|\__\___/|_| + \_____|\___|_| |_|\___|_| \__,_|\__\___/|_| From d2cb173a8b4aa33e6b7fa6703120bc4f46575bb8 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 31 May 2023 21:08:09 +0200 Subject: [PATCH 059/131] Fix comment name --- .../test-generation/util/add_check_comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py index bc276baf89..886731de63 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py @@ -23,7 +23,7 @@ def add_check_comments(file_path: str, undefined_instead_of_success: bool, verbo if match: modified_line = match.group(1) if undefined_instead_of_success: - modified_line += ' //UNDEFINED' + modified_line += ' //UNKNOWN //SUCCESS' else: modified_line += ' //SUCCESS' line = line.replace(match.group(1), modified_line) From e1887da45eadaa4ef88d824e1ecc2f9a022a4baa Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 1 Jun 2023 10:49:45 +0200 Subject: [PATCH 060/131] Generate Minimal Test for "nothing" problem --- .../sample-files/minimalTestNothing.c | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scripts/incremental-test-generation/sample-files/minimalTestNothing.c diff --git a/scripts/incremental-test-generation/sample-files/minimalTestNothing.c b/scripts/incremental-test-generation/sample-files/minimalTestNothing.c new file mode 100644 index 0000000000..d4db2f965f --- /dev/null +++ b/scripts/incremental-test-generation/sample-files/minimalTestNothing.c @@ -0,0 +1,38 @@ +// Run with: +// python3 RUN_CLI.py -m -dp -er -i ../sample-files/minimalTestNothing.c -rfb + +#include +#include +#include + +int main() { + int a = 10; + int b = 20; + + if (a > b) { + // Is never reached so there is registered nothing! + printf("a is greater than b\n"); + } + + if (a < b) { + // Is reached so there no problem + printf("a is greater than b\n"); + } + + // Try with random inputs -> No problem + srand(time(NULL)); + int r1 = rand(); + int r2 = rand(); + + if (r1 > r2) { + // No knowledge about outcome so no problem + printf("a is greater than b\n"); + } + + if (r1 < r2) { + // No knowledge about outcome so no problem + printf("a is greater than b\n"); + } + + return 0; +} From 0bdda1f6a6d4f241f9d8fd3492cd747115534c59 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 2 Jun 2023 16:15:56 +0200 Subject: [PATCH 061/131] Implementation of ML --- .../test-generation/.gitignore | 3 +- .../test-generation/RUN_CLI.py | 105 +++++++-- .../test-generation/generators/.gitignore | 3 +- .../test-generation/generators/generate_ml.py | 201 ++++++++++++++++++ .../generators/generate_mutations.py | 15 +- .../test-generation/util/add_check.py | 11 +- .../test-generation/util/generate_programs.py | 13 +- .../test-generation/util/generate_tests.py | 14 +- .../test-generation/util/util.py | 17 ++ 9 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 scripts/incremental-test-generation/test-generation/generators/generate_ml.py diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore index 343577906c..c0620d70e8 100644 --- a/scripts/incremental-test-generation/test-generation/.gitignore +++ b/scripts/incremental-test-generation/test-generation/.gitignore @@ -5,4 +5,5 @@ repo 99-* 98-* temp* -config.yaml \ No newline at end of file +config.yaml +api-key.yaml \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 9e8ff9db5b..c32c49bf53 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -29,7 +29,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests, api_key_path, ml_count): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -39,7 +39,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, 'TODO API KEY PATH', input_path, mutations, is_mutation, is_ml, is_git) + gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, input_path, mutations, is_mutation, is_ml, is_git, ml_count) #Write out custom test files print(SEPERATOR) @@ -66,24 +66,28 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio #TODO Print link to html result and give summary -def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input): +def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input, ml_count): # Check config file - config_path = Path("config.yaml") + config_path = Path(CONFIG_FILENAME) config = {} if not config_path.is_file(): print(f'Config file "{config_path}" not found. Please provide the paths:') goblint_path = questionary.text('Enter the path to the goblint repository: ', default="~/Goblint-Repo/analyzer").ask() llvm_path = questionary.text('Enter the path to the llvm repository with the modified clang-tidy: ', default="~/Clang-Repo/llvm-project").ask() - config.update({"goblint-path": goblint_path, "llvm-path": llvm_path}) + config.update({CONFIG_GOBLINT: goblint_path, CONFIG_LLVM: llvm_path, CONFIG_LAST_INPUT_MUTATION: '', CONFIG_LAST_INPUT_GIT: ''}) + last_input_mutation = '' + last_input_git = '' with open(config_path, 'w') as outfile: yaml.dump(config, outfile) else: with open(config_path, 'r') as stream: config = yaml.safe_load(stream) - goblint_path = config["goblint-path"] - llvm_path = config["llvm-path"] - print(f'Using goblint-path (change in ./config.yaml): {goblint_path}') - print(f'Using llvm-path (change in ./config.yaml): {llvm_path}') + goblint_path = config[CONFIG_GOBLINT] + llvm_path = config[CONFIG_LLVM] + last_input_mutation = config[CONFIG_LAST_INPUT_MUTATION] + last_input_git = config[CONFIG_LAST_INPUT_GIT] + print(f'Using goblint-path (change in ./{CONFIG_FILENAME}): {goblint_path}') + print(f'Using llvm-path (change in ./{CONFIG_FILENAME}): {llvm_path}') # Handle Questions if not (enable_mutations or enable_ml or enable_git): @@ -92,7 +96,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, 'Select one or more generator types (When git is checked no other can be checked!):', choices=[ questionary.Choice('Mutations', checked=True), - questionary.Choice('ML', checked=True), + 'ML', 'Git' ]).ask() @@ -102,7 +106,12 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, continue else: break - if 'Mutations' in generators: + + enable_mutations = 'Mutations' in generators + enable_ml = 'ML' in generators + enable_git = 'Git' in generators + + if enable_mutations: selected_mutations = questionary.checkbox( 'Select one or more mutation types:', choices=[ @@ -121,20 +130,67 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, rt='remove-thread (RT)' in selected_mutations, lcr='logical-connector-replacement (LCR)' in selected_mutations ) - enable_mutations = 'Mutations' in generators - enable_ml = 'ML' in generators - enable_git = 'Git' in generators + + # Check for API Key + if enable_ml: + key_path = Path(APIKEY_FILENAME) + key_data = {} + if not key_path.is_file(): + print(f'Api key file "{key_path}" for OpenAi not found. Please provide the informations:') + print('Be aware that the information is stored unencrypted. Do not remove the file from .gitignore!') + print('Create an account here: https://openai.com/blog/openai-api') + print('Create an API Key here: https://platform.openai.com/account/api-keys') + print('Get your organization id here: https://platform.openai.com/account/org-settings') + key = questionary.text('Enter the api key:').ask() + org = questionary.text('Enter the organisation id:').ask() + key_data.update({APIKEY_APIKEY: key, APIKEY_ORGANISATION: org}) + with open(key_path, 'w') as outfile: + yaml.dump(key_data, outfile) + else: + with open(key_path, 'r') as stream: + key_data = yaml.safe_load(stream) + key = key_data[APIKEY_APIKEY] + org = key_data[APIKEY_ORGANISATION] + print(f'Using api-key for ML (change in ./{APIKEY_FILENAME}): ...{key[-4:]}') + print(f'Using organisation id for ML (change in ./{APIKEY_FILENAME}): ...{org[-4:]}') + key_path = os.path.abspath(key_path) + else: + key_path = None + + if ml_count == None: + while True: + ml_count = questionary.text('How many different programs should be generated with ML?', default=str(DEFAULT_ML_COUNT)).ask() + if not ml_count.strip('\n').isdigit(): + print("Please enter a valid number.") + continue + ml_count = int(ml_count.strip('\n')) + if ml_count <= 0: + print("Please enter a number greater zero.") + continue + break + if precision == None: precision = questionary.confirm('Create precision test files?', default=False).ask() + if running == None: running = questionary.confirm('Run the tests?').ask() + if input == None: - if enable_mutations or enable_ml: - input = questionary.text('Enter the path to the c program for the mutations: ', default="input.c").ask() - else: - input = questionary.text('Enter the path or URL to the git repository for the mutations: ', default="repo").ask() + while True: + if enable_mutations or enable_ml: + input = questionary.text('Enter the path to the c program for the mutations: ', default=last_input_mutation).ask() + config.update({CONFIG_LAST_INPUT_MUTATION: input}) + else: + input = questionary.text('Enter the path to the git repository for the mutations: ', default=last_input_git).ask() + config.update({CONFIG_LAST_INPUT_GIT: input}) + if not os.path.exists(input): + print("Please enter a valid path.") + continue + with open(config_path, 'w') as outfile: + yaml.dump(config, outfile) + break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running, key_path, ml_count) if __name__ == "__main__": @@ -153,6 +209,9 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, # Add mutation options add_mutation_options(parser) + # Add ML options + parser.add_argument('-c', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') + args = parser.parse_args() if args.enable_mutations or args.enable_ml or args.enable_git: @@ -187,4 +246,10 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, else: running = None - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input) \ No newline at end of file + if args.ml_count > 0: + ml_count = args.ml_count + else: + ml_count = None + + + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input, ml_count) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/.gitignore b/scripts/incremental-test-generation/test-generation/generators/.gitignore index ed8ebf583f..834b16bde6 100644 --- a/scripts/incremental-test-generation/test-generation/generators/.gitignore +++ b/scripts/incremental-test-generation/test-generation/generators/.gitignore @@ -1 +1,2 @@ -__pycache__ \ No newline at end of file +__pycache__ +test diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py new file mode 100644 index 0000000000..38a764c548 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -0,0 +1,201 @@ +import argparse +import ast +import os +import random +import shutil +import sys +import openai +import yaml +sys.path.append("..") +from util.util import * + +SEPERATOR_EXPLANATION_START = 'EXPLANATION>' +SEPERATOR_EXPLANATION_END = '= with >, Replacing constants unequal 0 with 1, Replace pthread calls with function calls, Switching && with ||. Please do not consider these mutations as examples how your code changes should look like. Just try to prevent doing things that could be done with these mutations. + + Below is an snippet from a C file which represents a part of the finished program. My question is how a previous version of this code could have looked like before some typical code changes done by developers. Can you generate me such a previous version? + + The code you generate should be a self-contained snippet that could directly replace the provided excerpt in the original, complete program. It should preserve the overall functionality of the program and must not cause any compilation errors when reintegrated into the larger code base. Please consider the dependencies and interactions with other parts of the program when generating the previous version of the code. Your generated code should be able to interact correctly with the rest of the program just like the original excerpt does. You do not have to add import statements or function declarations or closing brackets when these are cut off in the snippet, but when they are in the snippet you need to add them to preserve the whole program. + + I use keywords ({SEPERATOR_EXPLANATION_START}, {SEPERATOR_EXPLANATION_END}, {SEPERATOR_CODE_START}, {SEPERATOR_CODE_END}) to interpret you answer. You answer should have the following structure for better identifying the different parts of the response: {SEPERATOR_EXPLANATION_START} (Explain what you have changed in one or two sentences) {SEPERATOR_EXPLANATION_END} {SEPERATOR_CODE_START} (the previous version of the code) {SEPERATOR_CODE_END} + + ```c + {snippet} + ``` + ''' + + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + n = 1, + messages=[ + {"role": "user", "content": prompt}, + ] + ).choices[0].message['content'] + + return response + +def _reformat_interesting_lines(num_selected_lines, interesting_lines, max_line): + for i in range(len(interesting_lines)): + # Adjust for line array starting with 0 but in input first line is 1 + interesting_lines[i] -= 1 + # When line + num_selected_lines is greater then max_line correct the line downwards + if interesting_lines[i] + num_selected_lines > (max_line): + interesting_lines[i] = (max_line) - num_selected_lines + return interesting_lines + +def _select_lines(interesting_lines, num_selected_lines, max_line): + if interesting_lines == []: + selected_line = random.randint(0, (max_line) - num_selected_lines) + else: + selected_line = random.choice(interesting_lines) + return range(selected_line, selected_line + num_selected_lines) + +#TODO Call when giving interesting lines +def validate_interesting_lines(intersting_lines_string: str, program_path): + with open(program_path, "r") as file: + max_line = len(file.readlines()) + + try: + intersting_lines = ast.literal_eval(intersting_lines_string) + except SyntaxError: + print(f"The format \"{intersting_lines_string}\" is incorrect! Please use a format like this: \"[1, 42, 99]\"") + return None + except Exception as e: + print(f"An unexpected error occurred reading the input \"{intersting_lines_string}\": {e}") + return None + + for line in intersting_lines: + if line <= 0: + print(f"All lines in \"{intersting_lines_string}\" should be greater zero!") + return None + if line > max_line: + print(f"All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!") + return None + + return intersting_lines + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate mutations with ML.") + parser.add_argument("program", help="Path to the C program") + parser.add_argument("apikey", help="Path to the api key") + parser.add_argument("meta_path", help="Path to the meta_file") + parser.add_argument("ml_count", help="How many different programs should be generated with ML") + parser.add_argument("num_selected_lines", help="How many lines to consider") + parser.add_argument("interesting_lines", help="Which parts are interesting (All: [], Specify: \"[1, 42, 99]\")") + + args = parser.parse_args() + + interesting_lines = validate_interesting_lines(args.interesting_lines, args.program) + if interesting_lines == None: + print('Stopped program execution') + sys.exit(-1) + + generate_ml(args.program, args.apikey, args.meta_path, int(args.ml_count), int(args.num_selected_lines), interesting_lines) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index ad19735ba1..22ace57761 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -35,10 +35,10 @@ def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): print(SEPERATOR) print(f"[{Generate_Type.MUTATION.value}] {mutation_name}") - lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path) + lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path, index) for lines in lineGroups: index += 1 - new_path = _make_copy(program_path, index) + new_path = make_program_copy(program_path, index) if mutation_name == Mutations().rt_s: # When Remove Thread create wrapper an then apply the mutations if len(lines) != 1: @@ -50,12 +50,7 @@ def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mut _write_meta_data(meta_path, new_path, index, mutation_name, lines) return index -def _make_copy(program_path, index): - new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' - shutil.copy2(program_path, new_path) - return new_path - -def _get_line_groups(clang_tidy_path, mutation_name, program_path): +def _get_line_groups(clang_tidy_path, mutation_name, program_path, index): #TODO Handle [MACRO] tags command = [ clang_tidy_path, @@ -65,7 +60,7 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path): ] result = subprocess.run(command, text=True, capture_output=True) - print(f"[CHECK] Check mutation {mutation_name} with return code {result.returncode}") + print(f"[MUTATION][CHECK] Check mutation {mutation_name} with return code {result.returncode}") if result.returncode != 0: print(result.stdout) print(result.stderr) @@ -97,7 +92,7 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path): # Remove duplicate line groups line_groups = [list(x) for x in set(tuple(x) for x in line_groups)] - print(f"[CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") + print(f"[MUTATION][CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") return line_groups def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index 64a735e720..f3453174f3 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -33,11 +33,14 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file) - if not compiling : - print(result.stdout) - print(result.stderr) + if not compiling: print(f"Error compiling program with index {index}.") - sys.exit(-1) + if index == 0: + print("The original program did not compile. Stopping program!") + sys.exit(-1) + return False + else: + return True if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate __goblint_check() for a C file. When not compiling write [NOT COMPILING] in the meta file') diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index c0987cd5e5..fa0233c5b1 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -4,6 +4,7 @@ import sys sys.path.append("..") from generators.generate_mutations import * +from generators.generate_ml import * from util.util import * from util.add_check import add_check from util.add_check_comments import add_check_comments @@ -13,7 +14,7 @@ generate_type_source = "SOURCE" -def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git): +def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git, ml_count): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -33,7 +34,11 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api index = generate_mutations(program_path, clang_tidy_path, meta_path, mutations) if enable_ml: - pass + #TODO Allow user to specify how many lines to select + NUM_SELECTED_LINES = 25 + #TODO Allow user to specify which part of the program is intresting + INTRESTING_LINES = [] + index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) if enable_git: pass @@ -45,7 +50,9 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if i % 9 == 0: print(f"Generating goblint checks [{i+1}/{index}]") file_path = os.path.join(temp_dir, f"p_{i}.c") - add_check(file_path, i, goblint_path, meta_path) + compiling = add_check(file_path, i, goblint_path, meta_path) + if not compiling: + continue file_path = os.path.join(temp_dir, f"p_{i}_c.c") if i == 0: add_check_comments(file_path, True) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index 4c92bf0b83..6885eb6ae7 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -45,9 +45,12 @@ def generate_tests(temp_dir, target_dir, precision_test): continue if (i-1) % 9 == 0: print(f"Generating test files [{i}/{n}]") - if type == Generate_Type.MUTATION.value: + if type == Generate_Type.MUTATION.value or type == Generate_Type.ML.value: sub_type = yaml_data[generated_id][META_SUB_TYPE] - test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) + if type == Generate_Type.MUTATION.value: + test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) + else: + test_name = _format_number(i) + '-' + type + '_' + _format_number(i) # Copy mutated code as the original code shutil.copy2(generated_program, os.path.join(target_dir, test_name + '.c')) # Create a patch file @@ -73,12 +76,11 @@ def generate_tests(temp_dir, target_dir, precision_test): data = {} with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: json.dump(data, f) - if type == Generate_Type.ML.value: - #TODO - print ('[ERROR] Generating ML Tests not implemented') - if type == Generate_Type.GIT.value: + elif type == Generate_Type.GIT.value: #TODO print ('[ERROR] Generating GIT Tests not implemented') + else: + print ('[ERROR] Generating Unknown Tests not implemented') print(f"Generating test files [DONE]") with open(meta_path, 'w') as file: diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index 4bfaaeea6b..80f412c222 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -1,4 +1,5 @@ from enum import Enum +import shutil class Mutations: @@ -32,3 +33,19 @@ class Generate_Type(Enum): META_SUB_TYPE = 'sub_type' META_LINES = 'lines' +CONFIG_FILENAME = 'config.yaml' +CONFIG_GOBLINT = "goblint-path" +CONFIG_LLVM = "llvm-path" +CONFIG_LAST_INPUT_MUTATION = "last-input-mutation" +CONFIG_LAST_INPUT_GIT = "last-input-git" + +APIKEY_FILENAME = 'api-key.yaml' +APIKEY_APIKEY = 'api-key' +APIKEY_ORGANISATION = 'organisation' + +DEFAULT_ML_COUNT = 5 + +def make_program_copy(program_path, index): + new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' + shutil.copy2(program_path, new_path) + return new_path From 26917caf71d6ef4151ccc1734f42f947da8b184d Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 2 Jun 2023 16:50:29 +0200 Subject: [PATCH 062/131] Run multiple requests for ML in parallel --- .../test-generation/generators/generate_ml.py | 48 +++++++++++-------- .../test-generation/util/generate_programs.py | 2 +- .../test-generation/util/util.py | 1 + 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index 38a764c548..43ffbaf645 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -2,10 +2,13 @@ import ast import os import random -import shutil import sys +import time import openai import yaml +from concurrent.futures import ThreadPoolExecutor +from multiprocessing import Lock + sys.path.append("..") from util.util import * @@ -38,17 +41,19 @@ def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lin print(SEPERATOR) interesting_lines_string = 'Start lines are randomly choosen from all lines.' if interesting_lines == [] else f' Start lines are randomly choosen from {interesting_lines}.' - print(f'[ML] Start making {ml_count} requests with ML. {num_selected_lines} from {max_line} lines will be selected. {interesting_lines_string}') - for _ in range(ml_count): - index = _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index) + print(f'[ML] Start making {ml_count} requests with ML. {ML_WORKERS} are executed in parallel. {num_selected_lines} from {max_line} lines will be selected. {interesting_lines_string}') + file_lock = Lock() + with ThreadPoolExecutor(max_workers=ML_WORKERS) as executor: + for i in range(ml_count): + executor.submit(_iterative_mutation_generation, program_path, meta_path, interesting_lines, num_selected_lines, max_line, index + i + 1, file_lock) - return index + return index + ml_count -def _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index): - index += 1 +def _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index, lock): + time.sleep((index * 50)/1000) new_path = make_program_copy(program_path, index) (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index) - _write_meta_data(meta_path, new_path, selected_lines, explanation, index) + _write_meta_data(meta_path, new_path, selected_lines, explanation, index, lock) return index def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index): @@ -102,18 +107,19 @@ def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, i return (explanation, selected_lines) -def _write_meta_data(meta_path, new_path, selected_lines, explanation, index): - name = os.path.basename(new_path) - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data[META_N] = index - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.ML.value, - META_SUB_TYPE: explanation, - META_LINES: f'[{selected_lines.start}, {selected_lines.stop}]' - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file, sort_keys=False) +def _write_meta_data(meta_path, new_path, selected_lines, explanation, index, lock): + with lock: + name = os.path.basename(new_path) + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data[META_N] = index + yaml_data[f"p_{index}"] = { + META_TYPE: Generate_Type.ML.value, + META_SUB_TYPE: explanation, + META_LINES: f'[{selected_lines.start}, {selected_lines.stop}]' + } + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file, sort_keys=False) def _make_gpt_request(snippet): prompt = f''' @@ -125,7 +131,7 @@ def _make_gpt_request(snippet): The code you generate should be a self-contained snippet that could directly replace the provided excerpt in the original, complete program. It should preserve the overall functionality of the program and must not cause any compilation errors when reintegrated into the larger code base. Please consider the dependencies and interactions with other parts of the program when generating the previous version of the code. Your generated code should be able to interact correctly with the rest of the program just like the original excerpt does. You do not have to add import statements or function declarations or closing brackets when these are cut off in the snippet, but when they are in the snippet you need to add them to preserve the whole program. - I use keywords ({SEPERATOR_EXPLANATION_START}, {SEPERATOR_EXPLANATION_END}, {SEPERATOR_CODE_START}, {SEPERATOR_CODE_END}) to interpret you answer. You answer should have the following structure for better identifying the different parts of the response: {SEPERATOR_EXPLANATION_START} (Explain what you have changed in one or two sentences) {SEPERATOR_EXPLANATION_END} {SEPERATOR_CODE_START} (the previous version of the code) {SEPERATOR_CODE_END} + Use these keywords (\"{SEPERATOR_EXPLANATION_START}\", \"{SEPERATOR_EXPLANATION_END}\", \"{SEPERATOR_CODE_START}\", \"{SEPERATOR_CODE_END}\") to structure you answer. You answer should have the following structure for better identifying the different parts of the response: {SEPERATOR_EXPLANATION_START} (Explain what you have changed in one or two sentences) {SEPERATOR_EXPLANATION_END} {SEPERATOR_CODE_START} (the previous version of the code) {SEPERATOR_CODE_END} ```c {snippet} diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index fa0233c5b1..a36bae4ecf 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -35,7 +35,7 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if enable_ml: #TODO Allow user to specify how many lines to select - NUM_SELECTED_LINES = 25 + NUM_SELECTED_LINES = 50 #TODO Allow user to specify which part of the program is intresting INTRESTING_LINES = [] index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index 80f412c222..e8daf1be7c 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -44,6 +44,7 @@ class Generate_Type(Enum): APIKEY_ORGANISATION = 'organisation' DEFAULT_ML_COUNT = 5 +ML_WORKERS = 10 def make_program_copy(program_path, index): new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' From d2cc0c34d6817b7144aa6e591350ddc948add1b7 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 6 Jun 2023 18:21:59 +0200 Subject: [PATCH 063/131] documentation of llvm version --- scripts/incremental-test-generation/clang-mutations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/clang-mutations/README.md b/scripts/incremental-test-generation/clang-mutations/README.md index 8c1863fff1..06e601b1e8 100644 --- a/scripts/incremental-test-generation/clang-mutations/README.md +++ b/scripts/incremental-test-generation/clang-mutations/README.md @@ -12,7 +12,7 @@ For building Clang you need to install some dependencies: - [ ] There are two alternatives for getting the repository For creating all the checks by yourself clone the **Official Clang Repository**: -`git clone https://github.com/llvm/llvm-project.git` +`git clone https://github.com/llvm/llvm-project.git` (Tested with Version 17.0.0) Alternatively you can clone the **Fork** with all the additional checks ready: `git clone https://github.com/J2000A/llvm-project.git` From a6368eeae41576065f94a73ead1f288bd6315b97 Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 6 Jun 2023 20:12:08 +0200 Subject: [PATCH 064/131] Trying around with git --- .../git-patches/generate_cil.py | 58 +++++++++++++ .../git-patches/generate_git.py | 87 +++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 scripts/incremental-test-generation/git-patches/generate_cil.py create mode 100644 scripts/incremental-test-generation/git-patches/generate_git.py diff --git a/scripts/incremental-test-generation/git-patches/generate_cil.py b/scripts/incremental-test-generation/git-patches/generate_cil.py new file mode 100644 index 0000000000..e9f9735fff --- /dev/null +++ b/scripts/incremental-test-generation/git-patches/generate_cil.py @@ -0,0 +1,58 @@ +import os +import argparse +import subprocess +import sys + +BUILD_CMAKE = 'cmake' +BUILD_MAKE = 'make' + +def cil_transform(goblint_path, project_dir, output_dir, build_system): + goblint_path = os.path.abspath(goblint_path) + project_dir = os.path.abspath(project_dir) + output_dir = os.path.abspath(output_dir) + + # Check if Makefile/CMakeLists.txt exists in the directory + build_file = 'Makefile' if build_system == BUILD_MAKE else 'CMakeLists.txt' + if not os.path.isfile(os.path.join(project_dir, build_file)): + print(f"No {build_file} found in the project directory: {project_dir}") + return + + # Navigate to the directory + original_dir = os.getcwd() + os.chdir(project_dir) + + # Run make/cmake command + if build_system == BUILD_MAKE: + build_process = subprocess.run([build_system, 'DCMAKE_EXPORT_COMPILE_COMMANDS=ON']) + elif build_system == BUILD_CMAKE: + build_process = subprocess.run(['bear', '--', 'make']) + if build_process.returncode != 0: + print(f"Error in {build_system} command:") + print(build_process.stderr.decode()) + return + + # Run goblint with cil option + goblint_process = subprocess.run([goblint_path, '--set', 'justcil', 'true', '--set', 'cil.merge.inlines', 'false', project_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if goblint_process.returncode != 0: + print("Error in goblint command:") + print(goblint_process.stderr.decode()) + return + + # Output the transformed CIL code + with open(os.path.join(output_dir, 'cil.c'), 'w') as f: + f.write(goblint_process.stdout.decode()) + print("CIL transformation completed. Transformed code written to cil.c") + + # Navigate back to the original directory + os.chdir(original_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Transform a project into a single CIL file using Goblint.') + parser.add_argument('goblint_path', type=str, help='Path to the goblint binary') + parser.add_argument('project_dir', type=str, help='Path to the project directory') + parser.add_argument('output_dir', type=str, help='Path to the output directory') + parser.add_argument('--build', choices=[BUILD_MAKE, BUILD_CMAKE], required=True, help=f'Build system to use ({BUILD_MAKE} or {BUILD_CMAKE})') + + args = parser.parse_args() + cil_transform(args.goblint_path, args.project_dir, args.output_dir, args.build) diff --git a/scripts/incremental-test-generation/git-patches/generate_git.py b/scripts/incremental-test-generation/git-patches/generate_git.py new file mode 100644 index 0000000000..f69d1831ac --- /dev/null +++ b/scripts/incremental-test-generation/git-patches/generate_git.py @@ -0,0 +1,87 @@ +# File based on bench/scripts/incremental/benchmarking/stats.py from the Goblint bench repository +# https://github.com/goblint/bench/blob/6333415e9b5b5157b3b462b673726faa93c97488/scripts/incremental/benchmarking/stats.py + +from pydriller import Repository +from datetime import datetime +from pathlib import Path +import os +import sys + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Wrong number of parameters.\nUse script like this: python3 generate_git.py ") + exit() + +analyzer_dir = sys.argv[1] +url = 'https://github.com/facebook/zstd' +repo_name = 'zstd' +begin = datetime(2021,8,1) +to = datetime(2022,2,1) +maxCLOC = 50 +dirs_to_exclude = ["build", "doc", "examples", "tests", "zlibWrapper", "contrib"] + +cwd = os.getcwd() +outdir = os.path.join(cwd, 'out') +repo_path = os.path.normpath(os.path.join(cwd, repo_name)) +paths_to_exclude = list(map(lambda x: os.path.join(repo_path, x), dirs_to_exclude)) + +analyzed_commits = {} +total_commits = 0 +count_nochanges = 0 +count_merge = 0 +count_big = 0 +count_small = 0 + +# Function based on bench/scripts/incremental/benchmarking/utils.py from Goblint bench repository +# https://github.com/goblint/bench/blob/6333415e9b5b5157b3b462b673726faa93c97488/scripts/incremental/benchmarking/utils.py +def _calculateRelCLOC(repo_path, commit, diff_exclude): + diff_exclude = list(map(lambda x: os.path.join(repo_path, x), diff_exclude)) + relcloc = 0 + for f in commit.modified_files: + _, extension = os.path.splitext(f.filename) + if not (extension == ".h" or extension == ".c"): + continue + filepath = f.new_path + if filepath is None: + filepath = f.old_path + parents = Path(filepath).parents + parents = list(map(lambda x: os.path.join(repo_path, x), parents)) + if any(dir in parents for dir in diff_exclude): + continue + relcloc = relcloc + f.added_lines + f.deleted_lines + return relcloc + +def iter_repo(): + global analyzed_commits + global total_commits + global count_merge + global count_nochanges + global count_big + global count_small + + for commit in Repository(url, since=begin, to=to, clone_repo_to=cwd).traverse_commits(): + total_commits += 1 + + # count merge commits + if commit.merge: + count_merge += 1 + continue + + # count commits that have less than maxCLOC of relevant code changes + relCLOC = _calculateRelCLOC(repo_path, commit, paths_to_exclude) # use this to filter commits by actually relevant changes + if relCLOC == 0: + count_nochanges += 1 + continue + + if maxCLOC is not None and relCLOC > maxCLOC: + count_big += 1 + continue + + count_small += 1 + +iter_repo() +print("\nCommits traversed in total: ", total_commits) +print("Merge commits: ", count_merge) +print("Commits without any relevant changes: ", count_nochanges) +print("Big commits: ", count_big) +print("Small commits with relevant changes: ", count_small) \ No newline at end of file From 7bb727bf6a277371ff0936d8f39bdbbf7a457d1d Mon Sep 17 00:00:00 2001 From: J2000A Date: Wed, 7 Jun 2023 21:59:12 +0200 Subject: [PATCH 065/131] Bugfix --- .../incremental-test-generation/test-generation/RUN_CLI.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index c32c49bf53..4172f04c83 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -10,7 +10,7 @@ from util.run_tests import run_tests from generators.generate_mutations import add_mutation_options, get_mutations_from_args -logo = '''Use -h to see the command line options +logo = '''Use [-h] to see the command line options __ __ _ _ _ | \/ | | | | | (_) @@ -157,7 +157,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, else: key_path = None - if ml_count == None: + if enable_ml and ml_count == None: while True: ml_count = questionary.text('How many different programs should be generated with ML?', default=str(DEFAULT_ML_COUNT)).ask() if not ml_count.strip('\n').isdigit(): From 5295752c9eca0aef89292f353a00c04a755bf34c Mon Sep 17 00:00:00 2001 From: J2000A Date: Wed, 7 Jun 2023 21:59:49 +0200 Subject: [PATCH 066/131] Fix imports --- .../test-generation/generators/generate_ml.py | 2 +- .../test-generation/util/__init__.py | 0 .../test-generation/util/add_check.py | 2 +- .../test-generation/util/generate_programs.py | 6 +++--- .../test-generation/util/generate_tests.py | 3 +-- .../test-generation/util/util.py | 1 - 6 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 scripts/incremental-test-generation/test-generation/util/__init__.py diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index 43ffbaf645..f6fedd9ab9 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -9,7 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from multiprocessing import Lock -sys.path.append("..") +sys.path.insert(0, "..") from util.util import * SEPERATOR_EXPLANATION_START = 'EXPLANATION>' diff --git a/scripts/incremental-test-generation/test-generation/util/__init__.py b/scripts/incremental-test-generation/test-generation/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index f3453174f3..1290f7a76e 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -6,7 +6,7 @@ import subprocess import sys import yaml -sys.path.append("..") +sys.path.insert(0, "..") from util.util import * def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index a36bae4ecf..b192047ea5 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -2,12 +2,12 @@ import os import shutil import sys -sys.path.append("..") -from generators.generate_mutations import * -from generators.generate_ml import * +sys.path.insert(0, "..") from util.util import * from util.add_check import add_check from util.add_check_comments import add_check_comments +from generators.generate_mutations import * +from generators.generate_ml import * # Run for example with: # python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index 6885eb6ae7..a9e24a3efb 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -6,8 +6,7 @@ import subprocess import yaml import sys -import datetime -sys.path.append("..") +sys.path.insert(0, "..") from util.util import * def generate_tests(temp_dir, target_dir, precision_test): diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index e8daf1be7c..a7d4f73764 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -1,7 +1,6 @@ from enum import Enum import shutil - class Mutations: def __init__(self, rfb=False, uoi=False, ror=False, cr=False, rt=False, lcr=False): self.rfb = rfb From 46405f507fcdc1d8a749adf929ed8abcf6c18241 Mon Sep 17 00:00:00 2001 From: J2000A Date: Wed, 7 Jun 2023 22:00:46 +0200 Subject: [PATCH 067/131] Update gitignore --- scripts/incremental-test-generation/git-patches/.gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/git-patches/.gitignore b/scripts/incremental-test-generation/git-patches/.gitignore index a0db2c2466..b3248a2919 100644 --- a/scripts/incremental-test-generation/git-patches/.gitignore +++ b/scripts/incremental-test-generation/git-patches/.gitignore @@ -1,2 +1,4 @@ testGit/* -testGitDif/* \ No newline at end of file +testGitDif/* +cJSON/* +cil.c \ No newline at end of file From b64399d0ef96d3199a351959eb51fab13ca46473 Mon Sep 17 00:00:00 2001 From: J2000A Date: Wed, 7 Jun 2023 22:25:36 +0200 Subject: [PATCH 068/131] Update gitignore --- scripts/incremental-test-generation/git-patches/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/git-patches/.gitignore b/scripts/incremental-test-generation/git-patches/.gitignore index b3248a2919..92da408abf 100644 --- a/scripts/incremental-test-generation/git-patches/.gitignore +++ b/scripts/incremental-test-generation/git-patches/.gitignore @@ -1,4 +1,5 @@ testGit/* testGitDif/* cJSON/* -cil.c \ No newline at end of file +cil.c +zlib/* From 1068617caabb95e9cc64dac40748eaa1cc74c6d4 Mon Sep 17 00:00:00 2001 From: J2000A Date: Wed, 7 Jun 2023 22:49:03 +0200 Subject: [PATCH 069/131] Add Repositories to sample files --- scripts/incremental-test-generation/sample-files/.gitignore | 1 + .../incremental-test-generation/sample-files/REPOSITORIES.md | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 scripts/incremental-test-generation/sample-files/.gitignore create mode 100644 scripts/incremental-test-generation/sample-files/REPOSITORIES.md diff --git a/scripts/incremental-test-generation/sample-files/.gitignore b/scripts/incremental-test-generation/sample-files/.gitignore new file mode 100644 index 0000000000..0193d313d6 --- /dev/null +++ b/scripts/incremental-test-generation/sample-files/.gitignore @@ -0,0 +1 @@ +**/ \ No newline at end of file diff --git a/scripts/incremental-test-generation/sample-files/REPOSITORIES.md b/scripts/incremental-test-generation/sample-files/REPOSITORIES.md new file mode 100644 index 0000000000..c3ecdad6e2 --- /dev/null +++ b/scripts/incremental-test-generation/sample-files/REPOSITORIES.md @@ -0,0 +1,3 @@ +# CMake Repositories +## zlib +`git clone https://github.com/madler/zlib.git` \ No newline at end of file From 09553e8db6c36e3a1787774917ff0ba5438dbfb3 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 11:08:02 +0200 Subject: [PATCH 070/131] Add Shell Script for Git Repositories --- .../test-generation/generators/generate_mutations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 22ace57761..0e721c2ad8 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -5,11 +5,10 @@ import json import os import re -import shutil import subprocess import sys import yaml -sys.path.append("..") +sys.path.insert(0, "..") from util.util import * def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): From edda276bf05ecee8bed6f11498a2c7ad738f133c Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 11:08:34 +0200 Subject: [PATCH 071/131] Add Shell Script for processing git repos --- .../sample-files/zlib.sh | 57 +++++++++++++++++++ .../generators/DEFAULT_GIT_PROCESSOR.sh | 57 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100755 scripts/incremental-test-generation/sample-files/zlib.sh create mode 100755 scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh new file mode 100755 index 0000000000..8ed7b443b5 --- /dev/null +++ b/scripts/incremental-test-generation/sample-files/zlib.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Script with relevant informations for processing a git repo + +############################################################## +####################### USER VARIABLES ####################### +############################################################## +# Variables +git_url="https://github.com/madler/zlib.git" +use_cmake=true +use_make=false +path_to_build="zlib/" +path_to_execute_cil="zlib/" + +# Functions before and after build +pre_build_commands() { + ./configure +} + +post_build_commands() { + : +} +############################################################## +####################### USER VARIABLES ####################### +############################################################## + +# Exit if any command fails +set -e + +# Export path for Mutation Generator +if [ "$1" == "--path" ]; then + echo "$(pwd)/$path_to_execute_cil" + exit 0 +fi + +# Check bool variables +if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then + echo "Error: Either cmake or make should be used, but not both or neither." + exit 1 +fi + +# Main script +repo_name=$(basename "$git_url" .git) +rm -rf $repo_name +git clone $git_url + +cd "$(dirname "$0")" +cd $path_to_build +pre_build_commands + +if $use_cmake; then + cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +elif $use_make; then + bear -- make +fi + +post_build_commands diff --git a/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh b/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh new file mode 100755 index 0000000000..4126ba4640 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Script with relevant informations for processing a git repo + +############################################################## +####################### USER VARIABLES ####################### +############################################################## +# Variables +git_url="" #e.g.: https://github.com/madler/zlib.git +use_cmake=true +use_make=false +path_to_build="" #e.g.: "zlib/" +path_to_execute_cil="" #e.g.: "zlib/" + +# Functions before and after build +pre_build_commands() { + : #e.g.: "./configure" +} + +post_build_commands() { + : +} +############################################################## +####################### USER VARIABLES ####################### +############################################################## + +# Exit if any command fails +set -e + +# Export path for Mutation Generator +if [ "$1" == "--path" ]; then + echo "$(pwd)/$path_to_execute_cil" + exit 0 +fi + +# Check bool variables +if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then + echo "Error: Either cmake or make should be used, but not both or neither." + exit 1 +fi + +# Main script +repo_name=$(basename "$git_url" .git) +rm -rf $repo_name +git clone $git_url + +cd "$(dirname "$0")" +cd $path_to_build +pre_build_commands + +if $use_cmake; then + cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +elif $use_make; then + bear -- make +fi + +post_build_commands From 402edb38dc49de7f3804eda552d013ab3d8db4bc Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 11:25:09 +0200 Subject: [PATCH 072/131] Seperate User Input and script itself --- .../sample-files/zlib.sh | 41 +++---------- .../generators/DEFAULT_GIT_PROCESSOR.sh | 57 ------------------- .../generators/generate_git_build.sh | 46 +++++++++++++++ .../generate_git_build_USER_INFO_PRESET.sh | 32 +++++++++++ 4 files changed, 86 insertions(+), 90 deletions(-) delete mode 100755 scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh create mode 100755 scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh create mode 100644 scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh index 8ed7b443b5..f576e131db 100755 --- a/scripts/incremental-test-generation/sample-files/zlib.sh +++ b/scripts/incremental-test-generation/sample-files/zlib.sh @@ -1,7 +1,5 @@ #!/bin/bash -# Script with relevant informations for processing a git repo - ############################################################## ####################### USER VARIABLES ####################### ############################################################## @@ -24,34 +22,11 @@ post_build_commands() { ####################### USER VARIABLES ####################### ############################################################## -# Exit if any command fails -set -e - -# Export path for Mutation Generator -if [ "$1" == "--path" ]; then - echo "$(pwd)/$path_to_execute_cil" - exit 0 -fi - -# Check bool variables -if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then - echo "Error: Either cmake or make should be used, but not both or neither." - exit 1 -fi - -# Main script -repo_name=$(basename "$git_url" .git) -rm -rf $repo_name -git clone $git_url - -cd "$(dirname "$0")" -cd $path_to_build -pre_build_commands - -if $use_cmake; then - cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -elif $use_make; then - bear -- make -fi - -post_build_commands +# Export variables so they can be used in the main script +export git_url +export use_cmake +export use_make +export path_to_build +export path_to_execute +export pre_build_commands +export post_build_commands \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh b/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh deleted file mode 100755 index 4126ba4640..0000000000 --- a/scripts/incremental-test-generation/test-generation/generators/DEFAULT_GIT_PROCESSOR.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Script with relevant informations for processing a git repo - -############################################################## -####################### USER VARIABLES ####################### -############################################################## -# Variables -git_url="" #e.g.: https://github.com/madler/zlib.git -use_cmake=true -use_make=false -path_to_build="" #e.g.: "zlib/" -path_to_execute_cil="" #e.g.: "zlib/" - -# Functions before and after build -pre_build_commands() { - : #e.g.: "./configure" -} - -post_build_commands() { - : -} -############################################################## -####################### USER VARIABLES ####################### -############################################################## - -# Exit if any command fails -set -e - -# Export path for Mutation Generator -if [ "$1" == "--path" ]; then - echo "$(pwd)/$path_to_execute_cil" - exit 0 -fi - -# Check bool variables -if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then - echo "Error: Either cmake or make should be used, but not both or neither." - exit 1 -fi - -# Main script -repo_name=$(basename "$git_url" .git) -rm -rf $repo_name -git clone $git_url - -cd "$(dirname "$0")" -cd $path_to_build -pre_build_commands - -if $use_cmake; then - cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -elif $use_make; then - bear -- make -fi - -post_build_commands diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh new file mode 100755 index 0000000000..63e1b13955 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Get the user infor about building the git file +if [[ -n $1 ]]; then + # Source the input script from the provided path + source $1 +else + echo "Please give the path to a build info sh file as an argument." + echo "The file should look like this:" + echo "" + cat generate_git_build_USER_INFO_PRESET.sh + echo "" + exit 1 +fi + +# Exit if any command fails +set -e + +# Export path for Mutation Generator +if [ "$2" == "--path" ]; then + echo "$(pwd)/$path_to_execute_cil" + exit 0 +fi + +# Check bool variables +if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then + echo "Error: Either cmake or make should be used, but not both or neither." + exit 1 +fi + +# Main script +repo_name=$(basename "$git_url" .git) +rm -rf $repo_name +git clone $git_url + +cd "$(dirname "$0")" +cd $path_to_build +pre_build_commands + +if $use_cmake; then + cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +elif $use_make; then + bear -- make +fi + +post_build_commands diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh new file mode 100644 index 0000000000..bd81a9f68e --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +############################################################## +####################### USER VARIABLES ####################### +############################################################## +# Variables +git_url="" #e.g.: https://github.com/madler/zlib.git +use_cmake=true +use_make=false +path_to_build="" #e.g.: "zlib/" +path_to_execute_cil="" #e.g.: "zlib/" + +# Functions before and after build +pre_build_commands() { + : #e.g.: ./configure +} + +post_build_commands() { + : +} +############################################################## +####################### USER VARIABLES ####################### +############################################################## + +# Export variables so they can be used in the main script +export git_url +export use_cmake +export use_make +export path_to_build +export path_to_execute +export pre_build_commands +export post_build_commands \ No newline at end of file From ae67de8bc5dac23c22da54d2da1835d3f5b1633e Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 11:50:50 +0200 Subject: [PATCH 073/131] Add option for output directory --- .../sample-files/zlib.sh | 4 +-- .../generators/generate_git.py | 0 .../generators/generate_git_build.sh | 26 ++++++++++++------- .../generate_git_build_USER_INFO_PRESET.sh | 10 +++---- 4 files changed, 23 insertions(+), 17 deletions(-) create mode 100644 scripts/incremental-test-generation/test-generation/generators/generate_git.py diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh index f576e131db..5c8e2df71a 100755 --- a/scripts/incremental-test-generation/sample-files/zlib.sh +++ b/scripts/incremental-test-generation/sample-files/zlib.sh @@ -7,8 +7,8 @@ git_url="https://github.com/madler/zlib.git" use_cmake=true use_make=false -path_to_build="zlib/" -path_to_execute_cil="zlib/" +path_to_build="." +path_to_execute_cil="." # Functions before and after build pre_build_commands() { diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 63e1b13955..5909d0446b 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -1,24 +1,31 @@ #!/bin/bash -# Get the user infor about building the git file +# Source the input script from the provided path if [[ -n $1 ]]; then - # Source the input script from the provided path source $1 else - echo "Please give the path to a build info sh file as an argument." - echo "The file should look like this:" + echo "Please give the following arguments: build_info_path output_path [--path]" echo "" + echo "The build_info sh file should look like this:" cat generate_git_build_USER_INFO_PRESET.sh echo "" exit 1 fi +# Get the output path from the provided path +if [[ -n $2 ]]; then + output_path="$(pwd)/$2" +else + echo "Please provide the relative path to the output as the second argument." + exit 1 +fi + # Exit if any command fails set -e # Export path for Mutation Generator -if [ "$2" == "--path" ]; then - echo "$(pwd)/$path_to_execute_cil" +if [ "$3" == "--path" ]; then + echo "$output_path/$path_to_execute_cil" exit 0 fi @@ -30,11 +37,10 @@ fi # Main script repo_name=$(basename "$git_url" .git) -rm -rf $repo_name -git clone $git_url +rm -rf "$output_path/$repo_name" +git clone $git_url "$output_path/$repo_name" -cd "$(dirname "$0")" -cd $path_to_build +cd "$output_path/$repo_name/$path_to_build" pre_build_commands if $use_cmake; then diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh index bd81a9f68e..477ebf496f 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh @@ -4,11 +4,11 @@ ####################### USER VARIABLES ####################### ############################################################## # Variables -git_url="" #e.g.: https://github.com/madler/zlib.git -use_cmake=true -use_make=false -path_to_build="" #e.g.: "zlib/" -path_to_execute_cil="" #e.g.: "zlib/" +git_url="" #e.g.: https://github.com/madler/zlib.git +use_cmake=true # Choose either cmake or make +use_make=false # Choose either cmake or make +path_to_build="" #e.g.: "." (Relative path in repo) +path_to_execute_cil="" #e.g.: "." (Relative path in repo) # Functions before and after build pre_build_commands() { From 545a8dc609e0019c472efaa9face4177efd2a302 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 12:04:07 +0200 Subject: [PATCH 074/131] Remove superfluous argument --- scripts/incremental-test-generation/sample-files/zlib.sh | 2 -- .../test-generation/generators/generate_git_build.sh | 6 ++++-- .../generators/generate_git_build_USER_INFO_PRESET.sh | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh index 5c8e2df71a..4e05e5a794 100755 --- a/scripts/incremental-test-generation/sample-files/zlib.sh +++ b/scripts/incremental-test-generation/sample-files/zlib.sh @@ -8,7 +8,6 @@ git_url="https://github.com/madler/zlib.git" use_cmake=true use_make=false path_to_build="." -path_to_execute_cil="." # Functions before and after build pre_build_commands() { @@ -27,6 +26,5 @@ export git_url export use_cmake export use_make export path_to_build -export path_to_execute export pre_build_commands export post_build_commands \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 5909d0446b..01608c1746 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -23,9 +23,12 @@ fi # Exit if any command fails set -e +# Get the name of the repo +repo_name=$(basename "$git_url" .git) + # Export path for Mutation Generator if [ "$3" == "--path" ]; then - echo "$output_path/$path_to_execute_cil" + echo "$output_path/$repo_name/$path_to_build" exit 0 fi @@ -36,7 +39,6 @@ if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then fi # Main script -repo_name=$(basename "$git_url" .git) rm -rf "$output_path/$repo_name" git clone $git_url "$output_path/$repo_name" diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh index 477ebf496f..b579a32be1 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh @@ -8,7 +8,6 @@ git_url="" #e.g.: https://github.com/madler/zlib.git use_cmake=true # Choose either cmake or make use_make=false # Choose either cmake or make path_to_build="" #e.g.: "." (Relative path in repo) -path_to_execute_cil="" #e.g.: "." (Relative path in repo) # Functions before and after build pre_build_commands() { @@ -27,6 +26,5 @@ export git_url export use_cmake export use_make export path_to_build -export path_to_execute export pre_build_commands export post_build_commands \ No newline at end of file From 7718a50786b098a0d8b92b6862a3199182a87a33 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 8 Jun 2023 12:13:42 +0200 Subject: [PATCH 075/131] Seperate parts of the script with options --- .../generators/generate_git_build.sh | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 01608c1746..7027cd2011 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -4,7 +4,7 @@ if [[ -n $1 ]]; then source $1 else - echo "Please give the following arguments: build_info_path output_path [--path]" + echo "Please give the following arguments: build_info_path output_path [--clone][--build][--path]" echo "" echo "The build_info sh file should look like this:" cat generate_git_build_USER_INFO_PRESET.sh @@ -20,9 +20,20 @@ else exit 1 fi +if [ "$3" == "" ]; then + echo "Please provide one of the options [--clone][--build][--path]" + exit 1 +fi + # Exit if any command fails set -e +# Check bool variables +if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then + echo "Error: Either cmake or make should be used, but not both or neither." + exit 1 +fi + # Get the name of the repo repo_name=$(basename "$git_url" .git) @@ -32,23 +43,22 @@ if [ "$3" == "--path" ]; then exit 0 fi -# Check bool variables -if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then - echo "Error: Either cmake or make should be used, but not both or neither." - exit 1 +# Clone repo +if [ "$3" == "--clone" ]; then + rm -rf "$output_path/$repo_name" + git clone $git_url "$output_path/$repo_name" fi -# Main script -rm -rf "$output_path/$repo_name" -git clone $git_url "$output_path/$repo_name" +# Build repo +if [ "$3" == "--build" ]; then + cd "$output_path/$repo_name/$path_to_build" + pre_build_commands -cd "$output_path/$repo_name/$path_to_build" -pre_build_commands + if $use_cmake; then + cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + elif $use_make; then + bear -- make + fi -if $use_cmake; then - cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -elif $use_make; then - bear -- make + post_build_commands fi - -post_build_commands From a1f9eed3d4d5bd375301150ec6ba64c1bdc07c60 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 11:45:21 +0200 Subject: [PATCH 076/131] integration of generateGit into whole script TODOs --- .../sample-files/zlib.sh | 4 +- .../generators/generate_git.py | 188 ++++++++++++++++++ .../generators/generate_git_build.sh | 2 +- .../test-generation/util/add_check.py | 6 +- .../util/add_check_comments.py | 26 +-- .../test-generation/util/generate_programs.py | 29 +-- .../test-generation/util/generate_tests.py | 100 ++++++---- 7 files changed, 271 insertions(+), 84 deletions(-) diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh index 4e05e5a794..b5192ef281 100755 --- a/scripts/incremental-test-generation/sample-files/zlib.sh +++ b/scripts/incremental-test-generation/sample-files/zlib.sh @@ -5,8 +5,8 @@ ############################################################## # Variables git_url="https://github.com/madler/zlib.git" -use_cmake=true -use_make=false +use_cmake=false +use_make=true path_to_build="." # Functions before and after build diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py index e69de29bb2..574fc0bd83 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git.py @@ -0,0 +1,188 @@ +import argparse +import copy +import os +import subprocess +import sys +from git import Repo +from datetime import datetime +import time +from pydriller import Repository +import yaml + +sys.path.insert(0, "..") +from util.util import * + +build_errors = 0 +checkout_errors = 0 +cil_errors = 0 +ids_errors = [] + +def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_window_ms, end_window_ms): + goblint_path = os.path.expanduser(os.path.abspath(goblint_path)) + temp_dir = os.path.expanduser(os.path.abspath(temp_dir)) + temp_repo_dir = os.path.join(temp_dir, 'repo') + meta_path = os.path.expanduser(os.path.abspath(meta_path)) + git_info_sh_path = os.path.expanduser(os.path.abspath(git_info_sh_path)) + + print(SEPERATOR) + print('[GIT] Cloning into temp/repo') + _clone_repo(args.git_info_sh_path, temp_repo_dir) + build_path = _get_build_path(args.git_info_sh_path, temp_repo_dir) + + repo = Repository(build_path) + all_commits = list(repo.traverse_commits()) + if not all_commits: + print(f"{COLOR_RED}No commits found in the repository{COLOR_RESET}") + sys.exit(-1) + earliest_commit = all_commits[0] + latest_commit = all_commits[-1] + if start_window_ms <= 0: + start = datetime.fromtimestamp(earliest_commit.committer_date.timestamp()) + else: + start = datetime.fromtimestamp(start_window_ms / 1000.0) + if end_window_ms <= 0: + end = datetime.fromtimestamp(latest_commit.committer_date.timestamp()) + else: + end = datetime.fromtimestamp(end_window_ms / 1000.0) + if not os.path.exists(temp_repo_dir): + os.mkdir(temp_repo_dir) + + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + index: int = yaml_data[META_N] + + def get_commit_traverser(): + return Repository(build_path, since=start, to=end).traverse_commits() + + num_of_commits = sum(1 for _ in get_commit_traverser()) + print(f'[GIT] Start traversing {num_of_commits} commits. Including checkout, build and cil generation. This may take a while...') + print(SEPERATOR) + t = 0 + for commit in get_commit_traverser(): + t += 1 + if commit.merge: + print(f"[GIT][{t}/{num_of_commits}] {COLOR_YELLOW}Skipping merge commit {commit.hash}{COLOR_RESET}, continuing with the next commit...") + continue + #TODO Skip small commits and summarize them in one big commit + try: + _checkout(build_path, meta_path, commit.hash) + _build_repo(args.git_info_sh_path, temp_repo_dir, meta_path, commit.hash) + new_path = os.path.join(temp_dir, f"p_{index}.c") + _create_cil_file(goblint_path, build_path, new_path, meta_path, commit.hash) + + if t % 10 == 0: + print(f"[GIT][{t}/{num_of_commits}] Written cil file with index {index}, continuing with the next commit...") + _write_meta_data(meta_path, commit.hash, index) + index += 1 + except Exception as e: + print(f"{COLOR_RED}[GIT][{t}/{num_of_commits}][FAIL] Analysis for commit {commit.hash} failed ({e}){COLOR_RESET}, continuing with the next commit...") + print(f"{COLOR_GREEN}[GIT][FINISHED] Finished creating cil files for the commits.{COLOR_RESET}") + if build_errors > 0 or checkout_errors > 0 or cil_errors > 0: + print(f"{COLOR_RED}There were the following errors: {build_errors} build errors, {checkout_errors} checkout errors and {cil_errors} cil errors.{COLOR_RESET}") + print(f"{COLOR_RED}The following commit ids resulted in errors:{COLOR_RESET} {', '.join(ids_errors)}") + return index + + +def _write_meta_data(meta_path, commit_hash, index): + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data[META_N] = index + yaml_data[f"p_{index}"] = { + META_TYPE: Generate_Type.GIT.value, + META_SUB_TYPE: commit_hash + } + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file) + +def _write_meta_data_failure(meta_path, commit_hash, stdout_msg, stderr_msg): + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data.setdefault(META_FAILURES, {})[commit_hash] = { + META_FAILURES_STD_OUT: stdout_msg, + META_FAILURES_STD_ERR: stderr_msg + } + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file) + +def _clone_repo(git_info_sh_path, temp_repo_path): + command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--clone"] + result = subprocess.run(command, text=True, capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + print(f"{COLOR_RED}Could not clone!{COLOR_RESET}") + sys.exit(-1) + +def _get_build_path(git_info_sh_path, temp_repo_path): + command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--path"] + result = subprocess.run(command, text=True, capture_output=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + print(f"{COLOR_RED}Could not get build path!{COLOR_RESET}") + sys.exit(-1) + build_path = os.path.normpath(result.stdout.strip()) + return build_path + +def _build_repo(git_info_sh_path, temp_repo_path, meta_path, commit_hash): + global build_errors + command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--build"] + result = subprocess.run(command, text=True, capture_output=True) + if result.returncode != 0: + build_errors += 1 + ids_errors.append(commit_hash) + _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) + raise Exception("Could not build repo!") + +def _checkout(build_path, meta_path, commit_hash): + global checkout_errors + # Clean untracked files + clean_command = ['git', '-C', build_path, 'clean', '-f'] + clean_result = subprocess.run(clean_command, text=True, capture_output=True) + if clean_result.returncode != 0: + checkout_errors += 1 + ids_errors.append(commit_hash) + _write_meta_data_failure(meta_path, commit_hash, clean_result.stdout, clean_result.stderr) + raise Exception("Could not clean untracked files!") + + # Stash any uncommitted changes + stash_command = ['git', '-C', build_path, 'stash'] + stash_result = subprocess.run(stash_command, text=True, capture_output=True) + if stash_result.returncode != 0: + checkout_errors += 1 + ids_errors.append(commit_hash) + _write_meta_data_failure(meta_path, commit_hash, stash_result.stdout, stash_result.stderr) + raise Exception("Could not stash changes!") + + # Checkout commit + command = ['git', '-C', build_path, 'checkout', commit_hash] + result = subprocess.run(command, text=True, capture_output=True) + if result.returncode != 0: + checkout_errors += 1 + ids_errors.append(commit_hash) + _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) + raise Exception("Could not checkout repo!") + +def _create_cil_file(goblint_path, build_path, output_path, meta_path, commit_hash): + global cil_errors + result = subprocess.run([goblint_path, '--set', 'justcil', 'true', '--set', 'cil.merge.inlines', 'false', build_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode != 0: + cil_errors += 1 + ids_errors.append(commit_hash) + _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) + raise Exception("Error creating cil!") + with open(output_path, 'w') as f: + f.write(result.stdout.decode()) + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Script for generating cil program files from commits") + parser.add_argument("goblint_path", help="Path to Goblint directory") + parser.add_argument("temp_dir", help="Path to the temporary directory") + parser.add_argument("meta_path", help="Path to the meta directory") + parser.add_argument("git_info_sh_path", help="Path to the Git information shell script") + parser.add_argument("start_window_ms", help="Start of the time window in milliseconds", type=int) + parser.add_argument("end_window_ms", help="End of the time window in milliseconds", type=int) + + args = parser.parse_args() + + generate_git(args.goblint_path, args.temp_dir, args.meta_path, args.git_info_sh_path, args.start_window_ms, args.end_window_ms) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 7027cd2011..524f72866f 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -55,7 +55,7 @@ if [ "$3" == "--build" ]; then pre_build_commands if $use_cmake; then - cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake "$output_path/$repo_name/$path_to_build" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON elif $use_make; then bear -- make fi diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index 1290f7a76e..53b7bfa368 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -19,7 +19,7 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): '["assert"]', "--set", "trans.output", - file_path.rsplit('.', 1)[0] + '_c.c', + file_path.rsplit('.', 1)[0] + '_check.c', file_path ] @@ -34,9 +34,9 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): yaml.safe_dump(yaml_data, file) if not compiling: - print(f"Error compiling program with index {index}.") + print(f"{COLOR_RED}Error compiling program with index {index}.{COLOR_RESET}") if index == 0: - print("The original program did not compile. Stopping program!") + print(f"{COLOR_RED}The original program did not compile. Stopping program!{COLOR_RESET}") sys.exit(-1) return False else: diff --git a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py index 886731de63..c0d3ae1449 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py @@ -6,13 +6,7 @@ import argparse import re -def add_check_comments(file_path: str, undefined_instead_of_success: bool, verbose = False): - if verbose: - if undefined_instead_of_success: - print("Add \"//UNDEFINED\" to the Goblint checks __goblint_check(exp);") - else: - print("Add \"//SUCCESS\" to the Goblint checks __goblint_check(exp);") - +def add_check_comments(file_path: str, unknown_instead_of_success: bool): with open(file_path, 'r') as file: lines = file.readlines() modified_lines = [] @@ -22,24 +16,16 @@ def add_check_comments(file_path: str, undefined_instead_of_success: bool, verbo match = re.search(r'(__goblint_check\(.*?\);)', line) if match: modified_line = match.group(1) - if undefined_instead_of_success: + if unknown_instead_of_success: modified_line += ' //UNKNOWN //SUCCESS' else: modified_line += ' //SUCCESS' line = line.replace(match.group(1), modified_line) modified_lines.append(line) - if undefined_instead_of_success: - new_file_name = file_path.rsplit('.', 1)[0] + '_u.c' - with open(new_file_name, 'w') as new_file: - new_file.writelines(modified_lines) - if verbose: - print("Processing complete. Modified lines have been added to the new file:", new_file_name) - else: - with open(file_path, 'w') as file: - file.writelines(modified_lines) - if verbose: - print("Processing complete. Modified lines have been added to the existing file:", file_path) + new_file_name = file_path.rsplit('.', 1)[0] + ('_unknown.c' if unknown_instead_of_success else '_success.c') + with open(new_file_name, 'w') as new_file: + new_file.writelines(modified_lines) if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -51,4 +37,4 @@ def add_check_comments(file_path: str, undefined_instead_of_success: bool, verbo if not (args.undefined or args.success): parser.error("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - add_check_comments(args.file, args.undefined, True) + add_check_comments(args.file, args.undefined) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index b192047ea5..6ca22199ae 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -8,13 +8,14 @@ from util.add_check_comments import add_check_comments from generators.generate_mutations import * from generators.generate_ml import * +from generators.generate_git import * # Run for example with: # python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations generate_type_source = "SOURCE" -def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git, ml_count): +def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git, ml_count): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -41,23 +42,26 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) if enable_git: - pass + #TODO Let user select start time + START_TIME_MS = 1315609200000 + #TODO Let user select end time + END_TIME_MS = 1315695600000 + index = generate_git(goblint_path, temp_dir, meta_path, program_path, START_TIME_MS, END_TIME_MS) # Add checks with comments print(SEPERATOR) - index += 1 - for i in range(index): + for i in range(index + 1): if i % 9 == 0: print(f"Generating goblint checks [{i+1}/{index}]") file_path = os.path.join(temp_dir, f"p_{i}.c") compiling = add_check(file_path, i, goblint_path, meta_path) if not compiling: continue - file_path = os.path.join(temp_dir, f"p_{i}_c.c") - if i == 0: - add_check_comments(file_path, True) - add_check_comments(file_path, False) - print(f"Generating goblint checks [DONE]") + file_path = os.path.join(temp_dir, f"p_{i}_check.c") + if i == 0 or enable_git: + add_check_comments(file_path, unknown_instead_of_success=True) + add_check_comments(file_path, unknown_instead_of_success=False) + print(f"{COLOR_GREEN}Generating goblint checks [DONE]{COLOR_RESET}") # Check how many and which files were not compiling print(SEPERATOR) @@ -71,12 +75,9 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api failed_count += 1 failed_compilation_keys.append(key) if failed_count == 0: - print("All files compiled succesfully") + print(f"{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") else: - print(f"There where {failed_count} files not compiling: {failed_compilation_keys}") - - - + print(f"{COLOR_RED}There where {failed_count} files not compiling:{COLOR_RESET} {failed_compilation_keys}") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index a9e24a3efb..e09a1c8be5 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -14,7 +14,7 @@ def generate_tests(temp_dir, target_dir, precision_test): directoryName = os.path.basename(target_dir) pattern = r"\d{2}-\w+" if not re.match(pattern, directoryName): - print("[ERROR] Target Directory name is not of the format 01-Name (\d{2}-\w+)") + print(f"{COLOR_RED}[ERROR] Target Directory name is not of the format 01-Name (\d{{2}}-\w+){COLOR_RESET}") return # Clear and create target_dir @@ -28,59 +28,71 @@ def generate_tests(temp_dir, target_dir, precision_test): yaml_data = yaml.safe_load(file) n = yaml_data[META_N] - source_program_id = 'p_0' - source_program = os.path.join(temp_dir, source_program_id + '_c_u.c') - source_program_precison = os.path.join(temp_dir, source_program_id + '_c.c') unchanged_count = 0 for i in range(n + 1): - generated_id = 'p_' + str(i) - generated_program = os.path.join(temp_dir, generated_id + '_c.c') - compilation_success = yaml_data[generated_id][META_COMPILING] + current_program_id = f'p_{i}' + compilation_success = yaml_data[current_program_id][META_COMPILING] if not compilation_success: - print(f"Generating test files [{i}/{n}] Skipped {i} (Not compiling)") + print(f"Generating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Not compiling){COLOR_RESET}") continue - type = yaml_data[generated_id][META_TYPE] - if type == Generate_Type.SOURCE.value: + if META_EXCEPTION in yaml_data[current_program_id]: + print(f"Generating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Exception occured during generation){COLOR_RESET}") + continue + type = yaml_data[current_program_id][META_TYPE] + if type == Generate_Type.SOURCE.value or (type == Generate_Type.GIT.value and i == 0): continue if (i-1) % 9 == 0: print(f"Generating test files [{i}/{n}]") + sub_type = yaml_data[current_program_id][META_SUB_TYPE] + if type == Generate_Type.MUTATION.value or type == Generate_Type.GIT.value: + test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) + else: + test_name = _format_number(i) + '-' + type + '_' + _format_number(i) + + # Select depending on generator the start and end file of at test if type == Generate_Type.MUTATION.value or type == Generate_Type.ML.value: - sub_type = yaml_data[generated_id][META_SUB_TYPE] - if type == Generate_Type.MUTATION.value: - test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) - else: - test_name = _format_number(i) + '-' + type + '_' + _format_number(i) - # Copy mutated code as the original code - shutil.copy2(generated_program, os.path.join(target_dir, test_name + '.c')) - # Create a patch file - patch_path = os.path.join(target_dir, test_name + '.patch') - command = 'diff -u {} {} > {}'.format( - os.path.join(target_dir, test_name + '.c'), - source_program_precison if precision_test else source_program, - patch_path - ) - result = subprocess.run(command, shell=True) - _fix_patch_file(patch_path, directoryName, test_name + '.c') - if result.returncode in [0, 1]: - if result.returncode == 0: - print(f"[WARNING] There were no changes in the patch for test {i}") - unchanged_count += 1 - yaml_data[generated_id][META_DIFF] = False - else: - yaml_data[generated_id][META_DIFF] = True + source_program_id = 'p_0' + start_program = os.path.join(temp_dir, current_program_id + '_check_success.c') + end_program = os.path.join(temp_dir, source_program_id + '_check_unknown.c') + end_program_precison = os.path.join(temp_dir, source_program_id + '_check_success.c') + elif type ==Generate_Type.GIT.value: + previous_program_id = f'p_{i-1}' + + start_program = os.path.join(temp_dir, previous_program_id + '_check_success.c') + end_program = os.path.join(temp_dir, current_program_id + '_check_unknown.c') + end_program_precison = os.path.join(temp_dir, current_program_id + '_check_success.c') + else: + print(f'{COLOR_RED}[ERROR] Trying to generate tests from unknown generator type{COLOR_RESET}') + sys.exit(-1) + + # Copy mutated code as the original code + shutil.copy2(start_program, os.path.join(target_dir, test_name + '.c')) + # Create a patch file + patch_path = os.path.join(target_dir, test_name + '.patch') + command = 'diff -u {} {} > {}'.format( + os.path.join(target_dir, test_name + '.c'), + end_program_precison if precision_test else end_program, + patch_path + ) + result = subprocess.run(command, shell=True) + _fix_patch_file(patch_path, directoryName, test_name + '.c') + if result.returncode in [0, 1]: + if result.returncode == 0: + print(f"{COLOR_YELLOW}[WARNING] There were no changes in the patch for test {i}{COLOR_RESET}") + unchanged_count += 1 + yaml_data[current_program_id][META_DIFF] = False else: - raise Exception("Command failed with return code: {}".format(result.returncode)) - # Create a empty config file - #TODO Support other config files - data = {} - with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: - json.dump(data, f) - elif type == Generate_Type.GIT.value: - #TODO - print ('[ERROR] Generating GIT Tests not implemented') + yaml_data[current_program_id][META_DIFF] = True else: - print ('[ERROR] Generating Unknown Tests not implemented') - print(f"Generating test files [DONE]") + raise Exception("Command failed with return code: {}".format(result.returncode)) + # Create a empty config file + #TODO Support other config files + data = {} + with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: + json.dump(data, f) + print(f"{COLOR_GREEN}Generating test files [DONE].{COLOR_RESET}") + if unchanged_count > 0: + print(f'{COLOR_YELLOW}There were {unchanged_count} patch files with no changes.{COLOR_RESET}') with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file) From 0708341798f644b5cba31d780c62c3e28b424565 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 11:45:35 +0200 Subject: [PATCH 077/131] Add coloring to terminal output --- .../test-generation/RUN_CLI.py | 39 ++++++----- .../test-generation/generators/generate_ml.py | 64 +++++++++++++------ .../generators/generate_mutations.py | 31 +++++---- .../test-generation/util/run_tests.py | 5 +- .../test-generation/util/util.py | 14 +++- 5 files changed, 97 insertions(+), 56 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 4172f04c83..a046746c55 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -10,8 +10,7 @@ from util.run_tests import run_tests from generators.generate_mutations import add_mutation_options, get_mutations_from_args -logo = '''Use [-h] to see the command line options - +logo = ''' __ __ _ _ _ | \/ | | | | | (_) | \ / |_ _| |_ __ _| |_ _ ___ _ __ @@ -41,24 +40,16 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, input_path, mutations, is_mutation, is_ml, is_git, ml_count) - #Write out custom test files - print(SEPERATOR) - print('Writing out custom test files:') - generate_tests(temp_path, os.path.join(os.path.curdir, '99-test'), precision_test=False) #TODO Custom name - if create_precision: - print(SEPERATOR) - print('Writing out custom precision files:') - generate_tests(temp_path, os.path.join(os.path.curdir, '98-precision'), precision_test=False) #TODO Custom name - + # Run tests if is_run_tests: test_path = os.path.abspath(os.path.join(os.path.curdir, '99-temp')) if create_precision: print(SEPERATOR) - print('Writing out precision test files for running:') + print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, precision_test=True) run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg print(SEPERATOR) - print('Writing out test files for running:') + print(f'Writing out {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, precision_test=False) run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg if os.path.exists(test_path): @@ -66,6 +57,19 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio #TODO Print link to html result and give summary + #Write out custom test files + print(SEPERATOR) + correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '99-test') + print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST{COLOR_RESET} files:') + generate_tests(temp_path, correctness_path, precision_test=False) #TODO Custom name + print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! + if create_precision: + print(SEPERATOR) + precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "98-precision") + print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST{COLOR_RESET} files:') + generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name + print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! + def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input, ml_count): # Check config file config_path = Path(CONFIG_FILENAME) @@ -102,7 +106,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, # check if 'Git' is selected along with other options if 'Git' in generators and len(generators) > 1: - print("If 'Git' is selected, no other options should be selected. Please select again.") + print(f"{COLOR_RED}If 'Git' is selected, no other options should be selected. Please select again.{COLOR_RESET}") continue else: break @@ -161,11 +165,11 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, while True: ml_count = questionary.text('How many different programs should be generated with ML?', default=str(DEFAULT_ML_COUNT)).ask() if not ml_count.strip('\n').isdigit(): - print("Please enter a valid number.") + print(f"{COLOR_RED}Please enter a valid number.{COLOR_RESET}") continue ml_count = int(ml_count.strip('\n')) if ml_count <= 0: - print("Please enter a number greater zero.") + print(f"{COLOR_RED}Please enter a number greater zero.{COLOR_RESET}") continue break @@ -184,7 +188,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input = questionary.text('Enter the path to the git repository for the mutations: ', default=last_input_git).ask() config.update({CONFIG_LAST_INPUT_GIT: input}) if not os.path.exists(input): - print("Please enter a valid path.") + print(f"{COLOR_RED}Please enter a valid path.{COLOR_RESET}") continue with open(config_path, 'w') as outfile: yaml.dump(config, outfile) @@ -194,6 +198,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, if __name__ == "__main__": + print(f'{COLOR_YELLOW}Use [-h] to see the command line options{COLOR_RESET}') print(logo) parser = argparse.ArgumentParser(description='Generates mutations for creating incremental tests') diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index f6fedd9ab9..55fe4a9b92 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -17,10 +17,15 @@ SEPERATOR_CODE_START = 'CODE>' SEPERATOR_CODE_END = ' 0 else '')) + return index + ml_count def _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index, lock): - time.sleep((index * 50)/1000) - new_path = make_program_copy(program_path, index) - (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index) - _write_meta_data(meta_path, new_path, selected_lines, explanation, index, lock) + try: + time.sleep((index * 50)/1000) + new_path = make_program_copy(program_path, index) + (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index) + _write_meta_data(meta_path, selected_lines, explanation, index, lock) + except Exception as e: + print(f"{COLOR_RED}[{index}] Error for request {index}:{COLOR_RESET} {e}") + _write_meta_data(meta_path, [], '', index, lock, exception=e) return index def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index): @@ -102,24 +115,35 @@ def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, i with open(new_path, "w") as file: file.writelines(lines) - print(f'[{index}] Finsihed request: {explanation}') + explanation_lines = explanation.splitlines() + limited_explanation = "\n".join(explanation_lines[:4]) + print(f'{COLOR_GREEN}[{index}] Finished request:{COLOR_RESET} {limited_explanation}') return (explanation, selected_lines) -def _write_meta_data(meta_path, new_path, selected_lines, explanation, index, lock): - with lock: - name = os.path.basename(new_path) +def _write_meta_data(meta_path, selected_lines, explanation, index, lock, exception=None): + global error_counter + lock.acquire() + try: with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) - yaml_data[META_N] = index - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.ML.value, - META_SUB_TYPE: explanation, - META_LINES: f'[{selected_lines.start}, {selected_lines.stop}]' - } + if exception == None: + yaml_data[f"p_{index}"] = { + META_TYPE: Generate_Type.ML.value, + META_SUB_TYPE: explanation, + META_LINES: f'[{selected_lines.start}, {selected_lines.stop}]' + } + else: + error_counter += 1 + yaml_data[f"p_{index}"] = { + META_TYPE: Generate_Type.ML.value, + META_EXCEPTION: str(exception) + } with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file, sort_keys=False) + yaml.safe_dump(yaml_data, file) + finally: + lock.release() def _make_gpt_request(snippet): prompt = f''' @@ -172,18 +196,18 @@ def validate_interesting_lines(intersting_lines_string: str, program_path): try: intersting_lines = ast.literal_eval(intersting_lines_string) except SyntaxError: - print(f"The format \"{intersting_lines_string}\" is incorrect! Please use a format like this: \"[1, 42, 99]\"") + print(f"{COLOR_RED}The format \"{intersting_lines_string}\" is incorrect! Please use a format like this: \"[1, 42, 99]\"{COLOR_RESET}") return None except Exception as e: - print(f"An unexpected error occurred reading the input \"{intersting_lines_string}\": {e}") + print(f"{COLOR_RED}An unexpected error occurred reading the input \"{intersting_lines_string}\":{COLOR_RESET} {e}") return None for line in intersting_lines: if line <= 0: - print(f"All lines in \"{intersting_lines_string}\" should be greater zero!") + print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be greater zero!{COLOR_RESET}") return None if line > max_line: - print(f"All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!") + print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!{COLOR_RESET}") return None return intersting_lines @@ -201,7 +225,7 @@ def validate_interesting_lines(intersting_lines_string: str, program_path): interesting_lines = validate_interesting_lines(args.interesting_lines, args.program) if interesting_lines == None: - print('Stopped program execution') + print(f'{COLOR_RED}Stopped program execution{COLOR_RESET}') sys.exit(-1) generate_ml(args.program, args.apikey, args.meta_path, int(args.ml_count), int(args.num_selected_lines), interesting_lines) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 0e721c2ad8..8a4d9e0d98 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -42,15 +42,14 @@ def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mut # When Remove Thread create wrapper an then apply the mutations if len(lines) != 1: # Needed to prevent conflicts on generating wrappers - print("ERROR When applying remove_thread there always should be exactly one line") + print(f"{COLOR_RED}ERROR When applying remove_thread there always should be exactly one line{COLOR_RESET}") function_name = _get_thread_function_name(clang_tidy_path, lines, new_path, index) _wrap_thread_function(clang_tidy_path, new_path, function_name, index) _apply_mutation(clang_tidy_path, mutation_name, lines, new_path, index) - _write_meta_data(meta_path, new_path, index, mutation_name, lines) + _write_meta_data(meta_path, index, mutation_name, lines) return index def _get_line_groups(clang_tidy_path, mutation_name, program_path, index): - #TODO Handle [MACRO] tags command = [ clang_tidy_path, "-checks=-*,readability-" + mutation_name, @@ -59,11 +58,11 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path, index): ] result = subprocess.run(command, text=True, capture_output=True) - print(f"[MUTATION][CHECK] Check mutation {mutation_name} with return code {result.returncode}") + print(f"[MUTATION][CHECK] Check mutation {mutation_name}") if result.returncode != 0: print(result.stdout) print(result.stderr) - print("ERROR Running Clang") + print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") sys.exit(-1) line_groups = [] @@ -109,11 +108,12 @@ def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): ] result = subprocess.run(command, text=True, capture_output=True) - print(f"[{index}] Run mutation {mutation_name} on lines {lines} with return code {result.returncode}") - if result.returncode != 0: + if result.returncode == 0: + print(f"{COLOR_GREEN}[{index}] Finished mutation:{COLOR_RESET} {mutation_name} on lines {lines}") + else: print(result.stdout) print(result.stderr) - print("ERROR Running Clang") + print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") sys.exit(-1) def _get_thread_function_name(clang_tidy_path, lines, program_path, index): @@ -128,11 +128,11 @@ def _get_thread_function_name(clang_tidy_path, lines, program_path, index): "--" ] result = subprocess.run(command, text=True, capture_output=True) - print(f"[{index}][WRAP] Check function name for wrapping thread function with return code {result.returncode}") + print(f"[{index}][WRAP] Check function name for wrapping thread function") if result.returncode != 0: print(result.stdout) print(result.stderr) - print("ERROR Running Clang") + print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") sys.exit(-1) function_name_pattern = r"\[FUNCTION_NAME\]\[(.*?)\]" @@ -149,7 +149,7 @@ def _get_thread_function_name(clang_tidy_path, lines, program_path, index): def _wrap_thread_function(clang_tidy_path, program_path, function_name, index): if function_name == None: - print(f"[{index}][WRAP FIX] No function name was provided. Hope the program will compile without wrapping") + print(f"{COLOR_YELLOW}[{index}][WRAP FIX] No function name was provided. Hope the program will compile without wrapping{COLOR_RESET}") return check_options = {"CheckOptions": {"readability-remove-thread-wrapper.WrapFunctionName": function_name}} @@ -163,15 +163,14 @@ def _wrap_thread_function(clang_tidy_path, program_path, function_name, index): "--" ] result = subprocess.run(command, text=True, capture_output=True) - print(f"[{index}][WRAP FIX] Apply the wrapping of {function_name} with return code {result.returncode}") + print(f"[{index}][WRAP FIX] Apply the wrapping of {function_name}") if result.returncode != 0: print(result.stdout) print(result.stderr) - print("ERROR Running Clang") + print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") sys.exit(-1) -def _write_meta_data(meta_path, new_path, index, mutation_name, lines): - name = os.path.basename(new_path) +def _write_meta_data(meta_path, index, mutation_name, lines): with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) yaml_data[META_N] = index @@ -181,7 +180,7 @@ def _write_meta_data(meta_path, new_path, index, mutation_name, lines): META_LINES: lines } with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file, sort_keys=False) + yaml.safe_dump(yaml_data, file) def add_mutation_options(parser): parser.add_argument("-rfb", "--remove-function-body", action="store_true", help="Option for \"remove function body\" mutation") diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index fef7fd40f3..9920395bf4 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -2,12 +2,15 @@ import os import shutil import subprocess +import sys +sys.path.insert(0, "..") +from util.util import * def run_tests(test_dir, goblint_repo_dir, cfg): # Check the name of the test_dir test_dir_name = os.path.basename(test_dir) if test_dir_name != "99-temp": - print("[ERROR] The test directory name has to be \'99-temp\'") + print(f"{COLOR_RED}[ERROR] The test directory name has to be \'99-temp\'{COLOR_RESET}") incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) if os.path.exists(incremental_tests_dir_abs): diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index a7d4f73764..c89a789a59 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -22,15 +22,25 @@ class Generate_Type(Enum): ML = 'ML' GIT = 'GIT' -SEPERATOR = "--------------------" +COLOR_RED = '\033[31m' +COLOR_BLUE = '\033[34m' +COLOR_GREEN = '\033[32m' +COLOR_YELLOW = '\033[33m' +COLOR_RESET = '\033[0m' + +SEPERATOR = f"{COLOR_BLUE}------------------------------------------------------------------------------------------------------------------{COLOR_RESET}" META_FILENAME = 'meta.yaml' META_N = 'n' META_COMPILING = 'compilation' +META_EXCEPTION = 'exception' META_DIFF = 'diff' META_TYPE = 'type' META_SUB_TYPE = 'sub_type' META_LINES = 'lines' +META_FAILURES = 'failures' +META_FAILURES_STD_OUT = 'stdout' +META_FAILURES_STD_ERR = 'stderr' CONFIG_FILENAME = 'config.yaml' CONFIG_GOBLINT = "goblint-path" @@ -43,7 +53,7 @@ class Generate_Type(Enum): APIKEY_ORGANISATION = 'organisation' DEFAULT_ML_COUNT = 5 -ML_WORKERS = 10 +ML_WORKERS = 5 def make_program_copy(program_path, index): new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' From ce1e0d4d7b79436a4befcf4a5dfdbab4c68d4080 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 15:13:51 +0200 Subject: [PATCH 078/131] Rename script --- .../test-generation/generators/generate_git_build.sh | 2 +- ..._INFO_PRESET.sh => generate_git_build_USER_INFO_TEMPLATE.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/incremental-test-generation/test-generation/generators/{generate_git_build_USER_INFO_PRESET.sh => generate_git_build_USER_INFO_TEMPLATE.sh} (100%) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 524f72866f..45ab92f875 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -7,7 +7,7 @@ else echo "Please give the following arguments: build_info_path output_path [--clone][--build][--path]" echo "" echo "The build_info sh file should look like this:" - cat generate_git_build_USER_INFO_PRESET.sh + cat generate_git_build_USER_INFO_TEMPLATE.sh echo "" exit 1 fi diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_PRESET.sh rename to scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh From ea3edf8caa52087f9bbe488800afbbdf4d17b783 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 15:14:15 +0200 Subject: [PATCH 079/131] Consider not compiling programs --- .../generators/generate_git.py | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py index 574fc0bd83..c26dc8f4bc 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git.py @@ -1,11 +1,8 @@ import argparse -import copy import os import subprocess import sys -from git import Repo from datetime import datetime -import time from pydriller import Repository import yaml @@ -17,7 +14,7 @@ cil_errors = 0 ids_errors = [] -def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_window_ms, end_window_ms): +def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_commit, end_commit): goblint_path = os.path.expanduser(os.path.abspath(goblint_path)) temp_dir = os.path.expanduser(os.path.abspath(temp_dir)) temp_repo_dir = os.path.join(temp_dir, 'repo') @@ -26,61 +23,70 @@ def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_wind print(SEPERATOR) print('[GIT] Cloning into temp/repo') - _clone_repo(args.git_info_sh_path, temp_repo_dir) - build_path = _get_build_path(args.git_info_sh_path, temp_repo_dir) + _clone_repo(git_info_sh_path, temp_repo_dir) + build_path = _get_build_path(git_info_sh_path, temp_repo_dir) - repo = Repository(build_path) - all_commits = list(repo.traverse_commits()) - if not all_commits: - print(f"{COLOR_RED}No commits found in the repository{COLOR_RESET}") - sys.exit(-1) - earliest_commit = all_commits[0] - latest_commit = all_commits[-1] - if start_window_ms <= 0: - start = datetime.fromtimestamp(earliest_commit.committer_date.timestamp()) - else: - start = datetime.fromtimestamp(start_window_ms / 1000.0) - if end_window_ms <= 0: - end = datetime.fromtimestamp(latest_commit.committer_date.timestamp()) - else: - end = datetime.fromtimestamp(end_window_ms / 1000.0) if not os.path.exists(temp_repo_dir): os.mkdir(temp_repo_dir) + def get_commit_traverser(): + all_commits = list(Repository(build_path).traverse_commits()) + if start_commit == None: + start_index = 0 + else: + start_index = next((i for i, commit in enumerate(all_commits) if commit.hash == start_commit), None) + if end_commit == None: + end_index = len(all_commits) - 1 + else: + end_index = next((i for i, commit in enumerate(all_commits) if commit.hash == end_commit), None) + + if start_index is None or end_index is None: + raise ValueError("One or both commit hashes not found in the repository") + + return all_commits[start_index:end_index + 1] + + with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) index: int = yaml_data[META_N] - def get_commit_traverser(): - return Repository(build_path, since=start, to=end).traverse_commits() - num_of_commits = sum(1 for _ in get_commit_traverser()) print(f'[GIT] Start traversing {num_of_commits} commits. Including checkout, build and cil generation. This may take a while...') + if num_of_commits <= 2: + print(f'{COLOR_RED}You must traverse at least two commits to generate a test!') + sys.exit(-1) print(SEPERATOR) t = 0 + #last_failed = False for commit in get_commit_traverser(): t += 1 if commit.merge: print(f"[GIT][{t}/{num_of_commits}] {COLOR_YELLOW}Skipping merge commit {commit.hash}{COLOR_RESET}, continuing with the next commit...") + last_failed = True continue #TODO Skip small commits and summarize them in one big commit try: _checkout(build_path, meta_path, commit.hash) - _build_repo(args.git_info_sh_path, temp_repo_dir, meta_path, commit.hash) + _build_repo(git_info_sh_path, temp_repo_dir, meta_path, commit.hash) new_path = os.path.join(temp_dir, f"p_{index}.c") _create_cil_file(goblint_path, build_path, new_path, meta_path, commit.hash) - if t % 10 == 0: - print(f"[GIT][{t}/{num_of_commits}] Written cil file with index {index}, continuing with the next commit...") + #if t % 10 == 0 or last_failed: + #last_failed = False + #print(f"{COLOR_GREEN}[GIT][{t}/{num_of_commits}] Written cil file with index {index}{COLOR_RESET}, continuing with the next commits...") + print(f"{COLOR_GREEN}[{t}/{num_of_commits}] Written cil file with index [{index}] for commit {commit.hash}{COLOR_RESET}, continuing with the next commits...") _write_meta_data(meta_path, commit.hash, index) index += 1 except Exception as e: - print(f"{COLOR_RED}[GIT][{t}/{num_of_commits}][FAIL] Analysis for commit {commit.hash} failed ({e}){COLOR_RESET}, continuing with the next commit...") - print(f"{COLOR_GREEN}[GIT][FINISHED] Finished creating cil files for the commits.{COLOR_RESET}") + #last_failed = True + print(f"{COLOR_RED}[{t}/{num_of_commits}][FAIL] Generating cil for commit {commit.hash} failed ({e}){COLOR_RESET}, continuing with the next commit...") + print(SEPERATOR) + print(f"{COLOR_GREEN}[FINISHED] Finished creating cil files for the commits.{COLOR_RESET}") if build_errors > 0 or checkout_errors > 0 or cil_errors > 0: print(f"{COLOR_RED}There were the following errors: {build_errors} build errors, {checkout_errors} checkout errors and {cil_errors} cil errors.{COLOR_RESET}") + print(f"{COLOR_RED}You can read the error messages in the {meta_path} file") print(f"{COLOR_RED}The following commit ids resulted in errors:{COLOR_RESET} {', '.join(ids_errors)}") - return index + return index - 1 def _write_meta_data(meta_path, commit_hash, index): @@ -105,7 +111,7 @@ def _write_meta_data_failure(meta_path, commit_hash, stdout_msg, stderr_msg): yaml.safe_dump(yaml_data, file) def _clone_repo(git_info_sh_path, temp_repo_path): - command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--clone"] + command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--clone"] result = subprocess.run(command, text=True, capture_output=True) if result.returncode != 0: print(result.stdout) @@ -114,7 +120,7 @@ def _clone_repo(git_info_sh_path, temp_repo_path): sys.exit(-1) def _get_build_path(git_info_sh_path, temp_repo_path): - command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--path"] + command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--path"] result = subprocess.run(command, text=True, capture_output=True) if result.returncode != 0: print(result.stdout) @@ -126,7 +132,7 @@ def _get_build_path(git_info_sh_path, temp_repo_path): def _build_repo(git_info_sh_path, temp_repo_path, meta_path, commit_hash): global build_errors - command = ["./generate_git_build.sh", git_info_sh_path, temp_repo_path, "--build"] + command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--build"] result = subprocess.run(command, text=True, capture_output=True) if result.returncode != 0: build_errors += 1 From 3d7be0f667658b5fdb60804b8e188a0ecbd07364 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 15:15:57 +0200 Subject: [PATCH 080/131] Print template script and ask for sh path --- .../test-generation/RUN_CLI.py | 18 ++++++++++++-- .../test-generation/util/generate_programs.py | 24 ++++++++----------- .../test-generation/util/generate_tests.py | 14 +++++++++-- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index a046746c55..9ca35bcfec 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -1,6 +1,7 @@ import argparse import os import shutil +import sys import questionary import yaml from pathlib import Path @@ -38,7 +39,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, input_path, mutations, is_mutation, is_ml, is_git, ml_count) + gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count) # Run tests if is_run_tests: @@ -185,7 +186,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input = questionary.text('Enter the path to the c program for the mutations: ', default=last_input_mutation).ask() config.update({CONFIG_LAST_INPUT_MUTATION: input}) else: - input = questionary.text('Enter the path to the git repository for the mutations: ', default=last_input_git).ask() + input = questionary.text('Enter the path to the sh script with informations about the git repository (Use [-s] to see the template script ): ', default=last_input_git).ask() config.update({CONFIG_LAST_INPUT_GIT: input}) if not os.path.exists(input): print(f"{COLOR_RED}Please enter a valid path.{COLOR_RESET}") @@ -217,8 +218,21 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, # Add ML options parser.add_argument('-c', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') + # Add GIT options + parser.add_argument('-s', '--template-script', action='store_true', help='Print the template script for git repositories') + args = parser.parse_args() + if args.template_script: + template_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'generators', 'generate_git_build_USER_INFO_TEMPLATE.sh')) + print(f'{COLOR_YELLOW}Template can be found at: {template_path}{COLOR_RESET}') + print('') + with open(template_path, 'r') as file: + content = file.read() + print(content) + print('') + sys.exit(0) + if args.enable_mutations or args.enable_ml or args.enable_git: # If using git, only git can be used if args.enable_git and (args.enable_ml or args.enable_mutations): diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 6ca22199ae..d225854f38 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -15,7 +15,7 @@ generate_type_source = "SOURCE" -def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, git_url, mutations, enable_mutations, enable_ml, enable_git, ml_count): +def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -25,7 +25,7 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api with open(meta_path, 'w') as outfile: yaml.dump({'n': 0, 'p_0': {META_TYPE: generate_type_source}}, outfile) # Copy the source program into the temp dir - program_path = os.path.join(temp_dir, 'p.c') + program_path = os.path.join(temp_dir, 'p.c' if not enable_git else 'p.sh') shutil.copy2(source_path, program_path) program_0_path = os.path.join(temp_dir, 'p_0.c') shutil.copy2(source_path, program_0_path) @@ -42,17 +42,17 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) if enable_git: - #TODO Let user select start time - START_TIME_MS = 1315609200000 - #TODO Let user select end time - END_TIME_MS = 1315695600000 - index = generate_git(goblint_path, temp_dir, meta_path, program_path, START_TIME_MS, END_TIME_MS) + #TODO Let user select start + START_COMMIT = 'da6f1623c177c5ebfa2b1ee3b50eb297da5a77e1' + #TODO Let user select end + END_COMMIT = '04f42ceca40f73e2978b50e93806c2a18c1281fc' + index = generate_git(goblint_path, temp_dir, meta_path, program_path, START_COMMIT, END_COMMIT) # Add checks with comments print(SEPERATOR) for i in range(index + 1): if i % 9 == 0: - print(f"Generating goblint checks [{i+1}/{index}]") + print(f"[{i+1}/{index}] Generating goblint checks...") file_path = os.path.join(temp_dir, f"p_{i}.c") compiling = add_check(file_path, i, goblint_path, meta_path) if not compiling: @@ -81,12 +81,11 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') - parser.add_argument('source_path', help='Path to the original program provided by the user') + parser.add_argument('source_path', help='Path to the original program or git sh file provided by the user') parser.add_argument('temp_dir', help='Path to the working directory') parser.add_argument('clang_tidy_path', help='Path to the modified clang-tidy executable') parser.add_argument('goblint_path', help='Path to the goblint executable') parser.add_argument('--apikey-path', help='Path to the API') - parser.add_argument('--git-url', help='Git URL') parser.add_argument('--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') parser.add_argument('--enable-ml', action='store_true', help='Enable ML') parser.add_argument('--enable-git', action='store_true', help='Enable Git') @@ -114,7 +113,4 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if args.enable_ml and not args.apikey_path: parser.error("--enable-ml requires --apikey-path") - if args.enable_git and not args.git_url: - parser.error("--enable-git requires --git-url") - - gernerate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, args.git_url, mutations, args.enable_mutations, args.enable_ml, args.enable_git) + gernerate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index e09a1c8be5..a01af85fb5 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -29,10 +29,13 @@ def generate_tests(temp_dir, target_dir, precision_test): n = yaml_data[META_N] unchanged_count = 0 + compiling_programs = [] for i in range(n + 1): current_program_id = f'p_{i}' compilation_success = yaml_data[current_program_id][META_COMPILING] - if not compilation_success: + if compilation_success: + compiling_programs.append(i) + else: print(f"Generating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Not compiling){COLOR_RESET}") continue if META_EXCEPTION in yaml_data[current_program_id]: @@ -56,7 +59,14 @@ def generate_tests(temp_dir, target_dir, precision_test): end_program = os.path.join(temp_dir, source_program_id + '_check_unknown.c') end_program_precison = os.path.join(temp_dir, source_program_id + '_check_success.c') elif type ==Generate_Type.GIT.value: - previous_program_id = f'p_{i-1}' + # If it's the first compiling program skip it. + if i == compiling_programs[0]: + print(f"Generating test files [{i}/{n}] {COLOR_BLUE}Skipped {i} as the source file for the first test{COLOR_RESET}") + continue + + # Find the index of the previous compiling program. + previous_program_index = compiling_programs.index(i) - 1 + previous_program_id = f'p_{compiling_programs[previous_program_index]}' start_program = os.path.join(temp_dir, previous_program_id + '_check_success.c') end_program = os.path.join(temp_dir, current_program_id + '_check_unknown.c') From 134a17d0504436f51b865d71da7bad762f083c7a Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 9 Jun 2023 15:25:57 +0200 Subject: [PATCH 081/131] Add exception printing for comilation errors --- .../test-generation/util/add_check.py | 10 +++++++++- .../test-generation/util/generate_programs.py | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index 53b7bfa368..856fd7b03b 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -35,9 +35,17 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): if not compiling: print(f"{COLOR_RED}Error compiling program with index {index}.{COLOR_RESET}") - if index == 0: + if index == 0 and not yaml_data["p_0"][META_TYPE] == Generate_Type.GIT.value: print(f"{COLOR_RED}The original program did not compile. Stopping program!{COLOR_RESET}") sys.exit(-1) + with open(meta_path, 'r') as file: + yaml_data = yaml.safe_load(file) + yaml_data[f"p_{index}"] = { + META_TYPE: Generate_Type.ML.value, + META_EXCEPTION: result.stderr + } + with open(meta_path, 'w') as file: + yaml.safe_dump(yaml_data, file) return False else: return True diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index d225854f38..3b778529a4 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -51,8 +51,8 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api # Add checks with comments print(SEPERATOR) for i in range(index + 1): - if i % 9 == 0: - print(f"[{i+1}/{index}] Generating goblint checks...") + if i % 10 == 0: + print(f"[{i}/{index}] Generating goblint checks...") file_path = os.path.join(temp_dir, f"p_{i}.c") compiling = add_check(file_path, i, goblint_path, meta_path) if not compiling: @@ -77,7 +77,7 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if failed_count == 0: print(f"{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") else: - print(f"{COLOR_RED}There where {failed_count} files not compiling:{COLOR_RESET} {failed_compilation_keys}") + print(f"{COLOR_RED}There where {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') From d0ea8c2874fe2eb5c26de6a06846edc36325c49f Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 13:49:24 +0200 Subject: [PATCH 082/131] Update gitignore --- scripts/incremental-test-generation/sample-files/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/sample-files/.gitignore b/scripts/incremental-test-generation/sample-files/.gitignore index 0193d313d6..9ca8693c30 100644 --- a/scripts/incremental-test-generation/sample-files/.gitignore +++ b/scripts/incremental-test-generation/sample-files/.gitignore @@ -1 +1,2 @@ -**/ \ No newline at end of file +**/ +test.* From 79b953cd79fe7e63146f6edf9a896f753c1ef1a7 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 15:00:36 +0200 Subject: [PATCH 083/131] Remove obsolete files --- .../git-patches/.gitignore | 5 - .../git-patches/generate_cil.py | 58 ----- .../git-patches/generate_git.py | 87 ------- .../git-patches/git-patches.py | 45 ---- .../openai-text-completion/.gitignore | 1 - .../openai-text-completion/ml.py | 77 ------ .../outputs/20230522-01.txt | 233 ------------------ .../outputs/20230523-01.txt | 25 -- 8 files changed, 531 deletions(-) delete mode 100644 scripts/incremental-test-generation/git-patches/.gitignore delete mode 100644 scripts/incremental-test-generation/git-patches/generate_cil.py delete mode 100644 scripts/incremental-test-generation/git-patches/generate_git.py delete mode 100644 scripts/incremental-test-generation/git-patches/git-patches.py delete mode 100644 scripts/incremental-test-generation/openai-text-completion/.gitignore delete mode 100644 scripts/incremental-test-generation/openai-text-completion/ml.py delete mode 100644 scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt delete mode 100644 scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt diff --git a/scripts/incremental-test-generation/git-patches/.gitignore b/scripts/incremental-test-generation/git-patches/.gitignore deleted file mode 100644 index 92da408abf..0000000000 --- a/scripts/incremental-test-generation/git-patches/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -testGit/* -testGitDif/* -cJSON/* -cil.c -zlib/* diff --git a/scripts/incremental-test-generation/git-patches/generate_cil.py b/scripts/incremental-test-generation/git-patches/generate_cil.py deleted file mode 100644 index e9f9735fff..0000000000 --- a/scripts/incremental-test-generation/git-patches/generate_cil.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import argparse -import subprocess -import sys - -BUILD_CMAKE = 'cmake' -BUILD_MAKE = 'make' - -def cil_transform(goblint_path, project_dir, output_dir, build_system): - goblint_path = os.path.abspath(goblint_path) - project_dir = os.path.abspath(project_dir) - output_dir = os.path.abspath(output_dir) - - # Check if Makefile/CMakeLists.txt exists in the directory - build_file = 'Makefile' if build_system == BUILD_MAKE else 'CMakeLists.txt' - if not os.path.isfile(os.path.join(project_dir, build_file)): - print(f"No {build_file} found in the project directory: {project_dir}") - return - - # Navigate to the directory - original_dir = os.getcwd() - os.chdir(project_dir) - - # Run make/cmake command - if build_system == BUILD_MAKE: - build_process = subprocess.run([build_system, 'DCMAKE_EXPORT_COMPILE_COMMANDS=ON']) - elif build_system == BUILD_CMAKE: - build_process = subprocess.run(['bear', '--', 'make']) - if build_process.returncode != 0: - print(f"Error in {build_system} command:") - print(build_process.stderr.decode()) - return - - # Run goblint with cil option - goblint_process = subprocess.run([goblint_path, '--set', 'justcil', 'true', '--set', 'cil.merge.inlines', 'false', project_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if goblint_process.returncode != 0: - print("Error in goblint command:") - print(goblint_process.stderr.decode()) - return - - # Output the transformed CIL code - with open(os.path.join(output_dir, 'cil.c'), 'w') as f: - f.write(goblint_process.stdout.decode()) - print("CIL transformation completed. Transformed code written to cil.c") - - # Navigate back to the original directory - os.chdir(original_dir) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Transform a project into a single CIL file using Goblint.') - parser.add_argument('goblint_path', type=str, help='Path to the goblint binary') - parser.add_argument('project_dir', type=str, help='Path to the project directory') - parser.add_argument('output_dir', type=str, help='Path to the output directory') - parser.add_argument('--build', choices=[BUILD_MAKE, BUILD_CMAKE], required=True, help=f'Build system to use ({BUILD_MAKE} or {BUILD_CMAKE})') - - args = parser.parse_args() - cil_transform(args.goblint_path, args.project_dir, args.output_dir, args.build) diff --git a/scripts/incremental-test-generation/git-patches/generate_git.py b/scripts/incremental-test-generation/git-patches/generate_git.py deleted file mode 100644 index f69d1831ac..0000000000 --- a/scripts/incremental-test-generation/git-patches/generate_git.py +++ /dev/null @@ -1,87 +0,0 @@ -# File based on bench/scripts/incremental/benchmarking/stats.py from the Goblint bench repository -# https://github.com/goblint/bench/blob/6333415e9b5b5157b3b462b673726faa93c97488/scripts/incremental/benchmarking/stats.py - -from pydriller import Repository -from datetime import datetime -from pathlib import Path -import os -import sys - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Wrong number of parameters.\nUse script like this: python3 generate_git.py ") - exit() - -analyzer_dir = sys.argv[1] -url = 'https://github.com/facebook/zstd' -repo_name = 'zstd' -begin = datetime(2021,8,1) -to = datetime(2022,2,1) -maxCLOC = 50 -dirs_to_exclude = ["build", "doc", "examples", "tests", "zlibWrapper", "contrib"] - -cwd = os.getcwd() -outdir = os.path.join(cwd, 'out') -repo_path = os.path.normpath(os.path.join(cwd, repo_name)) -paths_to_exclude = list(map(lambda x: os.path.join(repo_path, x), dirs_to_exclude)) - -analyzed_commits = {} -total_commits = 0 -count_nochanges = 0 -count_merge = 0 -count_big = 0 -count_small = 0 - -# Function based on bench/scripts/incremental/benchmarking/utils.py from Goblint bench repository -# https://github.com/goblint/bench/blob/6333415e9b5b5157b3b462b673726faa93c97488/scripts/incremental/benchmarking/utils.py -def _calculateRelCLOC(repo_path, commit, diff_exclude): - diff_exclude = list(map(lambda x: os.path.join(repo_path, x), diff_exclude)) - relcloc = 0 - for f in commit.modified_files: - _, extension = os.path.splitext(f.filename) - if not (extension == ".h" or extension == ".c"): - continue - filepath = f.new_path - if filepath is None: - filepath = f.old_path - parents = Path(filepath).parents - parents = list(map(lambda x: os.path.join(repo_path, x), parents)) - if any(dir in parents for dir in diff_exclude): - continue - relcloc = relcloc + f.added_lines + f.deleted_lines - return relcloc - -def iter_repo(): - global analyzed_commits - global total_commits - global count_merge - global count_nochanges - global count_big - global count_small - - for commit in Repository(url, since=begin, to=to, clone_repo_to=cwd).traverse_commits(): - total_commits += 1 - - # count merge commits - if commit.merge: - count_merge += 1 - continue - - # count commits that have less than maxCLOC of relevant code changes - relCLOC = _calculateRelCLOC(repo_path, commit, paths_to_exclude) # use this to filter commits by actually relevant changes - if relCLOC == 0: - count_nochanges += 1 - continue - - if maxCLOC is not None and relCLOC > maxCLOC: - count_big += 1 - continue - - count_small += 1 - -iter_repo() -print("\nCommits traversed in total: ", total_commits) -print("Merge commits: ", count_merge) -print("Commits without any relevant changes: ", count_nochanges) -print("Big commits: ", count_big) -print("Small commits with relevant changes: ", count_small) \ No newline at end of file diff --git a/scripts/incremental-test-generation/git-patches/git-patches.py b/scripts/incremental-test-generation/git-patches/git-patches.py deleted file mode 100644 index 31f6470eac..0000000000 --- a/scripts/incremental-test-generation/git-patches/git-patches.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import shutil -import subprocess -import tempfile - -script_location = os.path.dirname(__file__) -repoURL = 'https://github.com/cmatsuoka/figlet.git' -repoPath = os.path.join(script_location, '/testGit') -repoDifPath = os.path.join(script_location, '/testGitDif') -paths = [repoPath, repoDifPath] - -# Clear the directories -for path in paths: - if os.path.exists(path): - shutil.rmtree(path) - -# Clone the repository from repo URL into the folder ./testGit -print("Repo Path: " + repoPath) -subprocess.run(['git', 'clone', repoURL, repoPath]) - -# Change the working directory to the repository directory -os.chdir(repoPath) - -# Get a list of commit hashes using git log -output = subprocess.run(["git", "log", "--reverse", "--format=%H"], capture_output=True, text=True) -commit_hashes = output.stdout.strip().split('\n') - -# Create the directory ./testGitDif if it does not exist -os.makedirs(repoDifPath, exist_ok=True) - -# Create a diff file for each commit and store them in ./testGitDif -for i, commit_hash in enumerate(commit_hashes): - # Generate the diff using git diff - diff_output = subprocess.run(['git', 'diff', commit_hash + '^', commit_hash], capture_output=True, text=True) - diff = diff_output.stdout.strip() - - # Write the diff to a file in the repoDifPath - diff_path = os.path.join(repoDifPath, '{:06d}_{}.diff'.format(i, commit_hash)) - with open(diff_path, 'w') as f: - f.write(diff) - - # Update the paths to account for the change in working directory - repoPath = os.path.abspath(repoPath) - repoDifPath = os.path.abspath(repoDifPath) - diff --git a/scripts/incremental-test-generation/openai-text-completion/.gitignore b/scripts/incremental-test-generation/openai-text-completion/.gitignore deleted file mode 100644 index a3ba8c66d2..0000000000 --- a/scripts/incremental-test-generation/openai-text-completion/.gitignore +++ /dev/null @@ -1 +0,0 @@ -APIKEY.yaml diff --git a/scripts/incremental-test-generation/openai-text-completion/ml.py b/scripts/incremental-test-generation/openai-text-completion/ml.py deleted file mode 100644 index a5156bb862..0000000000 --- a/scripts/incremental-test-generation/openai-text-completion/ml.py +++ /dev/null @@ -1,77 +0,0 @@ -import os -import yaml -import openai - -# [TODO] run "sudo pip install openai" -# -# [TODO] get api key and store it in a yaml file: -# organisation: ... -# api-key: ... -# -api_key_path = os.path.expanduser("~/BA/Goblint-Repo/analyzer/scripts/incremental-test-generation/openai-text-completion/APIKEY.yaml") - -# Read the api key and organisation -with open(api_key_path, 'r') as file: - data = yaml.safe_load(file) -organisation = data.get('organisation') -api_key = data.get('api-key') - -# Authenticate -openai.organization = organisation -openai.api_key = api_key - -# Select Model -model = "text-davinci-edit-001" - -input = r''' -#include - -// Function to calculate the sum of two numbers -int sum(int a, int b) { - return a + b; -} - -// Function to calculate the difference of two numbers -int difference(int a, int b) { - return a - b; -} - -// Function to calculate the product of two numbers -int product(int a, int b) { - return a * b; -} - -// Function to calculate the quotient of two numbers -float divide(int a, int b) { - if (b != 0) { - return (float)a / b; - } else { - printf("Error: Division by zero is not allowed.\n"); - return 0; - } -} - -int main() { - int num1, num2; - - printf("Enter two numbers: "); - scanf("%d %d", &num1, &num2); - - printf("Sum: %d\n", sum(num1, num2)); - printf("Difference: %d\n", difference(num1, num2)); - printf("Product: %d\n", product(num1, num2)); - printf("Quotient: %.2f\n", divide(num1, num2)); - - return 0; -} -''' - -completion = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - n = 3, - messages=[ - {"role": "user", "content": "Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code, introduce bugs or do any other modifications that might happen in a development workflow. But the code should still compile. Explain shortly what you have changed. Here the C code: " + input}, - ] -) - -print(completion.choices[0].message) \ No newline at end of file diff --git a/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt b/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt deleted file mode 100644 index 0eef825b54..0000000000 --- a/scripts/incremental-test-generation/openai-text-completion/outputs/20230522-01.txt +++ /dev/null @@ -1,233 +0,0 @@ -Promt: Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code and introduce bugs. But the code should still compile. Here the C code: -############################################################################################### -#include -#include - -#define NUM_THREADS 5 - -void *threadFunction(void *arg) { - int threadID = *(int *)arg; - printf("Thread %d is running.\n", threadID); - - // Perform some task in the thread - - printf("Thread %d completed.\n", threadID); - pthread_exit(NULL); -} - -int main() { - pthread_t threads[NUM_THREADS]; - int threadArgs[NUM_THREADS]; - int i, result; - - // Create threads - for (i = 0; i < NUM_THREADS; i++) { - threadArgs[i] = i; - result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); - if (result != 0) { - printf("Error creating thread %d. Exiting program.\n", i); - return -1; - } - } - - // Wait for threads to finish - for (i = 0; i < NUM_THREADS; i++) { - result = pthread_join(threads[i], NULL); - if (result != 0) { - printf("Error joining thread %d. Exiting program.\n", i); - return -1; - } - } - - printf("All threads have completed. Exiting program.\n"); - - return 0; -} - -############################################################################################################ - -#include - -void *threadFunction(void *arg) { - int threadID = *(int *)arg; - printf("Thread %d is running.\n", threadID); - printf("Thread %d completed.\n", threadID); - pthread_exit(NULL); -} - -int main() { - pthread_t threads[NUM_THREADS]; - int threadArgs[NUM_THREADS]; - int i, result; - - for (i = 0; i < NUM_THREADS; i++) { - threadArgs[i] = i; - result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); - } - - for (i = 0; i < NUM_THREADS; i++) { - result = pthread_join(threads[i], NULL); - } - - printf("Exiting program.\n"); - - return 0; -} - -############################################################################################################ -#include - -#define NUM_THREADS 5 - -void *threadFunction(void *arg) { - int threadID = *(int *)arg; - printf("Thread %d is running.\n", threadID); - // Perform some task in the thread - printf("Thread %d completed.\n", threadID); - return NULL; -} - -int main() { - int threads[NUM_THREADS]; - int threadArgs[NUM_THREADS]; - int i, result; - - // Create threads - for (i = 0; i < NUM_THREADS; i++) { - threadArgs[i] = i; - result = threads[i]; // Introduce bug - if (result != 0) { - printf("Error creating thread %d. Exiting program.\n", i); - return -1; - } - } - - // Wait for threads to finish - for (i = 0; i < NUM_THREADS; i++) { - result = threads[i]; // Introduce bug - if (result != 0) { - printf("Error joining thread %d. Exiting program.\n", i); - return -1; - } - } - - printf("All threads have completed. Exiting program.\n"); - - return 0; -} - -############################################################################################################ -#include - -int sum(int a, int b) { - return a + b; -} - -int product(int a, int b) { - return a * b; -} - -float divide(int a, int b) { - if (b != 0) { - return (float) a / b; - } else { - printf("Error: Division by zero is not allowed.\n"); - return 0; - } -} - -int main() { - int num1, num2; - - printf("Enter two numbers: "); - scanf("%d", &num1); - scanf("%d", &num2); - - printf("Sum: %d\n", sum(num1, num2)); - printf("Product: %d\n", product(num1, num2)); - printf("Quotient: %.2f\n", divide(num1, num2)); - - return 0; -} - -############################################################################################################ -#include - -int sum(int a, int b) { - return a + b; -} - -int product(int a, int b) { - return a * b; -} - -float divide(int a, int b) { - if (b != 0) { - return (float) a / b; - } else { - printf("Error: Division by zero is not allowed.\n"); - return 0; - } -} - -int main() { - int num1, num2; - - printf("Enter two numbers: "); - scanf("%d", &num1); - scanf("%d", &num2); - - printf("Sum: %d\n", sum(num1, num2)); - printf("Product: %d\n", product(num1, num2)); - printf("Quotient: %.2f\n", divide(num1, num2)); - - return 0; -} - -############################################################################################################ -#include - -// Function to calculate the sum of two numbers -int sum(int a, int b) { - return a + b; // function remains unchanged -} - -// Function to calculate the difference of two numbers -int difference(int a, int b) { - return a - b; // function remains unchanged -} - -// Function to calculate the product of two numbers -int product(int a, int b) { - return a; // introducing bug by returning only the first number -} - -// Function to calculate the quotient of two numbers -float divide(int a, int b) { - if (b != 0) { - return a / b; // introducing bug by not casting to float - } else { - printf("Error: Division by zero is not allowed.\n"); - return a; // introducing bug by returning the first number instead of 0 - } -} - -int main() { - int num1, num2; - - printf("Enter two numbers: "); - scanf("%d %d", &num1, &num2); - - printf("Sum: %d\n", sum(num1, num2)); - printf("Difference: %d\n", difference(num1, num2)); - printf("Product: %d\n", product(num1, num2)); - printf("Quotient: %.2f\n", divide(num1, num2)); - - return 0; -} - -############################################################################################################ -############################################################################################################ -############################################################################################################ -############################################################################################################ -############################################################################################################ \ No newline at end of file diff --git a/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt b/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt deleted file mode 100644 index cf4b576f82..0000000000 --- a/scripts/incremental-test-generation/openai-text-completion/outputs/20230523-01.txt +++ /dev/null @@ -1,25 +0,0 @@ -Promt: Please modify the following C code to a state as it might have been in the earlier implementation. You are allowed to remove code, introduce bugs or do any other modifications that might happen in a development workflow. But the code should still compile. Explain shortly what you have changed. Here the C code: -################################################################################## -#include - -int main() { - int num1, num2; - - // User input - printf("Enter two numbers: "); - scanf("%d %d", &num1, &num2); - - // Calculation and output - printf("Sum: %d\n", num1 + num2); - printf("Difference: %d\n", num1 - num2); - printf("Product: %d\n", num1 * num2); - - if (num2 != 0) { - printf("Quotient: %.2f\n", (float)num1 / num2); - } else { - printf("Error: Division by zero is not allowed.\n"); - } - - return 0; -} -################################################################################## From 7f63d28d939d79baae1068e3ae78b5d65ddd47b4 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 15:00:53 +0200 Subject: [PATCH 084/131] Fix and document files --- .../test-generation/generators/README.md | 66 +++++++++++++++++++ .../generators/generate_git.py | 7 +- .../generators/generate_git_build.sh | 59 ++++++++--------- .../test-generation/generators/generate_ml.py | 4 +- 4 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 scripts/incremental-test-generation/test-generation/generators/README.md diff --git a/scripts/incremental-test-generation/test-generation/generators/README.md b/scripts/incremental-test-generation/test-generation/generators/README.md new file mode 100644 index 0000000000..92f0030960 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/generators/README.md @@ -0,0 +1,66 @@ +# Mutation Generators +This folder contains Python scripts that generate mutated programs. The different files and their purpose is shortly described in the following: +You can use for all python scripts the option `-h` to get additional information about the comand line arguments. +

+You can use this content for the `meta.yaml` file you need to pass: +``` +n: 0 +``` + +# generate_mutations.py +This script uses the custom clang-tidy checks to generate mutations on a given program. Informations about the mutations are written to the `meta.yaml` file. +

+More information about the mutations can be found in this file: [Mutations](../../clang-mutations/MUTATIONS.md) + +# generate_ml.py +This script uses gpt-3.5-turbo with the api from openai. It generates mutations on a program by asking it how a previous version of the code could have looked like before some typical code changes were done by developers. Informations about the mutations are written to the `meta.yaml` file. +

+You need to pass a `apikey.yaml` file with the following format: +``` +organisation: (Found at https://platform.openai.com/account/org-settings) +api-key: (Found at https://platform.openai.com/account/api-keys) +``` +You can specify `num_selected_lines` to tell the script how many consecutive lines should be send together with the prompt. To many lines could lead to an error because of a too large request. To few lines may result in a bad mutation because of less context. +

+You can specify `interesting_lines` to guide the selection of the lines send with the request. The selection process works by selecting a random start line out of a set of lines. From the start line on the `num_selected_lines` are selected. When `interesting_lines` equals to `[]` all the lines (`[1, 2, 3, ..., MAX_LINE - NUM_SELECTED_LINES]`) are intresting lines. Alternatively you can pass specific lines (`[1, 42, 99]`). Note that when a line is larger then `(MAX_LINE - NUM_SELECTED_LINES)` it will be set to this value. + +# generate_git.py +This script generates from a git repository per commit a cil file representing the whole project in a single c file. The differences between the commits are used as mutations. For passing a repository you need to specify a shell script containing all the needed information and optionally you can give commands that should be executed before and after the default build commands (`cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON` / `bear -- make`). +

+You can find the template for the shell script with the informations in the file `generate_git_build_USER_INFO_TEMPLATE.sh`: +``` +#!/bin/bash + +############################################################## +####################### USER VARIABLES ####################### +############################################################## +# Variables +git_url="" #e.g.: https://github.com/madler/zlib.git +use_cmake=true # Choose either cmake or make +use_make=false # Choose either cmake or make +path_to_build="" #e.g.: "." (Relative path in repo) + +# Functions before and after build +pre_build_commands() { + : #e.g.: ./configure +} + +post_build_commands() { + : +} +############################################################## +####################### USER VARIABLES ####################### +############################################################## + +# Export variables so they can be used in the main script +export git_url +export use_cmake +export use_make +export path_to_build +export pre_build_commands +export post_build_commands +``` + +The script `generate_git_build.sh` interacts with these user shell scripts and can clone the repositories, build the repositories and provide the build path. +

+For not analyzing all the commits you can specify a start and end commit hash. \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py index c26dc8f4bc..9208f2176e 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git.py @@ -64,7 +64,6 @@ def get_commit_traverser(): print(f"[GIT][{t}/{num_of_commits}] {COLOR_YELLOW}Skipping merge commit {commit.hash}{COLOR_RESET}, continuing with the next commit...") last_failed = True continue - #TODO Skip small commits and summarize them in one big commit try: _checkout(build_path, meta_path, commit.hash) _build_repo(git_info_sh_path, temp_repo_dir, meta_path, commit.hash) @@ -186,9 +185,9 @@ def _create_cil_file(goblint_path, build_path, output_path, meta_path, commit_ha parser.add_argument("temp_dir", help="Path to the temporary directory") parser.add_argument("meta_path", help="Path to the meta directory") parser.add_argument("git_info_sh_path", help="Path to the Git information shell script") - parser.add_argument("start_window_ms", help="Start of the time window in milliseconds", type=int) - parser.add_argument("end_window_ms", help="End of the time window in milliseconds", type=int) + parser.add_argument("--start_commit", help="Hash id of the first commit to consider", default=None) + parser.add_argument("--end_commit", help="Hash id of the last commit to consider", default=None) args = parser.parse_args() - generate_git(args.goblint_path, args.temp_dir, args.meta_path, args.git_info_sh_path, args.start_window_ms, args.end_window_ms) + generate_git(args.goblint_path, args.temp_dir, args.meta_path, args.git_info_sh_path, args.start_commit, args.end_commit) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh index 45ab92f875..0f5df72a9e 100755 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh @@ -20,11 +20,6 @@ else exit 1 fi -if [ "$3" == "" ]; then - echo "Please provide one of the options [--clone][--build][--path]" - exit 1 -fi - # Exit if any command fails set -e @@ -37,28 +32,32 @@ fi # Get the name of the repo repo_name=$(basename "$git_url" .git) -# Export path for Mutation Generator -if [ "$3" == "--path" ]; then - echo "$output_path/$repo_name/$path_to_build" - exit 0 -fi - -# Clone repo -if [ "$3" == "--clone" ]; then - rm -rf "$output_path/$repo_name" - git clone $git_url "$output_path/$repo_name" -fi - -# Build repo -if [ "$3" == "--build" ]; then - cd "$output_path/$repo_name/$path_to_build" - pre_build_commands - - if $use_cmake; then - cmake "$output_path/$repo_name/$path_to_build" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - elif $use_make; then - bear -- make - fi - - post_build_commands -fi +case "$3" in + "--path") + # Export path for Mutation Generator + echo "$output_path/$repo_name/$path_to_build" + exit 0 + ;; + "--clone") + # Clone repo + rm -rf "$output_path/$repo_name" + git clone $git_url "$output_path/$repo_name" + ;; + "--build") + # Build repo + cd "$output_path/$repo_name/$path_to_build" + pre_build_commands + + if $use_cmake; then + cmake "$output_path/$repo_name/$path_to_build" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + elif $use_make; then + bear -- make + fi + + post_build_commands + ;; + *) + echo "Please provide one of the options [--clone][--build][--path]" + exit 1 + ;; +esac \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index 55fe4a9b92..035704c880 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -60,7 +60,7 @@ def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lin def _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index, lock): try: - time.sleep((index * 50)/1000) + time.sleep((index * 50)/1000) # Sleep depending on index to print the start messages in the right order new_path = make_program_copy(program_path, index) (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index) _write_meta_data(meta_path, selected_lines, explanation, index, lock) @@ -123,8 +123,8 @@ def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, i def _write_meta_data(meta_path, selected_lines, explanation, index, lock, exception=None): - global error_counter lock.acquire() + global error_counter try: with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) From fe961f7894dec8f54580ca677e27889627e1c60b Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 15:49:47 +0200 Subject: [PATCH 085/131] Fixes and Readme files --- .../test-generation/README.md | 2 ++ .../test-generation/RUN_CLI.py | 4 ++-- .../test-generation/util/README.md | 14 ++++++++++++++ .../test-generation/util/add_check.py | 2 +- .../test-generation/util/add_check_comments.py | 2 +- .../test-generation/util/generate_programs.py | 6 +++--- .../test-generation/util/run_tests.py | 2 ++ 7 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 scripts/incremental-test-generation/test-generation/README.md create mode 100644 scripts/incremental-test-generation/test-generation/util/README.md diff --git a/scripts/incremental-test-generation/test-generation/README.md b/scripts/incremental-test-generation/test-generation/README.md new file mode 100644 index 0000000000..705d2b2058 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/README.md @@ -0,0 +1,2 @@ +# Mutation Generator +Run `python3 RUN_CLI.py` to start the Comand Line Interface. With `-h` you can see the command line options for skipping the interactive user input. \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 9ca35bcfec..117136d3e9 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -6,7 +6,7 @@ import yaml from pathlib import Path from util.util import * -from util.generate_programs import gernerate_programs +from util.generate_programs import generate_programs from util.generate_tests import generate_tests from util.run_tests import run_tests from generators.generate_mutations import add_mutation_options, get_mutations_from_args @@ -39,7 +39,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - gernerate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count) + generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count) # Run tests if is_run_tests: diff --git a/scripts/incremental-test-generation/test-generation/util/README.md b/scripts/incremental-test-generation/test-generation/util/README.md new file mode 100644 index 0000000000..be92ae3f16 --- /dev/null +++ b/scripts/incremental-test-generation/test-generation/util/README.md @@ -0,0 +1,14 @@ +# Util +This directory contains all the utility files required to generate incremental tests from the generated mutated files. Each Python script is briefly described below, and they are ordered according to their typical usage. Please use the `meta.yaml` files from the previous script calls. + +# generate_programs.py +This script generates an empty `meta.yaml` file and, depending on the passed options, calls the different [generators](../generators/README.md) to create mutated files in the `temp_dir` (`p_42.c`). It then adds the `___goblint_check()` function along with corresponding comments by calling `add_check.py` and `add_check_comments.py`. + +# add_check.py +This script calls Goblint to add the `___goblint_check()` function to the C files in the `temp_dir` (`p_42_check.c`). + +# add_check_comments.py +This script adds the corresponding `//UNKNOWN //SUCCESS` or `//SUCCESS` comments to all the Goblint checks (`___goblint_check()`) in the `temp_dir` (`p_42_check_unknown.c` / `p_42_check_success.c`). + +# generate_tests.py +This script generates the test files for each mutation. A test consists of a source `.c` file, a `.patch` file containing the changes from the mutation, and a `.json` file with the Goblint options. Normally, a patch is the difference between the original program provided by the user (`p_0_check_success.c`) and the mutated program (`p_42_check_unknown.c`). For precision tests, the mutated program is only annotated with success Goblint checks (`p_42_check_success.c`). For mutations generated by a Git project, the patch is formed from the previous file to the current file. diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index 856fd7b03b..ee5a5a1f83 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -1,6 +1,6 @@ # Takes a file and generates the goblint checks # Stores the file with an additional "_c" -# When there is a compilation error the process writes [COMPILE_FAIL] into the meta data file for the given index +# When there is a compilation error the process writes [META_EXCEPTION] into the meta data file for the given index import argparse import subprocess diff --git a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py index c0d3ae1449..275573c356 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check_comments.py @@ -1,6 +1,6 @@ # Adds "//UNDEFINED" or "//SUCCESS" to the Goblint checks "__goblint_check(exp);". # Stores the file with the same name when adding "//SUCCESS". -# Stores the file with the appendix _u when adding "//UNDEFINED". +# Stores the file with the appendix _u when adding "//UNKNOWN". # Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´. import argparse diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 3b778529a4..e63925d621 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -15,7 +15,7 @@ generate_type_source = "SOURCE" -def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count): +def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -77,7 +77,7 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if failed_count == 0: print(f"{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") else: - print(f"{COLOR_RED}There where {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") + print(f"{COLOR_RED}There were {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') @@ -113,4 +113,4 @@ def gernerate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, api if args.enable_ml and not args.apikey_path: parser.error("--enable-ml requires --apikey-path") - gernerate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git) + generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git) diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index 9920395bf4..3dc66f8fdc 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -18,6 +18,7 @@ def run_tests(test_dir, goblint_repo_dir, cfg): shutil.copytree(test_dir, incremental_tests_dir_abs) ruby_path_abs = os.path.abspath(os.path.join(goblint_repo_dir, "scripts", "update_suite.rb")) + original_dir = os.getcwd() os.chdir(goblint_repo_dir) command = f"{ruby_path_abs} group temp -i" if cfg: @@ -28,6 +29,7 @@ def run_tests(test_dir, goblint_repo_dir, cfg): process.wait() shutil.rmtree(incremental_tests_dir_abs) + os.chdir(original_dir) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') From d8450829b12e9d3650c0da29a23f0298281c4256 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 16:12:44 +0200 Subject: [PATCH 086/131] Do not overwrite direcotroy --- .../test-generation/util/run_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index 3dc66f8fdc..8f8113c683 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -8,9 +8,13 @@ def run_tests(test_dir, goblint_repo_dir, cfg): # Check the name of the test_dir + if os.path.exists(test_dir_name): + print(f'{COLOR_RED}The test directory 99-temp already exists. Please remove it manually to run the tests.{COLOR_RESET}') + sys.exit(-1) test_dir_name = os.path.basename(test_dir) if test_dir_name != "99-temp": print(f"{COLOR_RED}[ERROR] The test directory name has to be \'99-temp\'{COLOR_RESET}") + sys.exit(-1) incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) if os.path.exists(incremental_tests_dir_abs): From 304d289018305ccb609399612b39cdfcb9621ac1 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 12 Jun 2023 16:13:08 +0200 Subject: [PATCH 087/131] Add cfg option --- .../test-generation/RUN_CLI.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 117136d3e9..383fe36455 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -29,11 +29,11 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests, api_key_path, ml_count): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests, api_key_path, ml_count, cfg): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) - input_path = os.path.abspath(os.path.expanduser(input_path)) #TODO Handle git url + input_path = os.path.abspath(os.path.expanduser(input_path)) # Generate the programs goblint_executable_path = os.path.join(goblint_path, 'goblint') @@ -48,15 +48,15 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio print(SEPERATOR) print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, precision_test=True) - run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg + run_tests(test_path, goblint_path, cfg) print(SEPERATOR) print(f'Writing out {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, precision_test=False) - run_tests(test_path, goblint_path, cfg=True) #TODO Add Option for cfg + run_tests(test_path, goblint_path, cfg) if os.path.exists(test_path): shutil.rmtree(test_path) - #TODO Print link to html result and give summary + #TODO Copy html result and print the link #Write out custom test files print(SEPERATOR) @@ -71,7 +71,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input, ml_count): +def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input, ml_count, cfg): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -180,6 +180,9 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, if running == None: running = questionary.confirm('Run the tests?').ask() + if cfg == None: + cfg = questionary.confirm('Run the fine grained cfg tests?').ask() + if input == None: while True: if enable_mutations or enable_ml: @@ -195,7 +198,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running, key_path, ml_count) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running, key_path, ml_count, cfg) if __name__ == "__main__": @@ -208,8 +211,10 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, parser.add_argument('-g', '--enable-git', action='store_true', help='Enable Git') parser.add_argument('-ep', '--enable-precision', action='store_true', help='Enable Precision Tests') parser.add_argument('-dp', '--disable-precision', action='store_true', help='Disable Precision Tests') - parser.add_argument('-er', '--enable-running', action='store_true', help='Enable Running Tests') - parser.add_argument('-dr', '--disable-running', action='store_true', help='Disable Running Tests') + parser.add_argument('-er', '--enable-running', action='store_true', help='Enable running tests') + parser.add_argument('-dr', '--disable-running', action='store_true', help='Disable running tests') + parser.add_argument('-ec', '--enable-cfg', action='store_true', help='Enable fine grained cfg tests') + parser.add_argument('-dc', '--disable-cfg', action='store_true', help='Disable fine grained cfg tests') parser.add_argument('-i', '--input', help='Input File') # Add mutation options @@ -265,10 +270,18 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, else: running = None + if args.enable_cfg or args.disable_cfg: + # Only one can be selected + if args.enable_cfg and args.disable_cfg: + parser.error('Cfg can not be enabled AND diabled') + cfg = args.enable_cfg + else: + cfg = None + if args.ml_count > 0: ml_count = args.ml_count else: ml_count = None - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input, ml_count) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input, ml_count, cfg) \ No newline at end of file From 8db02d295b067ac93b89750b963e869e039374d5 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 15 Jun 2023 16:28:13 +0200 Subject: [PATCH 088/131] Pass option for dir names and test creation --- .../test-generation/RUN_CLI.py | 72 ++++++++++++++----- .../test-generation/util/generate_tests.py | 6 +- .../test-generation/util/run_tests.py | 6 +- .../test-generation/util/util.py | 12 ++++ 4 files changed, 70 insertions(+), 26 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 383fe36455..5f4077983f 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -29,7 +29,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, create_precision, is_run_tests, api_key_path, ml_count, cfg): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, cfg): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -44,7 +44,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio # Run tests if is_run_tests: test_path = os.path.abspath(os.path.join(os.path.curdir, '99-temp')) - if create_precision: + if enable_precision: print(SEPERATOR) print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, precision_test=True) @@ -59,19 +59,20 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio #TODO Copy html result and print the link #Write out custom test files - print(SEPERATOR) - correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '99-test') - print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST{COLOR_RESET} files:') - generate_tests(temp_path, correctness_path, precision_test=False) #TODO Custom name - print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! - if create_precision: + if create_tests: print(SEPERATOR) - precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "98-precision") - print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST{COLOR_RESET} files:') - generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name - print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! + correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), test_name) + print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') + generate_tests(temp_path, correctness_path, precision_test=False) #TODO Custom name + print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! + if enable_precision: + print(SEPERATOR) + precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), precision_name) + print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') + generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name + print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, input, ml_count, cfg): +def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, cfg): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -174,13 +175,28 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, continue break - if precision == None: - precision = questionary.confirm('Create precision test files?', default=False).ask() + if create_tests == None: + create_tests = questionary.confirm('Create test files?', default=False).ask() + + if create_tests and test_name == None: + while True: + test_name = questionary.text('Enter the test name: ', default="99-test").ask() + if check_test_name(test_name): + break + + if enable_precision == None: + enable_precision = questionary.confirm('Create precision test files?', default=False).ask() + + if create_tests and enable_precision and precision_name == None: + while True: + precision_name = questionary.text('Enter the precision test name: ', default="98-precision").ask() + if check_test_name(precision_name): + break if running == None: running = questionary.confirm('Run the tests?').ask() - if cfg == None: + if create_tests and cfg == None: cfg = questionary.confirm('Run the fine grained cfg tests?').ask() if input == None: @@ -198,7 +214,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, precision, running, key_path, ml_count, cfg) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, cfg) if __name__ == "__main__": @@ -213,8 +229,12 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, parser.add_argument('-dp', '--disable-precision', action='store_true', help='Disable Precision Tests') parser.add_argument('-er', '--enable-running', action='store_true', help='Enable running tests') parser.add_argument('-dr', '--disable-running', action='store_true', help='Disable running tests') + parser.add_argument('-et', '--enable-create-tests', action='store_true', help='Enable creating test files') + parser.add_argument('-dt', '--disable-create-tests', action='store_true', help='Disable creating test files') parser.add_argument('-ec', '--enable-cfg', action='store_true', help='Enable fine grained cfg tests') parser.add_argument('-dc', '--disable-cfg', action='store_true', help='Disable fine grained cfg tests') + parser.add_argument('-p', '--test-name', help='Test name') + parser.add_argument('-t', '--precision-name', help='Precision test name') parser.add_argument('-i', '--input', help='Input File') # Add mutation options @@ -270,6 +290,14 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, else: running = None + if args.enable_create_tests or args.disable_create_tests: + # Only one can be selected + if args.enable_create_tests and args.disable_create_tests: + parser.error('Create tests can not be enabled AND diabled') + create_tests = args.enable_create_tests + else: + create_tests = None + if args.enable_cfg or args.disable_cfg: # Only one can be selected if args.enable_cfg and args.disable_cfg: @@ -282,6 +310,14 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, precision, running, ml_count = args.ml_count else: ml_count = None + + test_name = args.test_name + if test_name != None and not check_test_name(test_name): + sys.exit(-1) + + precision_name = args.precision_name + if precision_name != None and not check_test_name(precision_name): + sys.exit(-1) - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, precision, running, args.input, ml_count, cfg) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, cfg) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index a01af85fb5..1f8d19d02c 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -12,10 +12,8 @@ def generate_tests(temp_dir, target_dir, precision_test): # Check the name of the target_dir directoryName = os.path.basename(target_dir) - pattern = r"\d{2}-\w+" - if not re.match(pattern, directoryName): - print(f"{COLOR_RED}[ERROR] Target Directory name is not of the format 01-Name (\d{{2}}-\w+){COLOR_RESET}") - return + if not check_test_name(directoryName): + sys.exit(-1) # Clear and create target_dir if os.path.exists(target_dir): diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index 8f8113c683..7f07223c5e 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -8,9 +8,6 @@ def run_tests(test_dir, goblint_repo_dir, cfg): # Check the name of the test_dir - if os.path.exists(test_dir_name): - print(f'{COLOR_RED}The test directory 99-temp already exists. Please remove it manually to run the tests.{COLOR_RESET}') - sys.exit(-1) test_dir_name = os.path.basename(test_dir) if test_dir_name != "99-temp": print(f"{COLOR_RED}[ERROR] The test directory name has to be \'99-temp\'{COLOR_RESET}") @@ -18,7 +15,8 @@ def run_tests(test_dir, goblint_repo_dir, cfg): incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) if os.path.exists(incremental_tests_dir_abs): - shutil.rmtree(incremental_tests_dir_abs) + print(f'{COLOR_RED}The test directory {incremental_tests_dir_abs} already exists. Please remove it manually to run the tests.{COLOR_RESET}') + sys.exit(-1) shutil.copytree(test_dir, incremental_tests_dir_abs) ruby_path_abs = os.path.abspath(os.path.join(goblint_repo_dir, "scripts", "update_suite.rb")) diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index c89a789a59..efa6b51804 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -1,4 +1,5 @@ from enum import Enum +import re import shutil class Mutations: @@ -59,3 +60,14 @@ def make_program_copy(program_path, index): new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' shutil.copy2(program_path, new_path) return new_path + +def check_test_name(directoryName): + if directoryName is None or not isinstance(directoryName, str): + print(f"{COLOR_RED}[ERROR] Target Directory name {directoryName} is not a string{COLOR_RESET}") + return False + + pattern = r"\d{2}-\w+" + if not re.match(pattern, directoryName): + print(f"{COLOR_RED}[ERROR] Target Directory name {directoryName} is not of the format 01-Name (\d{{2}}-\w+){COLOR_RESET}") + return False + return True \ No newline at end of file From 83144bfba00c2d1745363d851ecc05916941456a Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 15 Jun 2023 17:08:25 +0200 Subject: [PATCH 089/131] Pass git start and end hash --- .../test-generation/RUN_CLI.py | 23 +++++++++++++++---- .../generators/generate_git.py | 4 ++-- .../test-generation/util/generate_programs.py | 12 ++++------ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 5f4077983f..681ec6cd0d 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -29,7 +29,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, cfg): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, cfg, git_start, git_end): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -39,7 +39,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count) + generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, git_start, git_end) # Run tests if is_run_tests: @@ -72,7 +72,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, cfg): +def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, cfg, git_start, git_end, git_no_commit): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -175,6 +175,11 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te continue break + if enable_git and not (git_start != None and git_end != None) and not git_no_commit: + if questionary.confirm('Do you want to give a start and end commit hash?', default=False).ask(): + git_start = questionary.text('Enter start commit hash:').ask() + git_end = questionary.text('Enter end commit hash:').ask() + if create_tests == None: create_tests = questionary.confirm('Create test files?', default=False).ask() @@ -214,7 +219,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, cfg) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, cfg, git_start, git_end) if __name__ == "__main__": @@ -245,6 +250,9 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te # Add GIT options parser.add_argument('-s', '--template-script', action='store_true', help='Print the template script for git repositories') + parser.add_argument('-gs', '--git-start-commit', help='The hash of the first commit to consider') + parser.add_argument('-ge', '--git-end-commit', help='The hash of the last commit to consider') + parser.add_argument('-gn', '--git-no-commit', action='store_true', help='Suppress asking for commit hashes in CLI') args = parser.parse_args() @@ -274,6 +282,11 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te args.enable_git = None mutations = None + git_start_commit = args.git_start_commit + git_end_commit = args.git_end_commit + if (git_start_commit == None and git_end_commit != None) or (git_start_commit != None and git_end_commit == None): + parser.error('[ERROR] Give a git start commit hash AND a end commit hash') + if args.enable_precision or args.disable_precision: # Only one can be selected if args.enable_precision and args.disable_precision: @@ -320,4 +333,4 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te sys.exit(-1) - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, cfg) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py index 9208f2176e..5dcbc9e8b1 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git.py @@ -25,6 +25,7 @@ def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_comm print('[GIT] Cloning into temp/repo') _clone_repo(git_info_sh_path, temp_repo_dir) build_path = _get_build_path(git_info_sh_path, temp_repo_dir) + print(f'{COLOR_GREEN}[GIT] Cloning finished{COLOR_RESET}') if not os.path.exists(temp_repo_dir): os.mkdir(temp_repo_dir) @@ -51,11 +52,11 @@ def get_commit_traverser(): index: int = yaml_data[META_N] num_of_commits = sum(1 for _ in get_commit_traverser()) + print(SEPERATOR) print(f'[GIT] Start traversing {num_of_commits} commits. Including checkout, build and cil generation. This may take a while...') if num_of_commits <= 2: print(f'{COLOR_RED}You must traverse at least two commits to generate a test!') sys.exit(-1) - print(SEPERATOR) t = 0 #last_failed = False for commit in get_commit_traverser(): @@ -79,7 +80,6 @@ def get_commit_traverser(): except Exception as e: #last_failed = True print(f"{COLOR_RED}[{t}/{num_of_commits}][FAIL] Generating cil for commit {commit.hash} failed ({e}){COLOR_RESET}, continuing with the next commit...") - print(SEPERATOR) print(f"{COLOR_GREEN}[FINISHED] Finished creating cil files for the commits.{COLOR_RESET}") if build_errors > 0 or checkout_errors > 0 or cil_errors > 0: print(f"{COLOR_RED}There were the following errors: {build_errors} build errors, {checkout_errors} checkout errors and {cil_errors} cil errors.{COLOR_RESET}") diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index e63925d621..cf0a60f36f 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -15,7 +15,7 @@ generate_type_source = "SOURCE" -def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count): +def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, git_start, git_end): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -42,16 +42,14 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) if enable_git: - #TODO Let user select start - START_COMMIT = 'da6f1623c177c5ebfa2b1ee3b50eb297da5a77e1' - #TODO Let user select end - END_COMMIT = '04f42ceca40f73e2978b50e93806c2a18c1281fc' - index = generate_git(goblint_path, temp_dir, meta_path, program_path, START_COMMIT, END_COMMIT) + index = generate_git(goblint_path, temp_dir, meta_path, program_path, git_start, git_end) # Add checks with comments print(SEPERATOR) + if generate_git: + print('Generating goblint checks. This may take a while...') for i in range(index + 1): - if i % 10 == 0: + if i % 10 == 0 or generate_git: print(f"[{i}/{index}] Generating goblint checks...") file_path = os.path.join(temp_dir, f"p_{i}.c") compiling = add_check(file_path, i, goblint_path, meta_path) From df9ad70bb7c4ca7a05244ae3621e0a773c8ef574 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 15 Jun 2023 17:46:48 +0200 Subject: [PATCH 090/131] Add options for ml --- .../test-generation/RUN_CLI.py | 45 +++++++++++++++---- .../test-generation/generators/generate_ml.py | 24 ++++++---- .../test-generation/util/add_check.py | 3 +- .../test-generation/util/generate_programs.py | 8 +--- .../test-generation/util/util.py | 1 + 5 files changed, 57 insertions(+), 24 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 681ec6cd0d..0aaceed019 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -10,6 +10,7 @@ from util.generate_tests import generate_tests from util.run_tests import run_tests from generators.generate_mutations import add_mutation_options, get_mutations_from_args +from generators.generate_ml import validate_interesting_lines logo = ''' __ __ _ _ _ @@ -29,7 +30,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, cfg, git_start, git_end): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -39,7 +40,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, git_start, git_end) + generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, ml_select, ml_interesting, git_start, git_end) # Run tests if is_run_tests: @@ -56,8 +57,6 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio if os.path.exists(test_path): shutil.rmtree(test_path) - #TODO Copy html result and print the link - #Write out custom test files if create_tests: print(SEPERATOR) @@ -72,7 +71,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, cfg, git_start, git_end, git_no_commit): +def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, cfg, git_start, git_end, git_no_commit): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -175,6 +174,26 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te continue break + if enable_ml and ml_select == None: + while True: + ml_select = questionary.text('How many lines should be selected for the snippet from the input file?', default=str(DEFAULT_ML_SELECT)).ask() + if not ml_select.strip('\n').isdigit(): + print(f"{COLOR_RED}Please enter a valid number.{COLOR_RESET}") + continue + ml_select = int(ml_select.strip('\n')) + if ml_select <= 0: + print(f"{COLOR_RED}Please enter a number greater zero.{COLOR_RESET}") + continue + break + + if enable_ml and ml_interesting == None: + while True: + ml_interesting = questionary.text('From which start lines should the snippet start be choosen randomly ([] stands for all)?', default='[]').ask() + if validate_interesting_lines(ml_interesting, None) == None: + print(f'{COLOR_RED}Please enter a valid string like [1, 42], [99], ....{COLOR_RESET}') + continue + break + if enable_git and not (git_start != None and git_end != None) and not git_no_commit: if questionary.confirm('Do you want to give a start and end commit hash?', default=False).ask(): git_start = questionary.text('Enter start commit hash:').ask() @@ -219,7 +238,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, cfg, git_start, git_end) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end) if __name__ == "__main__": @@ -246,7 +265,9 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te add_mutation_options(parser) # Add ML options - parser.add_argument('-c', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') + parser.add_argument('-mc', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') + parser.add_argument('-ms', '--ml-select', type=int, default=-1, help='How many lines should be selected for the snippet from the input file?') + parser.add_argument('-mi', '--ml-interesting', help='From which start lines should the snippet start be choosen randomly? Exp. :[] = From all lines, [1, 42], ...') # Add GIT options parser.add_argument('-s', '--template-script', action='store_true', help='Print the template script for git repositories') @@ -324,6 +345,14 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te else: ml_count = None + if args.ml_select > 0: + ml_select = args.ml_select + else: + ml_select = None + + if args.ml_interesting != None and validate_interesting_lines(args.ml_interesting, None) == None: + sys.exit(-1) + test_name = args.test_name if test_name != None and not check_test_name(test_name): sys.exit(-1) @@ -333,4 +362,4 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te sys.exit(-1) - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index 035704c880..fda3f99a53 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -42,6 +42,7 @@ def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lin max_line = len(file.readlines()) if max_line < num_selected_lines: num_selected_lines = max_line + interesting_lines = validate_interesting_lines(interesting_lines, program_path) interesting_lines = _reformat_interesting_lines(num_selected_lines, interesting_lines, max_line) print(SEPERATOR) @@ -174,6 +175,7 @@ def _make_gpt_request(snippet): def _reformat_interesting_lines(num_selected_lines, interesting_lines, max_line): for i in range(len(interesting_lines)): + interesting_lines[i] = int(interesting_lines[i]) # Adjust for line array starting with 0 but in input first line is 1 interesting_lines[i] -= 1 # When line + num_selected_lines is greater then max_line correct the line downwards @@ -190,8 +192,11 @@ def _select_lines(interesting_lines, num_selected_lines, max_line): #TODO Call when giving interesting lines def validate_interesting_lines(intersting_lines_string: str, program_path): - with open(program_path, "r") as file: - max_line = len(file.readlines()) + if program_path != None: + with open(program_path, "r") as file: + max_line = len(file.readlines()) + else: + max_line = None try: intersting_lines = ast.literal_eval(intersting_lines_string) @@ -202,13 +207,14 @@ def validate_interesting_lines(intersting_lines_string: str, program_path): print(f"{COLOR_RED}An unexpected error occurred reading the input \"{intersting_lines_string}\":{COLOR_RESET} {e}") return None - for line in intersting_lines: - if line <= 0: - print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be greater zero!{COLOR_RESET}") - return None - if line > max_line: - print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!{COLOR_RESET}") - return None + if max_line != None: + for line in intersting_lines: + if line <= 0: + print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be greater zero!{COLOR_RESET}") + return None + if line > max_line: + print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!{COLOR_RESET}") + return None return intersting_lines diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/test-generation/util/add_check.py index ee5a5a1f83..b1ebc16099 100644 --- a/scripts/incremental-test-generation/test-generation/util/add_check.py +++ b/scripts/incremental-test-generation/test-generation/util/add_check.py @@ -42,7 +42,8 @@ def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): yaml_data = yaml.safe_load(file) yaml_data[f"p_{index}"] = { META_TYPE: Generate_Type.ML.value, - META_EXCEPTION: result.stderr + META_EXCEPTION: result.stderr, + META_COMPILING: False } with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index cf0a60f36f..c38d472327 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -15,7 +15,7 @@ generate_type_source = "SOURCE" -def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, git_start, git_end): +def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, ml_select, ml_interesting, git_start, git_end): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -35,11 +35,7 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik index = generate_mutations(program_path, clang_tidy_path, meta_path, mutations) if enable_ml: - #TODO Allow user to specify how many lines to select - NUM_SELECTED_LINES = 50 - #TODO Allow user to specify which part of the program is intresting - INTRESTING_LINES = [] - index = generate_ml(program_path, apikey_path, meta_path, ml_count, NUM_SELECTED_LINES, INTRESTING_LINES) + index = generate_ml(program_path, apikey_path, meta_path, ml_count, ml_select, ml_interesting) if enable_git: index = generate_git(goblint_path, temp_dir, meta_path, program_path, git_start, git_end) diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index efa6b51804..a566cf4886 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -54,6 +54,7 @@ class Generate_Type(Enum): APIKEY_ORGANISATION = 'organisation' DEFAULT_ML_COUNT = 5 +DEFAULT_ML_SELECT = 50 ML_WORKERS = 5 def make_program_copy(program_path, index): From 16c8b9ed59e46326607ab2e06f3e4791f3e3ee76 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 11:27:24 +0200 Subject: [PATCH 091/131] Add passing of goblint config file --- .../test-generation/RUN_CLI.py | 32 ++++++++++++------- .../test-generation/util/generate_tests.py | 18 +++++++---- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 0aaceed019..85d3cef9c7 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -30,7 +30,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -48,11 +48,11 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio if enable_precision: print(SEPERATOR) print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') - generate_tests(temp_path, test_path, precision_test=True) + generate_tests(temp_path, test_path, goblint_config, precision_test=True) run_tests(test_path, goblint_path, cfg) print(SEPERATOR) print(f'Writing out {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET} files for running:') - generate_tests(temp_path, test_path, precision_test=False) + generate_tests(temp_path, test_path, goblint_config, precision_test=False) run_tests(test_path, goblint_path, cfg) if os.path.exists(test_path): shutil.rmtree(test_path) @@ -62,16 +62,16 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio print(SEPERATOR) correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), test_name) print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') - generate_tests(temp_path, correctness_path, precision_test=False) #TODO Custom name + generate_tests(temp_path, correctness_path, goblint_config, precision_test=False) #TODO Custom name print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! if enable_precision: print(SEPERATOR) precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), precision_name) print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') - generate_tests(temp_path, precision_path, precision_test=False) #TODO Custom name + generate_tests(temp_path, precision_path, goblint_config, precision_test=False) #TODO Custom name print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, cfg, git_start, git_end, git_no_commit): +def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, cfg, git_start, git_end, git_no_commit): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -162,6 +162,13 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te else: key_path = None + # Check for config file + if goblint_config == None: + goblint_config = questionary.text('Path to a goblint config file used to create tests. Passing {} creates an empty config file.', default='{}').ask() + if goblint_config == '{}' or goblint_config == '': + goblint_config = None + + # ML Options if enable_ml and ml_count == None: while True: ml_count = questionary.text('How many different programs should be generated with ML?', default=str(DEFAULT_ML_COUNT)).ask() @@ -194,11 +201,13 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te continue break + # Git options if enable_git and not (git_start != None and git_end != None) and not git_no_commit: if questionary.confirm('Do you want to give a start and end commit hash?', default=False).ask(): git_start = questionary.text('Enter start commit hash:').ask() git_end = questionary.text('Enter end commit hash:').ask() + # Output options if create_tests == None: create_tests = questionary.confirm('Create test files?', default=False).ask() @@ -209,7 +218,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te break if enable_precision == None: - enable_precision = questionary.confirm('Create precision test files?', default=False).ask() + enable_precision = questionary.confirm('Enable precision tests?', default=False).ask() if create_tests and enable_precision and precision_name == None: while True: @@ -220,9 +229,10 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te if running == None: running = questionary.confirm('Run the tests?').ask() - if create_tests and cfg == None: + if running and cfg == None: cfg = questionary.confirm('Run the fine grained cfg tests?').ask() + # input options if input == None: while True: if enable_mutations or enable_ml: @@ -238,7 +248,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end) if __name__ == "__main__": @@ -249,6 +259,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te parser.add_argument('-m', '--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') parser.add_argument('-o', '--enable-ml', action='store_true', help='Enable ML') parser.add_argument('-g', '--enable-git', action='store_true', help='Enable Git') + parser.add_argument('-c', '--goblint-config', help='Path to a goblint config file used to create tests (passing "{}" as argument creates an empty config file)') parser.add_argument('-ep', '--enable-precision', action='store_true', help='Enable Precision Tests') parser.add_argument('-dp', '--disable-precision', action='store_true', help='Disable Precision Tests') parser.add_argument('-er', '--enable-running', action='store_true', help='Enable running tests') @@ -361,5 +372,4 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, test_name, create_te if precision_name != None and not check_test_name(precision_name): sys.exit(-1) - - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, args.goblint_config, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index 1f8d19d02c..02bdaaa9ca 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -9,7 +9,7 @@ sys.path.insert(0, "..") from util.util import * -def generate_tests(temp_dir, target_dir, precision_test): +def generate_tests(temp_dir, target_dir, goblint_config, precision_test): # Check the name of the target_dir directoryName = os.path.basename(target_dir) if not check_test_name(directoryName): @@ -93,11 +93,14 @@ def generate_tests(temp_dir, target_dir, precision_test): yaml_data[current_program_id][META_DIFF] = True else: raise Exception("Command failed with return code: {}".format(result.returncode)) - # Create a empty config file - #TODO Support other config files - data = {} - with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: - json.dump(data, f) + if goblint_config == None: + # Create a empty config file + data = {} + with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: + json.dump(data, f) + else: + # Copy config file + shutil.copy2(os.path.abspath(os.path.expanduser(goblint_config)), os.path.join(target_dir, test_name + '.json')) print(f"{COLOR_GREEN}Generating test files [DONE].{COLOR_RESET}") if unchanged_count > 0: print(f'{COLOR_YELLOW}There were {unchanged_count} patch files with no changes.{COLOR_RESET}') @@ -124,7 +127,8 @@ def _format_number(n): parser.add_argument('temp_dir', help='Path to the working directory') parser.add_argument('target_dir', help='Path to the target directory') parser.add_argument('-p', '--precision-test', action='store_true', help='Generate tests for precision') + parser.add_argument('-c', '--goblint-config', help='Optional path to the goblint config file used for the tests (using no option creates an empty one)') args = parser.parse_args() - generate_tests(args.temp_dir, args.target_dir, args.precision_test) + generate_tests(args.temp_dir, args.target_dir, args.goblint_config, args.precision_test) From 3e866068bc07c595171e5996506d82f83a790f39 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 14:14:54 +0200 Subject: [PATCH 092/131] Interpret //PARAM from input file --- .../test-generation/RUN_CLI.py | 4 +-- .../test-generation/util/run_tests.py | 29 ++++++++++++++++--- scripts/update_suite.rb | 23 +++++++++------ 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 85d3cef9c7..63d055eee4 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -49,11 +49,11 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio print(SEPERATOR) print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, goblint_config, precision_test=True) - run_tests(test_path, goblint_path, cfg) + run_tests(input_path, test_path, goblint_path, cfg) print(SEPERATOR) print(f'Writing out {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET} files for running:') generate_tests(temp_path, test_path, goblint_config, precision_test=False) - run_tests(test_path, goblint_path, cfg) + run_tests(input_path, test_path, goblint_path, cfg) if os.path.exists(test_path): shutil.rmtree(test_path) diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index 7f07223c5e..fbb4d8db07 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -2,11 +2,14 @@ import os import shutil import subprocess +import re import sys + +import questionary sys.path.insert(0, "..") from util.util import * -def run_tests(test_dir, goblint_repo_dir, cfg): +def run_tests(program_path, test_dir, goblint_repo_dir, cfg): # Check the name of the test_dir test_dir_name = os.path.basename(test_dir) if test_dir_name != "99-temp": @@ -15,14 +18,20 @@ def run_tests(test_dir, goblint_repo_dir, cfg): incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) if os.path.exists(incremental_tests_dir_abs): - print(f'{COLOR_RED}The test directory {incremental_tests_dir_abs} already exists. Please remove it manually to run the tests.{COLOR_RESET}') - sys.exit(-1) + print(f'{COLOR_RED}The test directory {incremental_tests_dir_abs} already exists.{COLOR_RESET}') + if questionary.confirm('Replace the directory?', default=False).ask(): + shutil.rmtree(incremental_tests_dir_abs) + else: + sys.exit(-1) shutil.copytree(test_dir, incremental_tests_dir_abs) ruby_path_abs = os.path.abspath(os.path.join(goblint_repo_dir, "scripts", "update_suite.rb")) + params = get_params_from_file(program_path) + if params != "": + print(f"\n{COLOR_YELLOW}Using parameters from input file:{COLOR_RESET} {params}") original_dir = os.getcwd() os.chdir(goblint_repo_dir) - command = f"{ruby_path_abs} group temp -i" + command = f"{ruby_path_abs} group temp -p \"{params}\" -i" if cfg: command += " -c" process = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True, shell=True) @@ -33,6 +42,18 @@ def run_tests(test_dir, goblint_repo_dir, cfg): shutil.rmtree(incremental_tests_dir_abs) os.chdir(original_dir) +def get_params_from_file(filename): + param_pattern = re.compile(r"\s*//.*PARAM\s*:\s*(.*)") + + with open(filename, 'r') as f: + for line in f: + match = param_pattern.match(line) + if match: + params = match.group(1).strip() + return params + + return "" + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') parser.add_argument('test_dir', help='Path to the directory with the tests') diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 5e65bb8c6c..01e609fab4 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -66,6 +66,10 @@ def clearline cfg = ARGV.last == "-c" && ARGV.pop incremental = (ARGV.last == "-i" && ARGV.pop) || cfg report = ARGV.last == "-r" && ARGV.pop +p_index = ARGV.index("-p") # Comand line parameters for Goblint (pass before other otions) +user_params = p_index.nil? || p_index == ARGV.length - 1 ? "" : ARGV[p_index + 1] +ARGV.delete_at(p_index) unless p_index.nil? +ARGV.delete_at(p_index) unless p_index.nil? || p_index >= ARGV.length only = ARGV[0] unless ARGV[0].nil? if marshal || witness || incremental then sequential = true @@ -134,7 +138,7 @@ def initialize(project, tests, tests_line, todo) def report filename = File.basename(p.path) system($highlighter.call(filename, orgfile)) - `#{$goblint} #{filename} --set justcil true #{p.params} >#{cilfile} 2> /dev/null` + `#{$goblint} #{filename} --set justcil true #{p.params} #{p.user_params} >#{cilfile} 2> /dev/null` p.size = `wc -l #{cilfile}`.split[0] end @@ -262,7 +266,7 @@ def to_html class Project attr_reader :id, :name, :group, :path, :params, :testset, :html_heading - attr_accessor :size, :testset + attr_accessor :size, :testset, :user_params def initialize(id, name, group, path, params) @id = id @name = name @@ -370,7 +374,7 @@ def run_testset (testset, cmd, starttime) def run filename = File.basename(@path) - cmd = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set goblint-dir .goblint-#{@id.sub('/','-')} 2>#{@testset.statsfile}" + cmd = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set goblint-dir .goblint-#{@id.sub('/','-')} 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd, starttime) end @@ -436,8 +440,8 @@ def create_test_set(lines) def run filename = File.basename(@path) - cmd = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --enable incremental.save --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-save 2>#{@testset.statsfile}" - cmd_incr = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset_incr.warnfile} --enable dbg.timing.enabled --enable incremental.load --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-load 2>#{@testset_incr.statsfile}" + cmd = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --enable incremental.save --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-save 2>#{@testset.statsfile}" + cmd_incr = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset_incr.warnfile} --enable dbg.timing.enabled --enable incremental.load --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-load 2>#{@testset_incr.statsfile}" starttime = Time.now run_testset(@testset_incr, cmd, starttime) # apply patch @@ -480,8 +484,8 @@ def create_test_set(lines) end def run () filename = File.basename(@path) - cmd1 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set save_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-save 2>#{@testset.statsfile}" - cmd2 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --conf run/config.json --set save_run '' --set load_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-load 2>#{@testset.statsfile}" + cmd1 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set save_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-save 2>#{@testset.statsfile}" + cmd2 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --conf run/config.json --set save_run '' --set load_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-load 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd1, starttime) run_testset(@testset, cmd2, starttime) @@ -496,8 +500,8 @@ def create_test_set(lines) end def run () filename = File.basename(@path) - cmd1 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile}0 --enable dbg.debug --set dbg.timing.enabled true --enable witness.yaml.enabled --set goblint-dir .goblint-#{@id.sub('/','-')}-witness1 2>#{@testset.statsfile}0" - cmd2 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --set ana.activated[+] unassume --enable dbg.debug --set dbg.timing.enabled true --set witness.yaml.unassume witness.yml --set goblint-dir .goblint-#{@id.sub('/','-')}-witness2 2>#{@testset.statsfile}" + cmd1 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile}0 --enable dbg.debug --set dbg.timing.enabled true --enable witness.yaml.enabled --set goblint-dir .goblint-#{@id.sub('/','-')}-witness1 2>#{@testset.statsfile}0" + cmd2 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --set ana.activated[+] unassume --enable dbg.debug --set dbg.timing.enabled true --set witness.yaml.unassume witness.yml --set goblint-dir .goblint-#{@id.sub('/','-')}-witness2 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd1, starttime) starttime = Time.now @@ -558,6 +562,7 @@ def run () else Project.new(id, testname, groupname, path, params) end + p.user_params = user_params p.create_test_set(lines) projects << p From 9ff18ed65b54b21188bfa8ca397c1e3f9efea124 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 14:43:09 +0200 Subject: [PATCH 093/131] Add option for gpt 16k model vs 4k --- .../test-generation/RUN_CLI.py | 27 ++++++++++++++----- .../test-generation/generators/generate_ml.py | 22 ++++++++------- .../test-generation/util/generate_programs.py | 4 +-- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 63d055eee4..2e5bfb2e11 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -30,7 +30,7 @@ ''' -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end): +def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end): # Make paths absolute goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) @@ -40,7 +40,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio goblint_executable_path = os.path.join(goblint_path, 'goblint') clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, ml_select, ml_interesting, git_start, git_end) + generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, ml_select, ml_interesting, ml_16k, git_start, git_end) # Run tests if is_run_tests: @@ -62,16 +62,16 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio print(SEPERATOR) correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), test_name) print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') - generate_tests(temp_path, correctness_path, goblint_config, precision_test=False) #TODO Custom name + generate_tests(temp_path, correctness_path, goblint_config, precision_test=False) print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! if enable_precision: print(SEPERATOR) precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), precision_name) print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') - generate_tests(temp_path, precision_path, goblint_config, precision_test=False) #TODO Custom name + generate_tests(temp_path, precision_path, goblint_config, precision_test=False) print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! -def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, cfg, git_start, git_end, git_no_commit): +def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end, git_no_commit): # Check config file config_path = Path(CONFIG_FILENAME) config = {} @@ -193,6 +193,9 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test continue break + if enable_ml and ml_16k == None: + ml_16k = questionary.confirm('Use the gpt-3.5-turbo-16k model instead of the gpt-3.5-turbo model', default=False).ask() + if enable_ml and ml_interesting == None: while True: ml_interesting = questionary.text('From which start lines should the snippet start be choosen randomly ([] stands for all)?', default='[]').ask() @@ -248,7 +251,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test yaml.dump(config, outfile) break - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, cfg, git_start, git_end) + run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end) if __name__ == "__main__": @@ -279,6 +282,8 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test parser.add_argument('-mc', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') parser.add_argument('-ms', '--ml-select', type=int, default=-1, help='How many lines should be selected for the snippet from the input file?') parser.add_argument('-mi', '--ml-interesting', help='From which start lines should the snippet start be choosen randomly? Exp. :[] = From all lines, [1, 42], ...') + parser.add_argument('-m4', '--ml-4k', action='store_true', help='Use the gpt-3.5-turbo model instead of the gpt-3.5-turbo-16k model') + parser.add_argument('-m16', '--ml-16k', action='store_true', help='Use the gpt-3.5-turbo-16k model instead of the gpt-3.5-turbo model') # Add GIT options parser.add_argument('-s', '--template-script', action='store_true', help='Print the template script for git repositories') @@ -364,6 +369,14 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test if args.ml_interesting != None and validate_interesting_lines(args.ml_interesting, None) == None: sys.exit(-1) + if args.ml_4k or args.ml_16k: + # Only one can be selected + if args.ml_4k and args.ml_16k: + parser.error('Only one ml model can be selected!') + ml_16k = args.ml_16k + else: + ml_16k = None + test_name = args.test_name if test_name != None and not check_test_name(test_name): sys.exit(-1) @@ -372,4 +385,4 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test if precision_name != None and not check_test_name(precision_name): sys.exit(-1) - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, args.goblint_config, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file + cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, args.goblint_config, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, ml_16k, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index fda3f99a53..9f7caf730e 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -19,7 +19,7 @@ error_counter = 0 -def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lines, interesting_lines): +def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lines, interesting_lines, ml_16k): with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) index: int = yaml_data[META_N] @@ -51,7 +51,7 @@ def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lin file_lock = Lock() with ThreadPoolExecutor(max_workers=ML_WORKERS) as executor: for i in range(ml_count): - executor.submit(_iterative_mutation_generation, program_path, meta_path, interesting_lines, num_selected_lines, max_line, index + i + 1, file_lock) + executor.submit(_iterative_mutation_generation, program_path, meta_path, interesting_lines, ml_16k, num_selected_lines, max_line, index + i + 1, file_lock) print(SEPERATOR) print('Check if all ML requests finsihed succesfully...') @@ -59,18 +59,18 @@ def generate_ml(program_path, apikey_path, meta_path, ml_count, num_selected_lin return index + ml_count -def _iterative_mutation_generation(program_path, meta_path, interesting_lines, num_selected_lines, max_line, index, lock): +def _iterative_mutation_generation(program_path, meta_path, interesting_lines, ml_16k, num_selected_lines, max_line, index, lock): try: time.sleep((index * 50)/1000) # Sleep depending on index to print the start messages in the right order new_path = make_program_copy(program_path, index) - (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index) + (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, ml_16k, num_selected_lines, max_line, index) _write_meta_data(meta_path, selected_lines, explanation, index, lock) except Exception as e: print(f"{COLOR_RED}[{index}] Error for request {index}:{COLOR_RESET} {e}") _write_meta_data(meta_path, [], '', index, lock, exception=e) return index -def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, index): +def _apply_mutation(new_path, interesting_lines, ml_16k, num_selected_lines, max_line, index): # Get the original lines with open(new_path, "r") as file: lines = file.readlines() @@ -84,7 +84,7 @@ def _apply_mutation(new_path, interesting_lines, num_selected_lines, max_line, i print(f"[{index}][{Generate_Type.ML.value}][REQUEST] Make request for lines [{selected_lines.start}, {selected_lines.stop}]. This may take a few seconds...") # Get response from gpt - response = _make_gpt_request(snippet) + response = _make_gpt_request(snippet, ml_16k) # Extract Explanation explanation_start = response.find(SEPERATOR_EXPLANATION_START) + len(SEPERATOR_EXPLANATION_START) @@ -146,7 +146,7 @@ def _write_meta_data(meta_path, selected_lines, explanation, index, lock, except finally: lock.release() -def _make_gpt_request(snippet): +def _make_gpt_request(snippet, ml_16k): prompt = f''' You are a developer for C helping me with my following question. I want to understand the typical process of code evolution by looking at how developers make changes over time for testing an incremental analysis of the static c analyzer Goblint. @@ -163,8 +163,13 @@ def _make_gpt_request(snippet): ``` ''' + if ml_16k: + model = "gpt-3.5-turbo-16k" + else: + model = "gpt-3.5-turbo" + response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", + model=model, n = 1, messages=[ {"role": "user", "content": prompt}, @@ -190,7 +195,6 @@ def _select_lines(interesting_lines, num_selected_lines, max_line): selected_line = random.choice(interesting_lines) return range(selected_line, selected_line + num_selected_lines) -#TODO Call when giving interesting lines def validate_interesting_lines(intersting_lines_string: str, program_path): if program_path != None: with open(program_path, "r") as file: diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index c38d472327..413cc2276f 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -15,7 +15,7 @@ generate_type_source = "SOURCE" -def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, ml_select, ml_interesting, git_start, git_end): +def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, ml_select, ml_interesting, ml_16k, git_start, git_end): # Clean working directory if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) @@ -35,7 +35,7 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik index = generate_mutations(program_path, clang_tidy_path, meta_path, mutations) if enable_ml: - index = generate_ml(program_path, apikey_path, meta_path, ml_count, ml_select, ml_interesting) + index = generate_ml(program_path, apikey_path, meta_path, ml_count, ml_select, ml_interesting, ml_16k) if enable_git: index = generate_git(goblint_path, temp_dir, meta_path, program_path, git_start, git_end) From 5671ed402d7622e30b5bb5faa6ffd3d565b1a298 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 15:04:00 +0200 Subject: [PATCH 094/131] Update CLI for individual files --- .../test-generation/generators/generate_ml.py | 3 ++- .../test-generation/util/generate_programs.py | 22 +++++++++++++++---- .../test-generation/util/run_tests.py | 3 ++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py index 9f7caf730e..f9d3a2db90 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_ml.py @@ -230,6 +230,7 @@ def validate_interesting_lines(intersting_lines_string: str, program_path): parser.add_argument("ml_count", help="How many different programs should be generated with ML") parser.add_argument("num_selected_lines", help="How many lines to consider") parser.add_argument("interesting_lines", help="Which parts are interesting (All: [], Specify: \"[1, 42, 99]\")") + parser.add_argument('-m16', '--model-16k', action='store_true', help='Run with the 16k model instead of the 4k') args = parser.parse_args() @@ -238,4 +239,4 @@ def validate_interesting_lines(intersting_lines_string: str, program_path): print(f'{COLOR_RED}Stopped program execution{COLOR_RESET}') sys.exit(-1) - generate_ml(args.program, args.apikey, args.meta_path, int(args.ml_count), int(args.num_selected_lines), interesting_lines) \ No newline at end of file + generate_ml(args.program, args.apikey, args.meta_path, int(args.ml_count), int(args.num_selected_lines), interesting_lines, args.model_16k) \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 413cc2276f..50418e4748 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -10,9 +10,6 @@ from generators.generate_ml import * from generators.generate_git import * -# Run for example with: -# python3 generate_programs.py ../../sample-files/threads.c test ~/BA/Clang-Repo/llvm-project/build/bin/clang-tidy ~/BA/Goblint-Repo/analyzer/goblint --enable-mutations - generate_type_source = "SOURCE" def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, ml_select, ml_interesting, ml_16k, git_start, git_end): @@ -83,6 +80,12 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik parser.add_argument('--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') parser.add_argument('--enable-ml', action='store_true', help='Enable ML') parser.add_argument('--enable-git', action='store_true', help='Enable Git') + parser.add_argument('--ml-count', type=int, default=DEFAULT_ML_COUNT, help='Number of ML programs to generate') + parser.add_argument('--ml-select', type=int, default=DEFAULT_ML_SELECT, help='Number of selected lines for ML') + parser.add_argument('--ml-interesting', default="[]", help='Lines to randomly choose the start line for selection (Defaul are all lines)') + parser.add_argument('--ml-16k', action='store_true', help='Use the 16k mode for ml') + parser.add_argument('--git-start', help='The starting commit hash for git generation') + parser.add_argument('--git-end', help='The ending commit hash for git generation') # Add mutation options add_mutation_options(parser) @@ -107,4 +110,15 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik if args.enable_ml and not args.apikey_path: parser.error("--enable-ml requires --apikey-path") - generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git) + # Check ml intersting string + if args.ml_interesting != "[]" and validate_interesting_lines(args.ml_interesting, None) == None: + sys.exit(-1) + + # Check git commit hashes + git_start_commit = args.git_start + git_end_commit = args.git_end + if (git_start_commit == None and git_end_commit != None) or (git_start_commit != None and git_end_commit == None): + parser.error('[ERROR] Give a git start commit hash AND a end commit hash') + +generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git, args.ml_count, args.ml_select, args.ml_interesting, args.ml_16k, args.git_start, args.git_end) + diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index fbb4d8db07..31f9f33385 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -56,10 +56,11 @@ def get_params_from_file(filename): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') + parser.add_argument('program_path', help='Path to the input file of the user') parser.add_argument('test_dir', help='Path to the directory with the tests') parser.add_argument('goblint_repo_dir', help='Path to the Goblint repository') parser.add_argument('-c', '--cfg', action='store_true', help='Run with fine-grained cfg-based change detection') args = parser.parse_args() - run_tests(args.test_dir, args.goblint_repo_dir, args.cfg) \ No newline at end of file + run_tests(args.program_path, args.test_dir, args.goblint_repo_dir, args.cfg) \ No newline at end of file From 8873ab2e8eab5535c5b0665cc99c0dd6c1706029 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 16:14:12 +0200 Subject: [PATCH 095/131] Fix --- .../test-generation/util/generate_programs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 50418e4748..f9b8935767 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -120,5 +120,5 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik if (git_start_commit == None and git_end_commit != None) or (git_start_commit != None and git_end_commit == None): parser.error('[ERROR] Give a git start commit hash AND a end commit hash') -generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git, args.ml_count, args.ml_select, args.ml_interesting, args.ml_16k, args.git_start, args.git_end) + generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git, args.ml_count, args.ml_select, args.ml_interesting, args.ml_16k, args.git_start, args.git_end) From feade93ba7e50956e09ef96788ab975f39036810 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 19:22:29 +0200 Subject: [PATCH 096/131] Allow multiple directories --- .../sample-files/large.c | 762 ++++++++++++++++++ .../test-generation/.gitignore | 5 +- .../test-generation/RUN_CLI.py | 44 +- .../generators/generate_mutations.py | 2 +- .../test-generation/util/generate_programs.py | 14 +- .../test-generation/util/generate_tests.py | 80 +- .../test-generation/util/run_tests.py | 40 +- .../test-generation/util/util.py | 1 + tests/incremental/.gitignore | 1 - 9 files changed, 901 insertions(+), 48 deletions(-) create mode 100644 scripts/incremental-test-generation/sample-files/large.c delete mode 100644 tests/incremental/.gitignore diff --git a/scripts/incremental-test-generation/sample-files/large.c b/scripts/incremental-test-generation/sample-files/large.c new file mode 100644 index 0000000000..b474747866 --- /dev/null +++ b/scripts/incremental-test-generation/sample-files/large.c @@ -0,0 +1,762 @@ +#include +#include +#include + +int main() { + int x; + srand(time(NULL)); + x = rand() % 250; // Generiere eine zufällige Zahl zwischen 0 und 249 + + printf("%d\n", x); + + if (x == 0) { + printf("x ist 0\n"); + } + if (x == 1) { + printf("x ist 1\n"); + } + if (x == 2) { + printf("x ist 2\n"); + } + if (x == 3) { + printf("x ist 3\n"); + } + if (x == 4) { + printf("x ist 4\n"); + } + if (x == 5) { + printf("x ist 5\n"); + } + if (x == 6) { + printf("x ist 6\n"); + } + if (x == 7) { + printf("x ist 7\n"); + }if (x == 8) { + printf("x ist 8\n"); + } + if (x == 9) { + printf("x ist 9\n"); + } + if (x == 10) { + printf("x ist 10\n"); + } + if (x == 11) { + printf("x ist 11\n"); + } + if (x == 12) { + printf("x ist 12\n"); + } + if (x == 13) { + printf("x ist 13\n"); + } + if (x == 14) { + printf("x ist 14\n"); + } + if (x == 15) { + printf("x ist 15\n"); + } + if (x == 16) { + printf("x ist 16\n"); + } + if (x == 17) { + printf("x ist 17\n"); + } + if (x == 18) { + printf("x ist 18\n"); + } + if (x == 19) { + printf("x ist 19\n"); + } + if (x == 20) { + printf("x ist 20\n"); + } + if (x == 21) { + printf("x ist 21\n"); + } + if (x == 22) { + printf("x ist 22\n"); + } + if (x == 23) { + printf("x ist 23\n"); + } + if (x == 24) { + printf("x ist 24\n"); + } + if (x == 25) { + printf("x ist 25\n"); + } + if (x == 26) { + printf("x ist 26\n"); + } + if (x == 27) { + printf("x ist 27\n"); + } + if (x == 28) { + printf("x ist 28\n"); + } + if (x == 29) { + printf("x ist 29\n"); + } + if (x == 30) { + printf("x ist 30\n"); + } + if (x == 31) { + printf("x ist 31\n"); + } + if (x == 32) { + printf("x ist 32\n"); + } + if (x == 33) { + printf("x ist 33\n"); + } + if (x == 34) { + printf("x ist 34\n"); + } + if (x == 35) { + printf("x ist 35\n"); + } + if (x == 36) { + printf("x ist 36\n"); + } + if (x == 37) { + printf("x ist 37\n"); + } + if (x == 38) { + printf("x ist 38\n"); + } + if (x == 39) { + printf("x ist 39\n"); + } + if (x == 40) { + printf("x ist 40\n"); + } + if (x == 41) { + printf("x ist 41\n"); + } + if (x == 42) { + printf("x ist 42\n"); + } + if (x == 43) { + printf("x ist 43\n"); + } + if (x == 44) { + printf("x ist 44\n"); + } + if (x == 45) { + printf("x ist 45\n"); + } + if (x == 46) { + printf("x ist 46\n"); + } + if (x == 47) { + printf("x ist 47\n"); + } + if (x == 48) { + printf("x ist 48\n"); + } + if (x == 49) { + printf("x ist 49\n"); + } + if (x == 50) { + printf("x ist 50\n"); + } + if (x == 51) { + printf("x ist 51\n"); + } + if (x == 52) { + printf("x ist 52\n"); + } + if (x == 53) { + printf("x ist 53\n"); + } + if (x == 54) { + printf("x ist 54\n"); + } + if (x == 55) { + printf("x ist 55\n"); + } + if (x == 56) { + printf("x ist 56\n"); + } + if (x == 57) { + printf("x ist 57\n"); + } + if (x == 58) { + printf("x ist 58\n"); + } + if (x == 59) { + printf("x ist 59\n"); + } + if (x == 60) { + printf("x ist 60\n"); + } + if (x == 61) { + printf("x ist 61\n"); + } + if (x == 62) { + printf("x ist 62\n"); + } + if (x == 63) { + printf("x ist 63\n"); + } + if (x == 64) { + printf("x ist 64\n"); + } + if (x == 65) { + printf("x ist 65\n"); + } + if (x == 66) { + printf("x ist 66\n"); + } + if (x == 67) { + printf("x ist 67\n"); + } + if (x == 68) { + printf("x ist 68\n"); + } + if (x == 69) { + printf("x ist 69\n"); + } + if (x == 70) { + printf("x ist 70\n"); + } + if (x == 71) { + printf("x ist 71\n"); + } + if (x == 72) { + printf("x ist 72\n"); + } + if (x == 73) { + printf("x ist 73\n"); + } + if (x == 74) { + printf("x ist 74\n"); + } + if (x == 75) { + printf("x ist 75\n"); + } + if (x == 76) { + printf("x ist 76\n"); + } + if (x == 77) { + printf("x ist 77\n"); + } + if (x == 78) { + printf("xist 78\n"); + } + if (x == 79) { + printf("x ist 79\n"); + } + if (x == 80) { + printf("x ist 80\n"); + } + if (x == 81) { + printf("x ist 81\n"); + } + if (x == 82) { + printf("x ist 82\n"); + } + if (x == 83) { + printf("x ist 83\n"); + } + if (x == 84) { + printf("x ist 84\n"); + } + if (x == 85) { + printf("x ist 85\n"); + } + if (x == 86) { + printf("x ist 86\n"); + } + if (x == 87) { + printf("x ist 87\n"); + } + if (x == 88) { + printf("x ist 88\n"); + } + if (x == 89) { + printf("x ist 89\n"); + } + if (x == 90) { + printf("x ist 90\n"); + } + if (x == 91) { + printf("x ist 91\n"); + } + if (x == 92) { + printf("x ist 92\n"); + } + if (x == 93) { + printf("x ist 93\n"); + } + if (x == 94) { + printf("x ist 94\n"); + } + if (x == 95) { + printf("x ist 95\n"); + } + if (x == 96) { + printf("x ist 96\n"); + } + if (x == 97) { + printf("x ist 97\n"); + } + if (x == 98) { + printf("x ist 98\n"); + } + if (x == 99) { + printf("x ist 99\n"); + } + if (x == 100) { + printf("x ist 100\n"); + } + if (x == 101) { + printf("x ist 101\n"); + } + if (x == 102) { + printf("x ist 102\n"); + } + if (x == 103) { + printf("x ist 103\n"); + } + if (x == 104) { + printf("x ist 104\n"); + } + if (x == 105) { + printf("x ist 105\n"); + } + if (x == 106) { + printf("x ist 106\n"); + } + if (x == 107) { + printf("x ist 107\n"); + } + if (x == 108) { + printf("x ist 108\n"); + } + if (x == 109) { + printf("x ist 109\n"); + } + if (x == 110) { + printf("x ist 110\n"); + } + if (x == 111) { + printf("x ist 111\n"); + } + if (x == 112) { + printf("x ist 112\n"); + } + if (x == 113) { + printf("x ist 113\n"); + } + if (x == 114) { + printf("x ist 114\n"); + } + if (x == 115) { + printf("x ist 115\n"); + } + if (x == 116) { + printf("x ist 116\n"); + } + if (x == 117) { + printf("x ist 117\n"); + } + if (x == 118) { + printf("x ist 118\n"); + } + if (x == 119) { + printf("x ist 119\n"); + } + if (x == 120) { + printf("x ist 120\n"); + } + if (x == 121) { + printf("x ist 121\n"); + } + if (x == 122) { + printf("x ist 122\n"); + } + if (x == 123) { + printf("x ist 123\n"); + } + if (x == 124) { + printf("x ist 124\n"); + } + if (x == 125) { + printf("x ist 125\n"); + } + if (x == 126) { + printf("x ist 126\n"); + } + if (x == 127) { + printf("x ist 127\n"); + } + if (x == 128) { + printf("x ist 128\n"); + } + if (x == 129) { + printf("x ist 129\n"); + } + if (x == 130) { + printf("x ist 130\n"); + } + if (x == 131) { + printf("x ist 131\n"); + } + if (x == 132) { + printf("x ist 132\n"); + } + if (x == 133) { + printf("x ist 133\n"); + } + if (x == 134) { + printf("x ist 134\n"); + } + if (x == 135) { + printf("x ist 135\n"); + } + if (x == 136) { + printf("x ist 136\n"); + } + if (x == 137) { + printf("x ist 137\n"); + } + if (x == 138) { + printf("x ist 138\n"); + } + if (x == 139) { + printf("x ist 139\n"); + } + if (x == 140) { + printf("x ist 140\n"); + } + if (x == 141) { + printf("x ist 141\n"); + } + if (x == 142) { + printf("x ist 142\n"); + } + if (x == 143) { + printf("x ist 143\n"); + } + if (x == 144) { + printf("x ist 144\n"); + } + if (x == 145) { + printf("x ist 145\n"); + } + if (x == 146) { + printf("x ist 146\n"); + } + if (x == 147) { + printf("x ist 147\n"); + } + if (x == 148) { + printf("x ist 148\n"); + } + if (x == 149) { + printf("x ist 149\n"); + } + if (x == 150) { + printf("x ist 150\n"); + } + if (x == 151) { + printf("x ist 151\n"); + } + if (x == 152) { + printf("x ist 152\n"); + } + if (x == 153) { + printf("x ist 153\n"); + } + if (x == 154) { + printf("x ist 154\n"); + } + if (x == 155) { + printf("x ist 155\n"); + } + if (x == 156) { + printf("x ist 156\n"); + } + if (x == 157) { + printf("x ist 157\n"); + } + if (x == 158) { + printf("x ist 158\n"); + } + if (x == 159) { + printf("x ist 159\n"); + } + if (x == 160) { + printf("x ist 160\n"); + } + if (x == 161) { + printf("x ist 161\n"); + } + if (x == 162) { + printf("x ist 162\n"); + } + if (x == 163) { + printf("x ist 163\n"); + } + if (x == 164) { + printf("x ist 164\n"); + } + if (x == 165) { + printf("x ist 165\n"); + } + if (x == 166) { + printf("x ist 166\n"); + } + if (x == 167) { + printf("x ist 167\n"); + } + if (x == 168) { + printf("x ist 168\n"); + } + if (x == 169) { + printf("x ist 169\n"); + } + if (x == 170) { + printf("x ist 170\n"); + } + if (x == 171) { + printf("x ist 171\n"); + } + if (x == 172) { + printf("x ist 172\n"); + } + if (x == 173) { + printf("x ist 173\n"); + } + if (x == 174) { + printf("x ist 174\n"); + } + if (x == 175) { + printf("x ist 175\n"); + } + if (x == 176) { + printf("x ist 176\n"); + } + if (x == 177) { + printf("x ist 177\n"); + } + if (x == 178) { + printf("x ist 178\n"); + } + if (x == 179) { + printf("x ist 179\n"); + } + if (x == 180) { + printf("x ist 180\n"); + } + if (x == 181) { + printf("x ist 181\n"); + } + if (x == 182) { + printf("x ist 182\n"); + } + if (x == 183) { + printf("x ist 183\n"); + } + if (x == 184) { + printf("x ist 184\n"); + } + if (x == 185) { + printf("x ist 185\n"); + } + if (x == 186) { + printf("x ist 186\n"); + } + if (x == 187) { + printf("x ist 187\n"); + } + if (x == 188) { + printf("x ist 188\n"); + } + if (x == 189) { + printf("x ist 189\n"); + } + if (x == 190) { + printf("x ist 190\n"); + } + if (x == 191) { + printf("x ist 191\n"); + } + if (x == 192) { + printf("x ist 192\n"); + } + if (x == 193) { + printf("x ist 193\n"); + } + if (x == 194) { + printf("x ist 194\n"); + } + if (x == 195) { + printf("x ist 195\n"); + } + if (x == 196) { + printf("x ist 196\n"); + } + if (x == 197) { + printf("x ist 197\n"); + } + if (x == 198) { + printf("x ist 198\n"); + } + if (x == 199) { + printf("x ist 199\n"); + } + if (x == 200) { + printf("x ist 200\n"); + } + if (x == 201) { + printf("x ist 201\n"); + } + if (x == 202) { + printf("x ist 202\n"); + } + if (x == 203) { + printf("x ist 203\n"); + } + if (x == 204) { + printf("x ist 204\n"); + } + if (x == 205) { + printf("x ist 205\n"); + } + if (x == 206) { + printf("x ist 206\n"); + } + if (x == 207) { + printf("x ist 207\n"); + } + if (x == 208) { + printf("x ist 208\n"); + } + if (x == 209) { + printf("x ist 209\n"); + } + if (x == 210) { + printf("x ist 210\n"); + } + if (x == 211) { + printf("x ist 211\n"); + } + if (x == 212) { + printf("x ist 212\n"); + } + if (x == 213) { + printf("x ist 213\n"); + } + if (x == 214) { + printf("x ist 214\n"); + } + if (x == 215) { + printf("x ist 215\n");} + if (x == 216) { + printf("x ist 216\n"); + } + if (x == 217) { + printf("x ist 217\n"); + } + if (x == 218) { + printf("x ist 218\n"); + } + if (x == 219) { + printf("x ist 219\n"); + } + if (x == 220) { + printf("x ist 220\n"); + } + if (x == 221) { + printf("x ist 221\n"); + } + if (x == 222) { + printf("x ist 222\n"); + } + if (x == 223) { + printf("x ist 223\n"); + } + if (x == 224) { + printf("x ist 224\n"); + } + if (x == 225) { + printf("x ist 225\n"); + } + if (x == 226) { + printf("x ist 226\n"); + } + if (x == 227) { + printf("x ist 227\n"); + } + if (x == 228) { + printf("x ist 228\n"); + } + if (x == 229) { + printf("x ist 229\n"); + } + if (x == 230) { + printf("x ist 230\n"); + } + if (x == 231) { + printf("x ist 231\n"); + } + if (x == 232) { + printf("x ist 232\n"); + } + if (x == 233) { + printf("x ist 233\n"); + } + if (x == 234) { + printf("x ist 234\n"); + } + if (x == 235) { + printf("x ist 235\n"); + } + if (x == 236) { + printf("x ist 236\n"); + } + if (x == 237) { + printf("x ist 237\n"); + } + if (x == 238) { + printf("x ist 238\n"); + } + if (x == 239) { + printf("x ist 239\n"); + } + if (x == 240) { + printf("x ist 240\n"); + } + if (x == 241) { + printf("x ist 241\n"); + } + if (x == 242) { + printf("x ist 242\n"); + } + if (x == 243) { + printf("x ist 243\n"); + } + if (x == 244) { + printf("x ist 244\n"); + } + if (x == 245) { + printf("x ist 245\n"); + } + if (x == 246) { + printf("x ist 246\n"); + } + if (x == 247) { + printf("x ist 247\n"); + } + if (x == 248) { + printf("x ist 248\n"); + } + if (x == 249) { + printf("x ist 249\n"); + } + + return 0; +} \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore index c0620d70e8..f1082b56eb 100644 --- a/scripts/incremental-test-generation/test-generation/.gitignore +++ b/scripts/incremental-test-generation/test-generation/.gitignore @@ -6,4 +6,7 @@ repo 98-* temp* config.yaml -api-key.yaml \ No newline at end of file +api-key.yaml +42-* +43-* +44-* \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 2e5bfb2e11..278d2f24eb 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -44,32 +44,42 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio # Run tests if is_run_tests: - test_path = os.path.abspath(os.path.join(os.path.curdir, '99-temp')) + test_path = os.path.abspath(os.path.join(os.path.curdir, 'temp/100-temp')) if enable_precision: print(SEPERATOR) - print(f'Writing out {COLOR_BLUE}PRECISION TEST{COLOR_RESET} files for running:') - generate_tests(temp_path, test_path, goblint_config, precision_test=True) - run_tests(input_path, test_path, goblint_path, cfg) + print(f'Running {COLOR_BLUE}PRECISION TEST{COLOR_RESET}:') + paths = generate_tests(temp_path, test_path, goblint_config, precision_test=True, temp_name=True) + if len(paths) > 1: + print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") + for path in paths: + run_tests(input_path, path, goblint_path, cfg) print(SEPERATOR) - print(f'Writing out {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET} files for running:') - generate_tests(temp_path, test_path, goblint_config, precision_test=False) - run_tests(input_path, test_path, goblint_path, cfg) - if os.path.exists(test_path): - shutil.rmtree(test_path) - - #Write out custom test files + print(f'Running {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET}:') + paths = generate_tests(temp_path, test_path, goblint_config, precision_test=False, temp_name=True) + if len(paths) > 1: + print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") + for path in paths: + run_tests(input_path, path, goblint_path, cfg) + + # Write out custom test files if create_tests: print(SEPERATOR) correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), test_name) print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') - generate_tests(temp_path, correctness_path, goblint_config, precision_test=False) - print(f'{COLOR_GREEN}Test stored in the directory: {correctness_path}{COLOR_RESET}') #TODO Multiple directories?! + paths = generate_tests(temp_path, correctness_path, goblint_config, precision_test=False, temp_name=False) + if len(paths) > 1: + print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") + for path in paths: + print(f'{COLOR_GREEN}Test stored in the file: {path}{COLOR_RESET}') if enable_precision: print(SEPERATOR) precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), precision_name) print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') - generate_tests(temp_path, precision_path, goblint_config, precision_test=False) - print(f'{COLOR_GREEN}Test stored in the directory: {precision_path}{COLOR_RESET}') #TODO Multiple directories?! + paths = generate_tests(temp_path, precision_path, goblint_config, precision_test=False, temp_name=False) + if len(paths) > 1: + print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") + for path in paths: + print(f'{COLOR_GREEN}Test stored in the file: {path}{COLOR_RESET}') def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end, git_no_commit): # Check config file @@ -271,8 +281,8 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test parser.add_argument('-dt', '--disable-create-tests', action='store_true', help='Disable creating test files') parser.add_argument('-ec', '--enable-cfg', action='store_true', help='Enable fine grained cfg tests') parser.add_argument('-dc', '--disable-cfg', action='store_true', help='Disable fine grained cfg tests') - parser.add_argument('-p', '--test-name', help='Test name') - parser.add_argument('-t', '--precision-name', help='Precision test name') + parser.add_argument('-t', '--test-name', help='Test name') + parser.add_argument('-p', '--precision-name', help='Precision test name') parser.add_argument('-i', '--input', help='Input File') # Add mutation options diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py index 8a4d9e0d98..fa6cfba434 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py @@ -91,7 +91,7 @@ def _get_line_groups(clang_tidy_path, mutation_name, program_path, index): line_groups = [list(x) for x in set(tuple(x) for x in line_groups)] print(f"[MUTATION][CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") - return line_groups + return sorted(line_groups, key=lambda x: x[0]) def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): lines_mapped = [[x,x] for x in lines] diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index f9b8935767..91e082a1ee 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -39,11 +39,11 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik # Add checks with comments print(SEPERATOR) - if generate_git: + if enable_git: print('Generating goblint checks. This may take a while...') for i in range(index + 1): - if i % 10 == 0 or generate_git: - print(f"[{i}/{index}] Generating goblint checks...") + print(f"\r[{i}/{index}] Generating goblint checks...", end='') + sys.stdout.flush() file_path = os.path.join(temp_dir, f"p_{i}.c") compiling = add_check(file_path, i, goblint_path, meta_path) if not compiling: @@ -52,11 +52,11 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik if i == 0 or enable_git: add_check_comments(file_path, unknown_instead_of_success=True) add_check_comments(file_path, unknown_instead_of_success=False) - print(f"{COLOR_GREEN}Generating goblint checks [DONE]{COLOR_RESET}") + print(f"\r{COLOR_GREEN}Generating goblint checks [DONE]{SPACE}{COLOR_RESET}") # Check how many and which files were not compiling print(SEPERATOR) - print("Check if the files compiled...") + print("Check if the files compiled...", end='') with open(meta_path, 'r') as file: yaml_data = yaml.safe_load(file) failed_count = 0 @@ -66,9 +66,9 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik failed_count += 1 failed_compilation_keys.append(key) if failed_count == 0: - print(f"{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") + print(f"\r{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") else: - print(f"{COLOR_RED}There were {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") + print(f"\r{COLOR_RED}There were {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/test-generation/util/generate_tests.py index 02bdaaa9ca..363bafcee4 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_tests.py @@ -4,20 +4,24 @@ import re import shutil import subprocess +import questionary import yaml import sys sys.path.insert(0, "..") from util.util import * -def generate_tests(temp_dir, target_dir, goblint_config, precision_test): +def generate_tests(temp_dir, target_dir, goblint_config, precision_test, temp_name): # Check the name of the target_dir - directoryName = os.path.basename(target_dir) - if not check_test_name(directoryName): + directory_name = os.path.basename(target_dir) + if not temp_name and not check_test_name(directory_name): sys.exit(-1) - # Clear and create target_dir if os.path.exists(target_dir): - shutil.rmtree(target_dir) + print(f'{COLOR_RED}The test directory {target_dir} already exists.{COLOR_RESET}') + if questionary.confirm('Replace the directory?', default=True).ask(): + shutil.rmtree(target_dir) + else: + sys.exit(-1) os.mkdir(target_dir) # Read the meta.yaml @@ -26,29 +30,62 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test): yaml_data = yaml.safe_load(file) n = yaml_data[META_N] + # loop threw all generated mutations + original_target_dir = target_dir + test_paths = [target_dir] + current_test_num = 1 # Skip num 0 to align program index with test number + current_dir_num = int(directory_name[:2]) unchanged_count = 0 compiling_programs = [] + if temp_name and int(directory_name[:3]) != 100: + print(f'{COLOR_RED}[ERROR] The directory number for temp files must be 100 but was {directory_name}{COLOR_RESET}') + sys.exit(-1) + elif temp_name: + current_dir_num = 100 + original_target_dir = os.path.join(os.path.dirname(target_dir), '99' + os.path.basename(target_dir)[3:]) for i in range(n + 1): + if current_test_num > 99: + current_dir_num += 1 + + # When temporary files let the files go over 99 to rename them later + if not temp_name and current_dir_num > 99: + print(f'{COLOR_RED}[ERROR] The directory number 100 is out of range. Consider starting with a lower than {directory_name} {COLOR_RESET}') + sys.exit(-1) + + group_name = re.match(r'\d+-(.*)', directory_name).group(1) + target_dir = os.path.join(os.path.dirname(target_dir), f'{current_dir_num:02}-{group_name}') + test_paths.append(target_dir) + + if os.path.exists(target_dir): + print(f'{COLOR_RED}The test directory {target_dir} already exists.{COLOR_RESET}') + if questionary.confirm('Replace the directory?', default=True).ask(): + shutil.rmtree(target_dir) + else: + sys.exit(-1) + os.mkdir(target_dir) + + current_test_num = 0 + current_program_id = f'p_{i}' compilation_success = yaml_data[current_program_id][META_COMPILING] if compilation_success: compiling_programs.append(i) else: - print(f"Generating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Not compiling){COLOR_RESET}") + print(f"\rGenerating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Not compiling){COLOR_RESET}") continue if META_EXCEPTION in yaml_data[current_program_id]: - print(f"Generating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Exception occured during generation){COLOR_RESET}") + print(f"\rGenerating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Exception occured during generation){COLOR_RESET}") continue type = yaml_data[current_program_id][META_TYPE] if type == Generate_Type.SOURCE.value or (type == Generate_Type.GIT.value and i == 0): continue if (i-1) % 9 == 0: - print(f"Generating test files [{i}/{n}]") + print(f"\rGenerating test files [{i}/{n}]", end='') sub_type = yaml_data[current_program_id][META_SUB_TYPE] if type == Generate_Type.MUTATION.value or type == Generate_Type.GIT.value: - test_name = _format_number(i) + '-' + type + '_' + sub_type + '_' + _format_number(i) + test_name = f'{_format_number(current_test_num)}-{type}_{sub_type}_{_format_number(i)}' else: - test_name = _format_number(i) + '-' + type + '_' + _format_number(i) + test_name = f'{_format_number(current_test_num)}-{type}_{_format_number(i)}' # Select depending on generator the start and end file of at test if type == Generate_Type.MUTATION.value or type == Generate_Type.ML.value: @@ -59,7 +96,7 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test): elif type ==Generate_Type.GIT.value: # If it's the first compiling program skip it. if i == compiling_programs[0]: - print(f"Generating test files [{i}/{n}] {COLOR_BLUE}Skipped {i} as the source file for the first test{COLOR_RESET}") + print(f"\rGenerating test files [{i}/{n}] {COLOR_BLUE}Skipped {i} as the source file for the first test{COLOR_RESET}") continue # Find the index of the previous compiling program. @@ -70,7 +107,7 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test): end_program = os.path.join(temp_dir, current_program_id + '_check_unknown.c') end_program_precison = os.path.join(temp_dir, current_program_id + '_check_success.c') else: - print(f'{COLOR_RED}[ERROR] Trying to generate tests from unknown generator type{COLOR_RESET}') + print(f'\n{COLOR_RED}[ERROR] Trying to generate tests from unknown generator type{COLOR_RESET}') sys.exit(-1) # Copy mutated code as the original code @@ -83,10 +120,14 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test): patch_path ) result = subprocess.run(command, shell=True) - _fix_patch_file(patch_path, directoryName, test_name + '.c') + if temp_name: + # For patch files keep the 99 for running inplace after renaming folder + _fix_patch_file(patch_path, os.path.basename(original_target_dir), test_name + '.c') + else: + _fix_patch_file(patch_path, os.path.basename(target_dir), test_name + '.c') if result.returncode in [0, 1]: if result.returncode == 0: - print(f"{COLOR_YELLOW}[WARNING] There were no changes in the patch for test {i}{COLOR_RESET}") + print(f"\r{COLOR_YELLOW}[WARNING] There were no changes in the patch for test {i}{COLOR_RESET}") unchanged_count += 1 yaml_data[current_program_id][META_DIFF] = False else: @@ -101,13 +142,19 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test): else: # Copy config file shutil.copy2(os.path.abspath(os.path.expanduser(goblint_config)), os.path.join(target_dir, test_name + '.json')) - print(f"{COLOR_GREEN}Generating test files [DONE].{COLOR_RESET}") + + current_test_num += 1 + + print(f"\r{COLOR_GREEN}Generating test files [DONE]{SPACE}{COLOR_RESET}") if unchanged_count > 0: print(f'{COLOR_YELLOW}There were {unchanged_count} patch files with no changes.{COLOR_RESET}') with open(meta_path, 'w') as file: yaml.safe_dump(yaml_data, file) + # Return the generated directories + return test_paths + def _fix_patch_file(patch_file, folder_name, file_name): with open(patch_file, 'r') as file: lines = file.readlines() @@ -128,7 +175,8 @@ def _format_number(n): parser.add_argument('target_dir', help='Path to the target directory') parser.add_argument('-p', '--precision-test', action='store_true', help='Generate tests for precision') parser.add_argument('-c', '--goblint-config', help='Optional path to the goblint config file used for the tests (using no option creates an empty one)') + parser.add_argument('-t', '--temp-name', action='store_true', help='Store name in special format for running the tests and removing them directly again') args = parser.parse_args() - generate_tests(args.temp_dir, args.target_dir, args.goblint_config, args.precision_test) + generate_tests(args.temp_dir, args.target_dir, args.goblint_config, args.precision_test, args.temp_name) diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/test-generation/util/run_tests.py index 31f9f33385..57a15d8f18 100644 --- a/scripts/incremental-test-generation/test-generation/util/run_tests.py +++ b/scripts/incremental-test-generation/test-generation/util/run_tests.py @@ -10,6 +10,20 @@ from util.util import * def run_tests(program_path, test_dir, goblint_repo_dir, cfg): + # When the directory has a starting number >99 rename it for in place running of the tests + match = re.match(r'(\d+)-(.*)', os.path.basename(test_dir)) + if match: + number, text = match.groups() + number = int(number) + if number > 99: + number = 99 + new_name = f'{number}-{text}' + os.rename(test_dir, os.path.join(os.path.dirname(test_dir), new_name)) + test_dir = os.path.join(os.path.dirname(test_dir), new_name) + else: + print(f"{COLOR_RED}[ERROR] The test directory had not the format number-text{COLOR_RESET}") + + # Check the name of the test_dir test_dir_name = os.path.basename(test_dir) if test_dir_name != "99-temp": @@ -19,7 +33,7 @@ def run_tests(program_path, test_dir, goblint_repo_dir, cfg): incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) if os.path.exists(incremental_tests_dir_abs): print(f'{COLOR_RED}The test directory {incremental_tests_dir_abs} already exists.{COLOR_RESET}') - if questionary.confirm('Replace the directory?', default=False).ask(): + if questionary.confirm('Replace the directory?', default=True).ask(): shutil.rmtree(incremental_tests_dir_abs) else: sys.exit(-1) @@ -34,12 +48,28 @@ def run_tests(program_path, test_dir, goblint_repo_dir, cfg): command = f"{ruby_path_abs} group temp -p \"{params}\" -i" if cfg: command += " -c" - process = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True, shell=True) - for line in process.stdout: - print(line, end='') + process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + + # Print the output including carriage returns + line = '' + while process.poll() is None: + char = process.stdout.read(1).decode('utf-8') + if char == '\r' or char == '\n': + sys.stdout.write('\r' + line) + sys.stdout.flush() + if char == '\n': + print() + line = '' + else: + line += char + if line: + sys.stdout.write('\r' + line + '\n') + sys.stdout.flush() + process.wait() shutil.rmtree(incremental_tests_dir_abs) + shutil.rmtree(test_dir) os.chdir(original_dir) def get_params_from_file(filename): @@ -57,7 +87,7 @@ def get_params_from_file(filename): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') parser.add_argument('program_path', help='Path to the input file of the user') - parser.add_argument('test_dir', help='Path to the directory with the tests') + parser.add_argument('test_dir', help='Path to the directory with the tests (WARNING Will be removed!)') parser.add_argument('goblint_repo_dir', help='Path to the Goblint repository') parser.add_argument('-c', '--cfg', action='store_true', help='Run with fine-grained cfg-based change detection') diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/test-generation/util/util.py index a566cf4886..9cc37cc406 100644 --- a/scripts/incremental-test-generation/test-generation/util/util.py +++ b/scripts/incremental-test-generation/test-generation/util/util.py @@ -30,6 +30,7 @@ class Generate_Type(Enum): COLOR_RESET = '\033[0m' SEPERATOR = f"{COLOR_BLUE}------------------------------------------------------------------------------------------------------------------{COLOR_RESET}" +SPACE = ' ' * 20 META_FILENAME = 'meta.yaml' META_N = 'n' diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore deleted file mode 100644 index 1a9f241854..0000000000 --- a/tests/incremental/.gitignore +++ /dev/null @@ -1 +0,0 @@ -99-* \ No newline at end of file From de00d79962e4bc08ab3f56f70a024fcacbf7bec4 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 16 Jun 2023 20:03:41 +0200 Subject: [PATCH 097/131] Update gitignore --- .../test-generation/.gitignore | 101 +++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore index f1082b56eb..98ebde540c 100644 --- a/scripts/incremental-test-generation/test-generation/.gitignore +++ b/scripts/incremental-test-generation/test-generation/.gitignore @@ -2,11 +2,106 @@ test* meta* input.c repo -99-* -98-* temp* config.yaml api-key.yaml +00-* +01-* +02-* +03-* +04-* +05-* +06-* +07-* +08-* +09-* +10-* +11-* +12-* +13-* +14-* +15-* +16-* +17-* +18-* +19-* +20-* +21-* +22-* +23-* +24-* +25-* +26-* +27-* +28-* +29-* +30-* +31-* +32-* +33-* +34-* +35-* +36-* +37-* +38-* +39-* +40-* +41-* 42-* 43-* -44-* \ No newline at end of file +44-* +45-* +46-* +47-* +48-* +49-* +50-* +51-* +52-* +53-* +54-* +55-* +56-* +57-* +58-* +59-* +60-* +61-* +62-* +63-* +64-* +65-* +66-* +67-* +68-* +69-* +70-* +71-* +72-* +73-* +74-* +75-* +76-* +77-* +78-* +79-* +80-* +81-* +82-* +83-* +84-* +85-* +86-* +87-* +88-* +89-* +90-* +91-* +92-* +93-* +94-* +95-* +96-* +97-* +98-* +99-* \ No newline at end of file From 44932c8553bb41e1011396ecae38e016bc63df37 Mon Sep 17 00:00:00 2001 From: J2000A Date: Sat, 17 Jun 2023 15:10:06 +0200 Subject: [PATCH 098/131] Give temp name only in CLI --- scripts/incremental-test-generation/clang-mutations/.gitignore | 2 -- scripts/incremental-test-generation/test-generation/RUN_CLI.py | 2 +- .../test-generation/generators/generate_git.py | 2 +- .../test-generation/util/generate_programs.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 scripts/incremental-test-generation/clang-mutations/.gitignore diff --git a/scripts/incremental-test-generation/clang-mutations/.gitignore b/scripts/incremental-test-generation/clang-mutations/.gitignore deleted file mode 100644 index 4f68c87b67..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -test.c -test-clean.c \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/test-generation/RUN_CLI.py index 278d2f24eb..72fee1e2b5 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/test-generation/RUN_CLI.py @@ -44,7 +44,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio # Run tests if is_run_tests: - test_path = os.path.abspath(os.path.join(os.path.curdir, 'temp/100-temp')) + test_path = os.path.abspath(os.path.join(temp_path, '/100-temp')) if enable_precision: print(SEPERATOR) print(f'Running {COLOR_BLUE}PRECISION TEST{COLOR_RESET}:') diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/test-generation/generators/generate_git.py index 5dcbc9e8b1..b06b1a686c 100644 --- a/scripts/incremental-test-generation/test-generation/generators/generate_git.py +++ b/scripts/incremental-test-generation/test-generation/generators/generate_git.py @@ -22,7 +22,7 @@ def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_comm git_info_sh_path = os.path.expanduser(os.path.abspath(git_info_sh_path)) print(SEPERATOR) - print('[GIT] Cloning into temp/repo') + print(f'[GIT] Cloning into {temp_repo_dir}') _clone_repo(git_info_sh_path, temp_repo_dir) build_path = _get_build_path(git_info_sh_path, temp_repo_dir) print(f'{COLOR_GREEN}[GIT] Cloning finished{COLOR_RESET}') diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/test-generation/util/generate_programs.py index 91e082a1ee..728e2c9608 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_programs.py +++ b/scripts/incremental-test-generation/test-generation/util/generate_programs.py @@ -68,7 +68,7 @@ def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apik if failed_count == 0: print(f"\r{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") else: - print(f"\r{COLOR_RED}There were {failed_count} files not compiling (stderr written to temp/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") + print(f"\r{COLOR_RED}There were {failed_count} files not compiling (stderr written to {temp_dir}/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate programs in the working directory') From b7f33e619d60862d04e87d82e28ace44c02d6ee0 Mon Sep 17 00:00:00 2001 From: J2000A Date: Sat, 17 Jun 2023 15:50:45 +0200 Subject: [PATCH 099/131] move test-generation up --- .../incremental-test-generation/.gitignore | 4 + scripts/incremental-test-generation/README.md | 8 ++ .../{test-generation => }/RUN_CLI.py | 11 +- .../generators/.gitignore | 0 .../generators/README.md | 0 .../generators/generate_git.py | 0 .../generators/generate_git_build.sh | 0 .../generate_git_build_USER_INFO_TEMPLATE.sh | 0 .../generators/generate_ml.py | 0 .../generators/generate_mutations.py | 0 .../sample-files/.gitignore | 1 - .../test-generation/.gitignore | 107 ------------------ .../test-generation/README.md | 2 - .../test-generation/util/__init__.py | 0 .../{test-generation => }/util/.gitignore | 3 +- .../{test-generation => }/util/README.md | 0 .../{test-generation => }/util/add_check.py | 0 .../util/add_check_comments.py | 0 .../util/generate_programs.py | 0 .../util/generate_tests.py | 2 +- .../{test-generation => }/util/run_tests.py | 0 .../{test-generation => }/util/util.py | 0 22 files changed, 19 insertions(+), 119 deletions(-) create mode 100644 scripts/incremental-test-generation/.gitignore create mode 100644 scripts/incremental-test-generation/README.md rename scripts/incremental-test-generation/{test-generation => }/RUN_CLI.py (98%) rename scripts/incremental-test-generation/{test-generation => }/generators/.gitignore (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/README.md (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/generate_git.py (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/generate_git_build.sh (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/generate_git_build_USER_INFO_TEMPLATE.sh (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/generate_ml.py (100%) rename scripts/incremental-test-generation/{test-generation => }/generators/generate_mutations.py (100%) delete mode 100644 scripts/incremental-test-generation/test-generation/.gitignore delete mode 100644 scripts/incremental-test-generation/test-generation/README.md delete mode 100644 scripts/incremental-test-generation/test-generation/util/__init__.py rename scripts/incremental-test-generation/{test-generation => }/util/.gitignore (50%) rename scripts/incremental-test-generation/{test-generation => }/util/README.md (100%) rename scripts/incremental-test-generation/{test-generation => }/util/add_check.py (100%) rename scripts/incremental-test-generation/{test-generation => }/util/add_check_comments.py (100%) rename scripts/incremental-test-generation/{test-generation => }/util/generate_programs.py (100%) rename scripts/incremental-test-generation/{test-generation => }/util/generate_tests.py (99%) rename scripts/incremental-test-generation/{test-generation => }/util/run_tests.py (100%) rename scripts/incremental-test-generation/{test-generation => }/util/util.py (100%) diff --git a/scripts/incremental-test-generation/.gitignore b/scripts/incremental-test-generation/.gitignore new file mode 100644 index 0000000000..6468e26489 --- /dev/null +++ b/scripts/incremental-test-generation/.gitignore @@ -0,0 +1,4 @@ +temp/** +config.yaml +api-key.yaml +out/** \ No newline at end of file diff --git a/scripts/incremental-test-generation/README.md b/scripts/incremental-test-generation/README.md new file mode 100644 index 0000000000..7d9d7657c5 --- /dev/null +++ b/scripts/incremental-test-generation/README.md @@ -0,0 +1,8 @@ +# Create the clang tidy checks +Read [this](clang-mutations/README.md) for creating the clang tidy checks needed to generate mutations. + +# Sample files +In `samle-files/` you find some examples you can use as input to get started. + +# Mutation Generator +Run `python3 RUN_CLI.py` to start the Comand Line Interface. With `-h` you can see the command line options for skipping the interactive user input. \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/RUN_CLI.py b/scripts/incremental-test-generation/RUN_CLI.py similarity index 98% rename from scripts/incremental-test-generation/test-generation/RUN_CLI.py rename to scripts/incremental-test-generation/RUN_CLI.py index 72fee1e2b5..55815c6f2f 100644 --- a/scripts/incremental-test-generation/test-generation/RUN_CLI.py +++ b/scripts/incremental-test-generation/RUN_CLI.py @@ -1,6 +1,5 @@ import argparse import os -import shutil import sys import questionary import yaml @@ -44,7 +43,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio # Run tests if is_run_tests: - test_path = os.path.abspath(os.path.join(temp_path, '/100-temp')) + test_path = os.path.abspath(os.path.join(temp_path, '100-temp')) if enable_precision: print(SEPERATOR) print(f'Running {COLOR_BLUE}PRECISION TEST{COLOR_RESET}:') @@ -64,7 +63,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio # Write out custom test files if create_tests: print(SEPERATOR) - correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), test_name) + correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'out', test_name) print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') paths = generate_tests(temp_path, correctness_path, goblint_config, precision_test=False, temp_name=False) if len(paths) > 1: @@ -73,7 +72,7 @@ def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutatio print(f'{COLOR_GREEN}Test stored in the file: {path}{COLOR_RESET}') if enable_precision: print(SEPERATOR) - precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), precision_name) + precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'out', precision_name) print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') paths = generate_tests(temp_path, precision_path, goblint_config, precision_test=False, temp_name=False) if len(paths) > 1: @@ -226,7 +225,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test if create_tests and test_name == None: while True: - test_name = questionary.text('Enter the test name: ', default="99-test").ask() + test_name = questionary.text('Enter the test name: ', default="90-test").ask() if check_test_name(test_name): break @@ -235,7 +234,7 @@ def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test if create_tests and enable_precision and precision_name == None: while True: - precision_name = questionary.text('Enter the precision test name: ', default="98-precision").ask() + precision_name = questionary.text('Enter the precision test name: ', default="80-precision").ask() if check_test_name(precision_name): break diff --git a/scripts/incremental-test-generation/test-generation/generators/.gitignore b/scripts/incremental-test-generation/generators/.gitignore similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/.gitignore rename to scripts/incremental-test-generation/generators/.gitignore diff --git a/scripts/incremental-test-generation/test-generation/generators/README.md b/scripts/incremental-test-generation/generators/README.md similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/README.md rename to scripts/incremental-test-generation/generators/README.md diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git.py b/scripts/incremental-test-generation/generators/generate_git.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_git.py rename to scripts/incremental-test-generation/generators/generate_git.py diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/generators/generate_git_build.sh similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_git_build.sh rename to scripts/incremental-test-generation/generators/generate_git_build.sh diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh b/scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh rename to scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/generators/generate_ml.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_ml.py rename to scripts/incremental-test-generation/generators/generate_ml.py diff --git a/scripts/incremental-test-generation/test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/generators/generate_mutations.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/generators/generate_mutations.py rename to scripts/incremental-test-generation/generators/generate_mutations.py diff --git a/scripts/incremental-test-generation/sample-files/.gitignore b/scripts/incremental-test-generation/sample-files/.gitignore index 9ca8693c30..530234efac 100644 --- a/scripts/incremental-test-generation/sample-files/.gitignore +++ b/scripts/incremental-test-generation/sample-files/.gitignore @@ -1,2 +1 @@ **/ -test.* diff --git a/scripts/incremental-test-generation/test-generation/.gitignore b/scripts/incremental-test-generation/test-generation/.gitignore deleted file mode 100644 index 98ebde540c..0000000000 --- a/scripts/incremental-test-generation/test-generation/.gitignore +++ /dev/null @@ -1,107 +0,0 @@ -test* -meta* -input.c -repo -temp* -config.yaml -api-key.yaml -00-* -01-* -02-* -03-* -04-* -05-* -06-* -07-* -08-* -09-* -10-* -11-* -12-* -13-* -14-* -15-* -16-* -17-* -18-* -19-* -20-* -21-* -22-* -23-* -24-* -25-* -26-* -27-* -28-* -29-* -30-* -31-* -32-* -33-* -34-* -35-* -36-* -37-* -38-* -39-* -40-* -41-* -42-* -43-* -44-* -45-* -46-* -47-* -48-* -49-* -50-* -51-* -52-* -53-* -54-* -55-* -56-* -57-* -58-* -59-* -60-* -61-* -62-* -63-* -64-* -65-* -66-* -67-* -68-* -69-* -70-* -71-* -72-* -73-* -74-* -75-* -76-* -77-* -78-* -79-* -80-* -81-* -82-* -83-* -84-* -85-* -86-* -87-* -88-* -89-* -90-* -91-* -92-* -93-* -94-* -95-* -96-* -97-* -98-* -99-* \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/README.md b/scripts/incremental-test-generation/test-generation/README.md deleted file mode 100644 index 705d2b2058..0000000000 --- a/scripts/incremental-test-generation/test-generation/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Mutation Generator -Run `python3 RUN_CLI.py` to start the Comand Line Interface. With `-h` you can see the command line options for skipping the interactive user input. \ No newline at end of file diff --git a/scripts/incremental-test-generation/test-generation/util/__init__.py b/scripts/incremental-test-generation/test-generation/util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/incremental-test-generation/test-generation/util/.gitignore b/scripts/incremental-test-generation/util/.gitignore similarity index 50% rename from scripts/incremental-test-generation/test-generation/util/.gitignore rename to scripts/incremental-test-generation/util/.gitignore index 9d0dd38eed..4ecb4bd38f 100644 --- a/scripts/incremental-test-generation/test-generation/util/.gitignore +++ b/scripts/incremental-test-generation/util/.gitignore @@ -1,3 +1,2 @@ __pycache__ -test* -99-* +__init__.py diff --git a/scripts/incremental-test-generation/test-generation/util/README.md b/scripts/incremental-test-generation/util/README.md similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/README.md rename to scripts/incremental-test-generation/util/README.md diff --git a/scripts/incremental-test-generation/test-generation/util/add_check.py b/scripts/incremental-test-generation/util/add_check.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/add_check.py rename to scripts/incremental-test-generation/util/add_check.py diff --git a/scripts/incremental-test-generation/test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/util/add_check_comments.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/add_check_comments.py rename to scripts/incremental-test-generation/util/add_check_comments.py diff --git a/scripts/incremental-test-generation/test-generation/util/generate_programs.py b/scripts/incremental-test-generation/util/generate_programs.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/generate_programs.py rename to scripts/incremental-test-generation/util/generate_programs.py diff --git a/scripts/incremental-test-generation/test-generation/util/generate_tests.py b/scripts/incremental-test-generation/util/generate_tests.py similarity index 99% rename from scripts/incremental-test-generation/test-generation/util/generate_tests.py rename to scripts/incremental-test-generation/util/generate_tests.py index 363bafcee4..985db64b85 100644 --- a/scripts/incremental-test-generation/test-generation/util/generate_tests.py +++ b/scripts/incremental-test-generation/util/generate_tests.py @@ -22,7 +22,7 @@ def generate_tests(temp_dir, target_dir, goblint_config, precision_test, temp_na shutil.rmtree(target_dir) else: sys.exit(-1) - os.mkdir(target_dir) + os.makedirs(target_dir) # Read the meta.yaml meta_path = os.path.join(temp_dir,META_FILENAME) diff --git a/scripts/incremental-test-generation/test-generation/util/run_tests.py b/scripts/incremental-test-generation/util/run_tests.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/run_tests.py rename to scripts/incremental-test-generation/util/run_tests.py diff --git a/scripts/incremental-test-generation/test-generation/util/util.py b/scripts/incremental-test-generation/util/util.py similarity index 100% rename from scripts/incremental-test-generation/test-generation/util/util.py rename to scripts/incremental-test-generation/util/util.py From 4bac14d618f8c3c11014b1cf1bdb58d90b75fedd Mon Sep 17 00:00:00 2001 From: J2000A Date: Sat, 17 Jun 2023 15:52:02 +0200 Subject: [PATCH 100/131] move to bench repo --- .../incremental-test-generation/.gitignore | 4 - scripts/incremental-test-generation/README.md | 8 - .../incremental-test-generation/RUN_CLI.py | 397 --------- .../ConstantReplacementCheck.cpp | 40 - .../ConstantReplacementCheck.h | 19 - .../LogicalConnectorReplacementCheck.cpp | 40 - .../LogicalConnectorReplacementCheck.h | 19 - .../clang-mutations/MUTATIONS.md | 114 --- .../clang-mutations/README.md | 66 -- .../RelationalOperatorReplacementCheck.cpp | 51 -- .../RelationalOperatorReplacementCheck.h | 19 - .../RemoveFunctionBodyCheck.cpp | 68 -- .../clang-mutations/RemoveFunctionBodyCheck.h | 23 - .../clang-mutations/RemoveThreadCheck.cpp | 63 -- .../clang-mutations/RemoveThreadCheck.h | 22 - .../RemoveThreadWrapperCheck.cpp | 38 - .../RemoveThreadWrapperCheck.h | 23 - .../UnaryOperatorInversionCheck.cpp | 33 - .../UnaryOperatorInversionCheck.h | 19 - .../generators/.gitignore | 2 - .../generators/README.md | 66 -- .../generators/generate_git.py | 193 ----- .../generators/generate_git_build.sh | 63 -- .../generate_git_build_USER_INFO_TEMPLATE.sh | 30 - .../generators/generate_ml.py | 242 ------ .../generators/generate_mutations.py | 207 ----- .../sample-files/.gitignore | 1 - .../sample-files/REPOSITORIES.md | 3 - .../sample-files/comparisons.c | 40 - .../sample-files/constants.c | 11 - .../sample-files/functions.c | 40 - .../sample-files/large.c | 762 ------------------ .../sample-files/minimalTestNothing.c | 38 - .../sample-files/pthread.c | 124 --- .../sample-files/threads.c | 41 - .../sample-files/zlib.sh | 30 - .../util/.gitignore | 2 - .../util/README.md | 14 - .../util/add_check.py | 67 -- .../util/add_check_comments.py | 40 - .../util/generate_programs.py | 124 --- .../util/generate_tests.py | 182 ----- .../util/run_tests.py | 96 --- .../incremental-test-generation/util/util.py | 75 -- 44 files changed, 3559 deletions(-) delete mode 100644 scripts/incremental-test-generation/.gitignore delete mode 100644 scripts/incremental-test-generation/README.md delete mode 100644 scripts/incremental-test-generation/RUN_CLI.py delete mode 100644 scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/MUTATIONS.md delete mode 100644 scripts/incremental-test-generation/clang-mutations/README.md delete mode 100644 scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h delete mode 100644 scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp delete mode 100644 scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h delete mode 100644 scripts/incremental-test-generation/generators/.gitignore delete mode 100644 scripts/incremental-test-generation/generators/README.md delete mode 100644 scripts/incremental-test-generation/generators/generate_git.py delete mode 100755 scripts/incremental-test-generation/generators/generate_git_build.sh delete mode 100644 scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh delete mode 100644 scripts/incremental-test-generation/generators/generate_ml.py delete mode 100644 scripts/incremental-test-generation/generators/generate_mutations.py delete mode 100644 scripts/incremental-test-generation/sample-files/.gitignore delete mode 100644 scripts/incremental-test-generation/sample-files/REPOSITORIES.md delete mode 100644 scripts/incremental-test-generation/sample-files/comparisons.c delete mode 100644 scripts/incremental-test-generation/sample-files/constants.c delete mode 100644 scripts/incremental-test-generation/sample-files/functions.c delete mode 100644 scripts/incremental-test-generation/sample-files/large.c delete mode 100644 scripts/incremental-test-generation/sample-files/minimalTestNothing.c delete mode 100644 scripts/incremental-test-generation/sample-files/pthread.c delete mode 100644 scripts/incremental-test-generation/sample-files/threads.c delete mode 100755 scripts/incremental-test-generation/sample-files/zlib.sh delete mode 100644 scripts/incremental-test-generation/util/.gitignore delete mode 100644 scripts/incremental-test-generation/util/README.md delete mode 100644 scripts/incremental-test-generation/util/add_check.py delete mode 100644 scripts/incremental-test-generation/util/add_check_comments.py delete mode 100644 scripts/incremental-test-generation/util/generate_programs.py delete mode 100644 scripts/incremental-test-generation/util/generate_tests.py delete mode 100644 scripts/incremental-test-generation/util/run_tests.py delete mode 100644 scripts/incremental-test-generation/util/util.py diff --git a/scripts/incremental-test-generation/.gitignore b/scripts/incremental-test-generation/.gitignore deleted file mode 100644 index 6468e26489..0000000000 --- a/scripts/incremental-test-generation/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -temp/** -config.yaml -api-key.yaml -out/** \ No newline at end of file diff --git a/scripts/incremental-test-generation/README.md b/scripts/incremental-test-generation/README.md deleted file mode 100644 index 7d9d7657c5..0000000000 --- a/scripts/incremental-test-generation/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Create the clang tidy checks -Read [this](clang-mutations/README.md) for creating the clang tidy checks needed to generate mutations. - -# Sample files -In `samle-files/` you find some examples you can use as input to get started. - -# Mutation Generator -Run `python3 RUN_CLI.py` to start the Comand Line Interface. With `-h` you can see the command line options for skipping the interactive user input. \ No newline at end of file diff --git a/scripts/incremental-test-generation/RUN_CLI.py b/scripts/incremental-test-generation/RUN_CLI.py deleted file mode 100644 index 55815c6f2f..0000000000 --- a/scripts/incremental-test-generation/RUN_CLI.py +++ /dev/null @@ -1,397 +0,0 @@ -import argparse -import os -import sys -import questionary -import yaml -from pathlib import Path -from util.util import * -from util.generate_programs import generate_programs -from util.generate_tests import generate_tests -from util.run_tests import run_tests -from generators.generate_mutations import add_mutation_options, get_mutations_from_args -from generators.generate_ml import validate_interesting_lines - -logo = ''' - __ __ _ _ _ - | \/ | | | | | (_) - | \ / |_ _| |_ __ _| |_ _ ___ _ __ - | |\/| | | | | __/ _` | __| |/ _ \| '_ \ - | | | | |_| | || (_| | |_| | (_) | | | | - |_| |_|\__,_|\__\__,_|\__|_|\___/|_| |_| - _____ _ - / ____| | | - | | __ ___ _ __ ___ _ __ __ _| |_ ___ _ __ - | | |_ |/ _ \ '_ \ / _ \ '__/ _` | __/ _ \| '__| - | |__| | __/ | | | __/ | | (_| | || (_) | | - \_____|\___|_| |_|\___|_| \__,_|\__\___/|_| - - - - ''' - -def run(goblint_path, llvm_path, input_path, is_mutation, is_ml, is_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, is_run_tests, api_key_path, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end): - # Make paths absolute - goblint_path = os.path.abspath(os.path.expanduser(goblint_path)) - llvm_path = os.path.abspath(os.path.expanduser(llvm_path)) - input_path = os.path.abspath(os.path.expanduser(input_path)) - - # Generate the programs - goblint_executable_path = os.path.join(goblint_path, 'goblint') - clang_tidy_path = os.path.join(llvm_path, 'build', 'bin', 'clang-tidy') - temp_path = os.path.abspath(os.path.join(os.path.curdir, 'temp')) - generate_programs(input_path, temp_path, clang_tidy_path, goblint_executable_path, api_key_path, mutations, is_mutation, is_ml, is_git, ml_count, ml_select, ml_interesting, ml_16k, git_start, git_end) - - # Run tests - if is_run_tests: - test_path = os.path.abspath(os.path.join(temp_path, '100-temp')) - if enable_precision: - print(SEPERATOR) - print(f'Running {COLOR_BLUE}PRECISION TEST{COLOR_RESET}:') - paths = generate_tests(temp_path, test_path, goblint_config, precision_test=True, temp_name=True) - if len(paths) > 1: - print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") - for path in paths: - run_tests(input_path, path, goblint_path, cfg) - print(SEPERATOR) - print(f'Running {COLOR_BLUE}CORRECTNESS TEST{COLOR_RESET}:') - paths = generate_tests(temp_path, test_path, goblint_config, precision_test=False, temp_name=True) - if len(paths) > 1: - print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") - for path in paths: - run_tests(input_path, path, goblint_path, cfg) - - # Write out custom test files - if create_tests: - print(SEPERATOR) - correctness_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'out', test_name) - print(f'Writing out {COLOR_BLUE}CUSTOM CORRECTNESS TEST {test_name}{COLOR_RESET} files:') - paths = generate_tests(temp_path, correctness_path, goblint_config, precision_test=False, temp_name=False) - if len(paths) > 1: - print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") - for path in paths: - print(f'{COLOR_GREEN}Test stored in the file: {path}{COLOR_RESET}') - if enable_precision: - print(SEPERATOR) - precision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'out', precision_name) - print(f'Writing out {COLOR_BLUE}CUSTOM PRECISION TEST {precision_name}{COLOR_RESET} files:') - paths = generate_tests(temp_path, precision_path, goblint_config, precision_test=False, temp_name=False) - if len(paths) > 1: - print(f"{COLOR_YELLOW}[INFO] There were more than 99 programs generated, so the tests had to be spitted into multiple directories{COLOR_RESET}") - for path in paths: - print(f'{COLOR_GREEN}Test stored in the file: {path}{COLOR_RESET}') - -def cli(enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, input, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end, git_no_commit): - # Check config file - config_path = Path(CONFIG_FILENAME) - config = {} - if not config_path.is_file(): - print(f'Config file "{config_path}" not found. Please provide the paths:') - goblint_path = questionary.text('Enter the path to the goblint repository: ', default="~/Goblint-Repo/analyzer").ask() - llvm_path = questionary.text('Enter the path to the llvm repository with the modified clang-tidy: ', default="~/Clang-Repo/llvm-project").ask() - config.update({CONFIG_GOBLINT: goblint_path, CONFIG_LLVM: llvm_path, CONFIG_LAST_INPUT_MUTATION: '', CONFIG_LAST_INPUT_GIT: ''}) - last_input_mutation = '' - last_input_git = '' - with open(config_path, 'w') as outfile: - yaml.dump(config, outfile) - else: - with open(config_path, 'r') as stream: - config = yaml.safe_load(stream) - goblint_path = config[CONFIG_GOBLINT] - llvm_path = config[CONFIG_LLVM] - last_input_mutation = config[CONFIG_LAST_INPUT_MUTATION] - last_input_git = config[CONFIG_LAST_INPUT_GIT] - print(f'Using goblint-path (change in ./{CONFIG_FILENAME}): {goblint_path}') - print(f'Using llvm-path (change in ./{CONFIG_FILENAME}): {llvm_path}') - - # Handle Questions - if not (enable_mutations or enable_ml or enable_git): - while True: - generators = questionary.checkbox( - 'Select one or more generator types (When git is checked no other can be checked!):', - choices=[ - questionary.Choice('Mutations', checked=True), - 'ML', - 'Git' - ]).ask() - - # check if 'Git' is selected along with other options - if 'Git' in generators and len(generators) > 1: - print(f"{COLOR_RED}If 'Git' is selected, no other options should be selected. Please select again.{COLOR_RESET}") - continue - else: - break - - enable_mutations = 'Mutations' in generators - enable_ml = 'ML' in generators - enable_git = 'Git' in generators - - if enable_mutations: - selected_mutations = questionary.checkbox( - 'Select one or more mutation types:', - choices=[ - questionary.Choice('remove-function-body (RFB)', checked=True), - questionary.Choice('unary-operator-inversion (UOI)', checked=True), - questionary.Choice('relational-operator-replacement (ROR)', checked=True), - questionary.Choice('constant-replacement (CR)', checked=True), - questionary.Choice('remove-thread (RT)', checked=True), - questionary.Choice('logical-connector-replacement (LCR)', checked=True), - ]).ask() - mutations = Mutations( - rfb='remove-function-body (RFB)' in selected_mutations, - uoi='unary-operator-inversion (UOI)' in selected_mutations, - ror='relational-operator-replacement (ROR)' in selected_mutations, - cr='constant-replacement (CR)' in selected_mutations, - rt='remove-thread (RT)' in selected_mutations, - lcr='logical-connector-replacement (LCR)' in selected_mutations - ) - - # Check for API Key - if enable_ml: - key_path = Path(APIKEY_FILENAME) - key_data = {} - if not key_path.is_file(): - print(f'Api key file "{key_path}" for OpenAi not found. Please provide the informations:') - print('Be aware that the information is stored unencrypted. Do not remove the file from .gitignore!') - print('Create an account here: https://openai.com/blog/openai-api') - print('Create an API Key here: https://platform.openai.com/account/api-keys') - print('Get your organization id here: https://platform.openai.com/account/org-settings') - key = questionary.text('Enter the api key:').ask() - org = questionary.text('Enter the organisation id:').ask() - key_data.update({APIKEY_APIKEY: key, APIKEY_ORGANISATION: org}) - with open(key_path, 'w') as outfile: - yaml.dump(key_data, outfile) - else: - with open(key_path, 'r') as stream: - key_data = yaml.safe_load(stream) - key = key_data[APIKEY_APIKEY] - org = key_data[APIKEY_ORGANISATION] - print(f'Using api-key for ML (change in ./{APIKEY_FILENAME}): ...{key[-4:]}') - print(f'Using organisation id for ML (change in ./{APIKEY_FILENAME}): ...{org[-4:]}') - key_path = os.path.abspath(key_path) - else: - key_path = None - - # Check for config file - if goblint_config == None: - goblint_config = questionary.text('Path to a goblint config file used to create tests. Passing {} creates an empty config file.', default='{}').ask() - if goblint_config == '{}' or goblint_config == '': - goblint_config = None - - # ML Options - if enable_ml and ml_count == None: - while True: - ml_count = questionary.text('How many different programs should be generated with ML?', default=str(DEFAULT_ML_COUNT)).ask() - if not ml_count.strip('\n').isdigit(): - print(f"{COLOR_RED}Please enter a valid number.{COLOR_RESET}") - continue - ml_count = int(ml_count.strip('\n')) - if ml_count <= 0: - print(f"{COLOR_RED}Please enter a number greater zero.{COLOR_RESET}") - continue - break - - if enable_ml and ml_select == None: - while True: - ml_select = questionary.text('How many lines should be selected for the snippet from the input file?', default=str(DEFAULT_ML_SELECT)).ask() - if not ml_select.strip('\n').isdigit(): - print(f"{COLOR_RED}Please enter a valid number.{COLOR_RESET}") - continue - ml_select = int(ml_select.strip('\n')) - if ml_select <= 0: - print(f"{COLOR_RED}Please enter a number greater zero.{COLOR_RESET}") - continue - break - - if enable_ml and ml_16k == None: - ml_16k = questionary.confirm('Use the gpt-3.5-turbo-16k model instead of the gpt-3.5-turbo model', default=False).ask() - - if enable_ml and ml_interesting == None: - while True: - ml_interesting = questionary.text('From which start lines should the snippet start be choosen randomly ([] stands for all)?', default='[]').ask() - if validate_interesting_lines(ml_interesting, None) == None: - print(f'{COLOR_RED}Please enter a valid string like [1, 42], [99], ....{COLOR_RESET}') - continue - break - - # Git options - if enable_git and not (git_start != None and git_end != None) and not git_no_commit: - if questionary.confirm('Do you want to give a start and end commit hash?', default=False).ask(): - git_start = questionary.text('Enter start commit hash:').ask() - git_end = questionary.text('Enter end commit hash:').ask() - - # Output options - if create_tests == None: - create_tests = questionary.confirm('Create test files?', default=False).ask() - - if create_tests and test_name == None: - while True: - test_name = questionary.text('Enter the test name: ', default="90-test").ask() - if check_test_name(test_name): - break - - if enable_precision == None: - enable_precision = questionary.confirm('Enable precision tests?', default=False).ask() - - if create_tests and enable_precision and precision_name == None: - while True: - precision_name = questionary.text('Enter the precision test name: ', default="80-precision").ask() - if check_test_name(precision_name): - break - - if running == None: - running = questionary.confirm('Run the tests?').ask() - - if running and cfg == None: - cfg = questionary.confirm('Run the fine grained cfg tests?').ask() - - # input options - if input == None: - while True: - if enable_mutations or enable_ml: - input = questionary.text('Enter the path to the c program for the mutations: ', default=last_input_mutation).ask() - config.update({CONFIG_LAST_INPUT_MUTATION: input}) - else: - input = questionary.text('Enter the path to the sh script with informations about the git repository (Use [-s] to see the template script ): ', default=last_input_git).ask() - config.update({CONFIG_LAST_INPUT_GIT: input}) - if not os.path.exists(input): - print(f"{COLOR_RED}Please enter a valid path.{COLOR_RESET}") - continue - with open(config_path, 'w') as outfile: - yaml.dump(config, outfile) - break - - run(goblint_path, llvm_path, input, enable_mutations, enable_ml, enable_git, mutations, goblint_config, test_name, create_tests, enable_precision, precision_name, running, key_path, ml_count, ml_select, ml_interesting, ml_16k, cfg, git_start, git_end) - - -if __name__ == "__main__": - print(f'{COLOR_YELLOW}Use [-h] to see the command line options{COLOR_RESET}') - print(logo) - - parser = argparse.ArgumentParser(description='Generates mutations for creating incremental tests') - parser.add_argument('-m', '--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') - parser.add_argument('-o', '--enable-ml', action='store_true', help='Enable ML') - parser.add_argument('-g', '--enable-git', action='store_true', help='Enable Git') - parser.add_argument('-c', '--goblint-config', help='Path to a goblint config file used to create tests (passing "{}" as argument creates an empty config file)') - parser.add_argument('-ep', '--enable-precision', action='store_true', help='Enable Precision Tests') - parser.add_argument('-dp', '--disable-precision', action='store_true', help='Disable Precision Tests') - parser.add_argument('-er', '--enable-running', action='store_true', help='Enable running tests') - parser.add_argument('-dr', '--disable-running', action='store_true', help='Disable running tests') - parser.add_argument('-et', '--enable-create-tests', action='store_true', help='Enable creating test files') - parser.add_argument('-dt', '--disable-create-tests', action='store_true', help='Disable creating test files') - parser.add_argument('-ec', '--enable-cfg', action='store_true', help='Enable fine grained cfg tests') - parser.add_argument('-dc', '--disable-cfg', action='store_true', help='Disable fine grained cfg tests') - parser.add_argument('-t', '--test-name', help='Test name') - parser.add_argument('-p', '--precision-name', help='Precision test name') - parser.add_argument('-i', '--input', help='Input File') - - # Add mutation options - add_mutation_options(parser) - - # Add ML options - parser.add_argument('-mc', '--ml-count', type=int, default=-1, help='How many different programs should be generated with ML?') - parser.add_argument('-ms', '--ml-select', type=int, default=-1, help='How many lines should be selected for the snippet from the input file?') - parser.add_argument('-mi', '--ml-interesting', help='From which start lines should the snippet start be choosen randomly? Exp. :[] = From all lines, [1, 42], ...') - parser.add_argument('-m4', '--ml-4k', action='store_true', help='Use the gpt-3.5-turbo model instead of the gpt-3.5-turbo-16k model') - parser.add_argument('-m16', '--ml-16k', action='store_true', help='Use the gpt-3.5-turbo-16k model instead of the gpt-3.5-turbo model') - - # Add GIT options - parser.add_argument('-s', '--template-script', action='store_true', help='Print the template script for git repositories') - parser.add_argument('-gs', '--git-start-commit', help='The hash of the first commit to consider') - parser.add_argument('-ge', '--git-end-commit', help='The hash of the last commit to consider') - parser.add_argument('-gn', '--git-no-commit', action='store_true', help='Suppress asking for commit hashes in CLI') - - args = parser.parse_args() - - if args.template_script: - template_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'generators', 'generate_git_build_USER_INFO_TEMPLATE.sh')) - print(f'{COLOR_YELLOW}Template can be found at: {template_path}{COLOR_RESET}') - print('') - with open(template_path, 'r') as file: - content = file.read() - print(content) - print('') - sys.exit(0) - - if args.enable_mutations or args.enable_ml or args.enable_git: - # If using git, only git can be used - if args.enable_git and (args.enable_ml or args.enable_mutations): - parser.error("--enable-git cannot be used with --enable-ml or --enable-mutations") - - # If all mutation options are false, set all to true - mutations = get_mutations_from_args(args) - non_str_attributes = [attr for attr in vars(mutations) if not attr.endswith('_s')] - if all(getattr(mutations, attr) is False for attr in non_str_attributes): - mutations = Mutations(True, True, True, True, True, True) - else: - args.enable_mutations = None - args.enable_ml = None - args.enable_git = None - mutations = None - - git_start_commit = args.git_start_commit - git_end_commit = args.git_end_commit - if (git_start_commit == None and git_end_commit != None) or (git_start_commit != None and git_end_commit == None): - parser.error('[ERROR] Give a git start commit hash AND a end commit hash') - - if args.enable_precision or args.disable_precision: - # Only one can be selected - if args.enable_precision and args.disable_precision: - parser.error('Precision can not be enabled AND diabled') - precision = args.enable_precision - else: - precision = None - - if args.enable_running or args.disable_running: - # Only one can be selected - if args.enable_running and args.disable_running: - parser.error('Running can not be enabled AND diabled') - running = args.enable_running - else: - running = None - - if args.enable_create_tests or args.disable_create_tests: - # Only one can be selected - if args.enable_create_tests and args.disable_create_tests: - parser.error('Create tests can not be enabled AND diabled') - create_tests = args.enable_create_tests - else: - create_tests = None - - if args.enable_cfg or args.disable_cfg: - # Only one can be selected - if args.enable_cfg and args.disable_cfg: - parser.error('Cfg can not be enabled AND diabled') - cfg = args.enable_cfg - else: - cfg = None - - if args.ml_count > 0: - ml_count = args.ml_count - else: - ml_count = None - - if args.ml_select > 0: - ml_select = args.ml_select - else: - ml_select = None - - if args.ml_interesting != None and validate_interesting_lines(args.ml_interesting, None) == None: - sys.exit(-1) - - if args.ml_4k or args.ml_16k: - # Only one can be selected - if args.ml_4k and args.ml_16k: - parser.error('Only one ml model can be selected!') - ml_16k = args.ml_16k - else: - ml_16k = None - - test_name = args.test_name - if test_name != None and not check_test_name(test_name): - sys.exit(-1) - - precision_name = args.precision_name - if precision_name != None and not check_test_name(precision_name): - sys.exit(-1) - - cli(args.enable_mutations, args.enable_ml, args.enable_git, mutations, args.goblint_config, test_name, create_tests, precision, precision_name, running, args.input, ml_count, ml_select, args.ml_interesting, ml_16k, cfg, git_start_commit, git_end_commit, args.git_no_commit) \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp deleted file mode 100644 index 18fa68b1c6..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "ConstantReplacementCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" -#include "clang/Lex/Lexer.h" - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void ConstantReplacementCheck::registerMatchers(MatchFinder *Finder) { - Finder->addMatcher(integerLiteral(isExpansionInMainFile(), unless(anyOf(equals(0), equals(1)))).bind("constant-replacement"), this); -} - -// Replaces all Integer Literals != 0 with 1 -void ConstantReplacementCheck::check(const MatchFinder::MatchResult &Result) { - const auto *MatchedDecl = Result.Nodes.getNodeAs("constant-replacement"); - - // Get old int value - llvm::APInt value = MatchedDecl->getValue(); - std::string valueAsString; - llvm::SmallString<8> buffer; - value.toString(buffer, 10, true); - valueAsString = buffer.str(); - // Get locations - SourceLocation Start = MatchedDecl->getBeginLoc(); - auto Range = MatchedDecl->getSourceRange(); - // Check if it is a Macro - std::string MacroInfo = " "; - if (MatchedDecl->getLocation().isMacroID()) { - std::string MacroId = Lexer::getImmediateMacroName(MatchedDecl->getLocation(), *Result.SourceManager, Result.Context->getLangOpts()).str(); - MacroInfo = "[MACRO][" + MacroId + "] "; - } - // Replace with 1 - std::string Replacement = "1 /* [MUTATION][CR]" + MacroInfo + "Replaced Constant " + valueAsString + " */"; - diag(Start, "[MUTATION][CR]" + MacroInfo + "Replaced Constant %0 with 1") - << valueAsString - << FixItHint::CreateReplacement(Range, Replacement); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h deleted file mode 100644 index 182c389aad..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/ConstantReplacementCheck.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Constant Replacement -class ConstantReplacementCheck : public ClangTidyCheck { -public: - ConstantReplacementCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_CONSTANTREPLACEMENTCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp deleted file mode 100644 index 21a04f1c6b..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "LogicalConnectorReplacementCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void LogicalConnectorReplacementCheck::registerMatchers(MatchFinder *Finder) { - Finder->addMatcher(stmt(expr(binaryOperator())).bind("logical-connector-replacement"), this); -} - -// "||" <-> "&&" -void LogicalConnectorReplacementCheck::check(const MatchFinder::MatchResult &Result) { - const auto *MatchedDecl = Result.Nodes.getNodeAs("logical-connector-replacement"); - - std::string Replacement; - - if (MatchedDecl->getOpcode() == BO_LOr) { - // || -> && - Replacement = "&&"; - } else if (MatchedDecl->getOpcode() == BO_LAnd) { - // && -> || - Replacement = "||"; - } else { - return; - } - Replacement += " /* [MUTATION][LCR] Replaced Logical Connector */"; - - // Get locations - SourceLocation Start = MatchedDecl->getOperatorLoc(); - SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(2); - auto Range = CharSourceRange::getCharRange(Start, End); - - diag(Start, "[MUTATION][LCR] Replaced Logical Connector %0") - << MatchedDecl->getOpcodeStr() - << FixItHint::CreateReplacement(Range, Replacement); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h deleted file mode 100644 index 9dd76e2856..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/LogicalConnectorReplacementCheck.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Logical Connector Replacement -class LogicalConnectorReplacementCheck : public ClangTidyCheck { -public: - LogicalConnectorReplacementCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_LOGICALCONNECTORREPLACEMENTCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md b/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md deleted file mode 100644 index 84d1a46310..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/MUTATIONS.md +++ /dev/null @@ -1,114 +0,0 @@ -# Mutations -In this document is described what the different mutations do. In the [Readme](README.md) file it is described how to run the mutations. - -## Remove Function Body - RFB -**readability-remove-function-body**
-Removes the function body and adds when needed a generic return statement. -``` -int sum(int a, int b) { - return a + b; -} -``` -`clang-tidy -checks=-*,readability-remove-function-body -fix test.c --` -``` -int sum(int a, int b) { return 0; /* [MUTATION][RFB] Stripped function of its body */ } -``` -### Special Option -`-config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'sum, divide, ...'}}"` Special option for the **remove-function-body** check to only remove the function body of functions named foo1 and foo2. - -## Unary Operator Inversion - UOI -**readability-unary-operator-inversion**
-Flips if statements by adding a negation. -``` -if (a < b) { - printf("a is less than b\n"); -} -``` -`clang-tidy -checks=-*,readability-unary-operator-inversion -fix comparisons.c --` -``` -if (!(a < b) /* [MUTATION][UOI] Inverted if statement */) { - printf("a is less than b\n"); -} -``` - -## Relational Operator Replacement - ROR -**readability-relational-operator-replacement**
-Replaces `<=` with `<`, `<` with `<=`, `>=` with `>` and `>` with `>=`. -``` -if (a <= b) { - printf("a is less than or equal to b\n"); -} -``` -`clang-tidy -checks=-*,readability-relational-operator-replacement -fix comparisons.c --` -``` -if (a < /* [MUTATION][ROR] Replaced Relational Operator */ b) { - printf("a is less than or equal to b\n"); -} -``` - -## Constant Replacement - CR -**readability-constant-replacement**
-Replaces constants unequal 0 and unequal 1 with 1. The usage of macros is replaced, but not the definition. This is marked with `[MACRO][macro_name]`. -``` -#define MY_MACRO_5 5 -#define MY_MACRO_0 0 -int a = 42; -int b = 0; -int c = MY_MACRO_5; -int d = MY_MACRO_0; -``` -`clang-tidy -checks=-*,readability-constant-replacement -fix constants.c --` -``` -#define MY_MACRO_5 5 -#define MY_MACRO_0 0 -int a = 1 /* [MUTATION][CR] Replaced Constant 42 */; -int b = 0; -int c = 1 /* [MUTATION][CR][MACRO][MY_MACRO_5] Replaced Constant 5 */; -int d = MY_MACRO_0; -``` - -## Remove Thread - RT -**readability-remove-thread**
-Replaces a `pthread_create` call with the function call itself. Additionally `0;` is added for the case that the result was checked in the form `result = pthread_create()` The arguments of the function call are kept. Symbols like `*` or `&` in front of the function name are ignored. -``` -result = pthread_create(&thread, &attr, thread_function, NULL); -``` -`clang-tidy -checks=-*,readability-remove-thread -fix pthread.c --` -``` -result = 0; thread_function(NULL) /* [MUTATION][RT][FUNCTION_NAME][thread_function] Thread creation was substituted with function call */; -``` - -### Remove Thread Wrapper - RTW -**readability-remove-thread-wrapper**
-Wraps the given Function Name for a pthread_create call. This should be run before `remove-thread`. The function name has to be passed. -``` -void *threadFunction(void *arg) { - //... -} -``` -`clang-tidy -checks=-*,readability-remove-thread-wrapper -config="{CheckOptions: {readability-remove-thread-wrapper.WrapFunctionName: 'threadFunction'}}" -fix pthread.c --` -``` -void *threadFunction(void *arg) { - //... -} -int threadFunction_wrap(void *arg) { - /*[MUTATION][RTW] Wrapped function for remove-thread */ - threadFunction(arg); - return 0; -} -``` - -## Logical Connector Replacement - LCR -**readability-logical-connector-replacement**
-Replaces the operator `&&` with `||` and vice versa. -``` -if (a && b) { - printf("Both a and b are non-zero\n"); -} -``` -`clang-tidy -checks=-*,readability-logical-connector-replacement -fix comparisons.c --` -``` -if (a || /* [MUTATION][LCR] Replaced Logical Connector */ b) { - printf("Both a and b are non-zero\n"); -} -``` \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/README.md b/scripts/incremental-test-generation/clang-mutations/README.md deleted file mode 100644 index 06e601b1e8..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Create Clang-Tidy Checks for mutations -In this document is described how you can create all the Clang-Tidy checks needed for generating the code mutations. - -## Dependencies -For building Clang you need to install some dependencies: - - - [ ] `sudo apt install ninja-build` - - [ ] `sudo apt install ccache` - - [ ] `sudo apt install lld` - -## Cloning the repository - - [ ] There are two alternatives for getting the repository - -For creating all the checks by yourself clone the **Official Clang Repository**: -`git clone https://github.com/llvm/llvm-project.git` (Tested with Version 17.0.0) -Alternatively you can clone the **Fork** with all the additional checks ready: -`git clone https://github.com/J2000A/llvm-project.git` - -## Creating the checks - - [ ] When you cloned the Official Clang Repository you need to add the checks. Otherwise you can skip this part. - - [ ] Move to the directory `llvm-project/clang-tools-extra` - -In this directory are the implementations of the checks with their corresponding `.cpp` and `.h` file. In the following we will use the **>>check-name<<** of each check. You can get it by the filename without the word "Check" at the end and changing the capital letters to small ones with a minus in front (e.g. the >>check-name<< of `RemoveFunctionBodyCheck.cpp` is `remove-function-body`). Repeat the following steps for all new checks. - - - [ ] Run `clang-tidy/add_new_check.py readability >>check-name<<` - - [ ] Replace `./clang-tidy/readability/>>check-name<<.cpp` with the implementation in this directory - - [ ] Replace `./clang-tidy/readability/>>check-name<<.h` with the header file in this directory - -Now you have added all the check we need for the mutations. - -## Build -The first build can take a while (up to multiple hours). But you can increase the performance by changing the parallel compile and link jobs. For me using the value 5 for both of them got me the fastest results. When using too many jobs the memory becomes a bottleneck. You can check the memory status with `free -h --giga`. -Additionally you may need to change the build target. Avaiable targets are: AMDGPU, ARM, AVR, BPF, Hexagon, Lanai, LoongArch, Mips, MSP430, NVPTX, PowerPC, RISCV, Sparc, SystemZ, VE, WebAssembly, X86, XCore - - - [ ] Move to the directory `llvm-project/` - - [ ] `mkdir build && cd build` - - [ ] `cmake -G "Ninja" -DLLVM_CCACHE_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_LINKER=lld -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_PARALLEL_COMPILE_JOBS=5 -DLLVM_PARALLEL_LINK_JOBS=5 ../llvm` - - [ ] `sudo ninja install` - -## Running Clang-Tidy -We will use the **>>check-name<<** again as defined in "Creating the checks". - -**Example:** Create the mutation "remove function body" on a file "test.c" in lines "4" and "14" when the function name is "foo": -`clang-tidy -checks=-*,readability-remove-function-body -fix --fix-errors -config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'foo'}}" -line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]' test.c --` - -**The command consists of the following components:** - - [ ] Clang-Tidy -`clang-tidy` The command itself. - - - [ ] General Options - `-checks=-*,readability->>check-name<<` Deactivating all checks except >>check-name<<. -`-fix` Applying the mutations. -`--fix-errors` Apply also when errors where detected -`-line filter='[{"name":"test.c","lines":[[4,4],[14,14]]}]'` Apply the mutations only on line 4 and 14. - - - [ ] Special Options -`-config="{CheckOptions: {readability-remove-function-body.RemoveOnlyFunctionName: 'foo1, foo2'}}"` Special option for the **remove-function-body** check to only remove the function body of functions named foo1 and foo2. - - - [ ] Filename -`test.c --` The filename. - -## Mutations -You find more details about the different Mutations in the [Mutations](MUTATIONS.md) file. - -## Workflow -First run the check without `-fix --fix-errors` to see where mutations are possible without applying them. Remember the lines where you actually want to apply the mutation. Make a copy of the input file that you will mutate. The run the check again with `-fix --fix-errors` and `-line filter=...` on the copied file to apply only specific mutations and not all at ones. \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp deleted file mode 100644 index 489cb48018..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "RelationalOperatorReplacementCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void RelationalOperatorReplacementCheck::registerMatchers(MatchFinder *Finder) { - Finder->addMatcher(stmt(expr(binaryOperator())).bind("relational-operator-replacement"), this); -} - -// ">=" <-> ">" and "=<" <-> "<" -void RelationalOperatorReplacementCheck::check(const MatchFinder::MatchResult &Result) { - const auto *MatchedDecl = Result.Nodes.getNodeAs("relational-operator-replacement"); - - int OperatorSize; - std::string Replacement; - - if (MatchedDecl->getOpcode() == BO_GE) { - // GE -> G - OperatorSize = 2; - Replacement = ">"; - } else if (MatchedDecl->getOpcode() == BO_GT) { - // G -> GE - OperatorSize = 1; - Replacement = ">="; - } else if (MatchedDecl->getOpcode() == BO_LE) { - // LE -> L - OperatorSize = 2; - Replacement = "<"; - } else if (MatchedDecl->getOpcode() == BO_LT) { - // LE -> L - OperatorSize = 1; - Replacement = "<="; - } else { - return; - } - Replacement += " /* [MUTATION][ROR] Replaced Relational Operator */"; - - // Get locations - SourceLocation Start = MatchedDecl->getOperatorLoc(); - SourceLocation End = MatchedDecl->getOperatorLoc().getLocWithOffset(OperatorSize); - auto Range = CharSourceRange::getCharRange(Start, End); - - diag(Start, "[MUTATION][ROR] Replaced Relational Operator %0") - << MatchedDecl->getOpcodeStr() - << FixItHint::CreateReplacement(Range, Replacement); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h b/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h deleted file mode 100644 index c0aeee12cc..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RelationalOperatorReplacementCheck.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Relational Operator Replacement -class RelationalOperatorReplacementCheck : public ClangTidyCheck { -public: - RelationalOperatorReplacementCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_RELATIONALOPERATORREPLACEMENTCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp deleted file mode 100644 index 5001a5a5de..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "RemoveFunctionBodyCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" -#include - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void RemoveFunctionBodyCheck::registerMatchers(MatchFinder *Finder) { - if (RemoveOnlyFunctionName.empty()) { - Finder->addMatcher(functionDecl(isDefinition()).bind("remove_function_body"), this); - } else { - std::vector names; - std::istringstream iss(RemoveOnlyFunctionName); - for (std::string s; std::getline(iss, s, ','); ) { - // trim leading and trailing whitespace from function name - s.erase(0, s.find_first_not_of(" ")); - s.erase(s.find_last_not_of(" ") + 1); - if (!s.empty()) { - names.push_back(s); - } - } - for (const auto& name : names) { - Finder->addMatcher(functionDecl(hasName(name), isDefinition()).bind("remove_function_body"), this); - } - } -} - -void RemoveFunctionBodyCheck::check(const MatchFinder::MatchResult &Result) { - auto *MatchedDecl = Result.Nodes.getNodeAs("remove_function_body"); - - // Remove the function body - std::string Replacement = " { "; - const auto ReturnType = MatchedDecl->getReturnType(); - if (!ReturnType->isVoidType()) { - Replacement += "return "; - if (ReturnType->isPointerType() || ReturnType->isNullPtrType()) { - Replacement += "0"; - } else if (ReturnType->isIntegralType(*Result.Context) || ReturnType->isCharType()) { - Replacement += "0"; - } else if (ReturnType->isFloatingType()) { - Replacement += "0.0"; - } else if (const RecordType *RT = ReturnType->getAsStructureType()) { - Replacement += "(struct " + RT->getDecl()->getNameAsString() + "){}"; - } else if (const RecordType *RT = ReturnType->getAsUnionType()) { - Replacement += "(union " + RT->getDecl()->getNameAsString() + "){}"; - } else { - Replacement += "/* [TODO]: Add generic return value for " + ReturnType.getAsString() + " */"; - } - Replacement += "; "; - } - Replacement += "/* [MUTATION][RFB] Stripped function of its body */ }"; - - // Get locations - SourceLocation Start = MatchedDecl->getTypeSpecEndLoc().getLocWithOffset(1); - SourceLocation End = MatchedDecl->getBodyRBrace().getLocWithOffset(1); - auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "[MUTATION][RFB] Function %0 has been stripped of its body") - << MatchedDecl - << FixItHint::CreateReplacement(Range, Replacement); -} - -void RemoveFunctionBodyCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { - Options.store(Opts, "RemoveOnlyFunctionName", RemoveOnlyFunctionName); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h deleted file mode 100644 index cfc808448d..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveFunctionBodyCheck.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Removes the body of a function and adds when needed generic return statements. -class RemoveFunctionBodyCheck : public ClangTidyCheck { - const std::string RemoveOnlyFunctionName; - -public: - RemoveFunctionBodyCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context), - RemoveOnlyFunctionName(Options.get("RemoveOnlyFunctionName", "")){} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; - void storeOptions(ClangTidyOptions::OptionMap &Opts) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVEFUNCTIONBODYCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp deleted file mode 100644 index ec76b96fff..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "RemoveThreadCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" -#include - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void RemoveThreadCheck::registerMatchers(MatchFinder *Finder) { - Finder->addMatcher(callExpr(callee(functionDecl(hasName("pthread_create")))).bind("remove-thread"), this); -} - -void RemoveThreadCheck::check(const MatchFinder::MatchResult &Result) { - const auto *MatchedDecl = Result.Nodes.getNodeAs("remove-thread"); - - // Get locations - SourceLocation Start = MatchedDecl->getBeginLoc(); - SourceLocation End = MatchedDecl->getEndLoc().getLocWithOffset(1); - auto Range = CharSourceRange::getCharRange(Start, End); - // Get the arguments - const SourceManager *SM = Result.SourceManager; - const char *StartPtr = SM->getCharacterData(Start); - const char *EndPtr = SM->getCharacterData(End); - size_t Length = EndPtr - StartPtr; - std::string S(StartPtr, Length); - std::vector Arguments; - std::istringstream iss(S); - std::string argument; - while (std::getline(iss, argument, ',')) { - Arguments.push_back(argument); - } - // Get the function name - std::string FunctionName = parseFunctionName(Arguments[2]); - // Get the argument - std::string Argument = parseArgument(Arguments[3]); - // Create the replacement - std::string Replacement = FunctionName + "_wrap(" + Argument + ") /* [MUTATION][RT][FUNCTION_NAME][" + FunctionName + "] Thread creation was substituted with function call */"; - diag(Start, "[MUTATION][RT][FUNCTION_NAME][%0] Thread creation was substituted with function call %0_wrapper") - << FunctionName - << FixItHint::CreateReplacement(Range, Replacement); -} - -std::string RemoveThreadCheck::parseFunctionName(std::string FunctionName) { - // Remove leading and trailing whitespaces - FunctionName.erase(0, FunctionName.find_first_not_of(" \t")); - FunctionName.erase(FunctionName.find_last_not_of(" \t") + 1); - // Remove leading pointers - FunctionName.erase(0, FunctionName.find_first_not_of("*")); - // Remove leading dereference - FunctionName.erase(0, FunctionName.find_first_not_of("&")); - return FunctionName; -} - -std::string RemoveThreadCheck::parseArgument(std::string Argument) { - // Remove leading and trailing whitespaces - Argument.erase(0, Argument.find_first_not_of(" \t")); - Argument.erase(Argument.find_last_not_of(" \t") + 1); - Argument.erase(Argument.find_last_of(')'), 1); - return Argument; -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h deleted file mode 100644 index 71cf8d4ca7..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveThreadCheck.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Remove Thread -class RemoveThreadCheck : public ClangTidyCheck { -public: - RemoveThreadCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -private: - std::string parseFunctionName(std::string FunctionName); - std::string parseArgument(std::string Argument); -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADCHECK_H diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp deleted file mode 100644 index f900586be6..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "RemoveThreadWrapperCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void RemoveThreadWrapperCheck::registerMatchers(MatchFinder *Finder) { - if (WrapFunctionName.empty()) { - exit(-1); - } else { - Finder->addMatcher(functionDecl(hasName(WrapFunctionName), isDefinition()).bind("wrap_function"), this); - } -} - -void RemoveThreadWrapperCheck::check(const MatchFinder::MatchResult &Result) { - auto *MatchedDecl = Result.Nodes.getNodeAs("wrap_function"); - - std::string Replacement = "\nint " + WrapFunctionName + "_wrap(void *arg) {\n" - + "\t/*[MUTATION][RTW] Wrapped function for remove-thread */\n" - + "\t" + WrapFunctionName + "(arg);\n" - + "\treturn 0;\n}"; - - // Get locations - SourceLocation Start = MatchedDecl->getEndLoc().getLocWithOffset(1); - SourceLocation End = MatchedDecl->getEndLoc().getLocWithOffset(1); - auto Range = CharSourceRange::getCharRange(Start, End); - diag(Start, "[MUTATION][RTW] Function %0 has been wrapped") - << MatchedDecl - << FixItHint::CreateReplacement(Range, Replacement); -} - -void RemoveThreadWrapperCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { - Options.store(Opts, "WrapFunctionName", WrapFunctionName); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h b/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h deleted file mode 100644 index 91e1989482..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/RemoveThreadWrapperCheck.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Create wrapper for remove thread check -class RemoveThreadWrapperCheck : public ClangTidyCheck { - const std::string WrapFunctionName; - -public: - RemoveThreadWrapperCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context), - WrapFunctionName(Options.get("WrapFunctionName", "")) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; - void storeOptions(ClangTidyOptions::OptionMap &Opts) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REMOVETHREADWRAPPERCHECK_H \ No newline at end of file diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp deleted file mode 100644 index 98b3b6f35c..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "UnaryOperatorInversionCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -using namespace clang::ast_matchers; - -namespace clang::tidy::readability { - -void UnaryOperatorInversionCheck::registerMatchers(MatchFinder *Finder) { - Finder->addMatcher(stmt(ifStmt()).bind("unary-operator-inversion"), this); -} - -// Replace if(e) with if(!(e)) -void UnaryOperatorInversionCheck::check(const MatchFinder::MatchResult &Result) { - const auto *MatchedDecl = Result.Nodes.getNodeAs("unary-operator-inversion"); - // Get locations - SourceLocation Start = MatchedDecl->getLParenLoc().getLocWithOffset(1); - SourceLocation End = MatchedDecl->getRParenLoc(); - auto Range = CharSourceRange::getCharRange(Start, End); - // Get the current expression in the if statement - const SourceManager *SM = Result.SourceManager; - const char *StartPtr = SM->getCharacterData(Start); - const char *EndPtr = SM->getCharacterData(End); - size_t Length = EndPtr - StartPtr; - std::string Expression(StartPtr, Length); - // Invert the expression with the unary operator - std::string Replacement = "!(" + Expression + ")" + " /* [MUTATION][UOI] Inverted if statement */"; - diag(Start, "[MUTATION][UOI] If Statement %0 was inverted") - << Expression - << FixItHint::CreateReplacement(Range, Replacement); -} - -} // namespace clang::tidy::readability diff --git a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h b/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h deleted file mode 100644 index d56556539e..0000000000 --- a/scripts/incremental-test-generation/clang-mutations/UnaryOperatorInversionCheck.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H - -#include "../ClangTidyCheck.h" - -namespace clang::tidy::readability { - -/// Unary Operator Inversion for if statements -class UnaryOperatorInversionCheck : public ClangTidyCheck { -public: - UnaryOperatorInversionCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -}; - -} // namespace clang::tidy::readability - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_UNARYOPERATORINVERSIONCHECK_H diff --git a/scripts/incremental-test-generation/generators/.gitignore b/scripts/incremental-test-generation/generators/.gitignore deleted file mode 100644 index 834b16bde6..0000000000 --- a/scripts/incremental-test-generation/generators/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -__pycache__ -test diff --git a/scripts/incremental-test-generation/generators/README.md b/scripts/incremental-test-generation/generators/README.md deleted file mode 100644 index 92f0030960..0000000000 --- a/scripts/incremental-test-generation/generators/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Mutation Generators -This folder contains Python scripts that generate mutated programs. The different files and their purpose is shortly described in the following: -You can use for all python scripts the option `-h` to get additional information about the comand line arguments. -

-You can use this content for the `meta.yaml` file you need to pass: -``` -n: 0 -``` - -# generate_mutations.py -This script uses the custom clang-tidy checks to generate mutations on a given program. Informations about the mutations are written to the `meta.yaml` file. -

-More information about the mutations can be found in this file: [Mutations](../../clang-mutations/MUTATIONS.md) - -# generate_ml.py -This script uses gpt-3.5-turbo with the api from openai. It generates mutations on a program by asking it how a previous version of the code could have looked like before some typical code changes were done by developers. Informations about the mutations are written to the `meta.yaml` file. -

-You need to pass a `apikey.yaml` file with the following format: -``` -organisation: (Found at https://platform.openai.com/account/org-settings) -api-key: (Found at https://platform.openai.com/account/api-keys) -``` -You can specify `num_selected_lines` to tell the script how many consecutive lines should be send together with the prompt. To many lines could lead to an error because of a too large request. To few lines may result in a bad mutation because of less context. -

-You can specify `interesting_lines` to guide the selection of the lines send with the request. The selection process works by selecting a random start line out of a set of lines. From the start line on the `num_selected_lines` are selected. When `interesting_lines` equals to `[]` all the lines (`[1, 2, 3, ..., MAX_LINE - NUM_SELECTED_LINES]`) are intresting lines. Alternatively you can pass specific lines (`[1, 42, 99]`). Note that when a line is larger then `(MAX_LINE - NUM_SELECTED_LINES)` it will be set to this value. - -# generate_git.py -This script generates from a git repository per commit a cil file representing the whole project in a single c file. The differences between the commits are used as mutations. For passing a repository you need to specify a shell script containing all the needed information and optionally you can give commands that should be executed before and after the default build commands (`cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON` / `bear -- make`). -

-You can find the template for the shell script with the informations in the file `generate_git_build_USER_INFO_TEMPLATE.sh`: -``` -#!/bin/bash - -############################################################## -####################### USER VARIABLES ####################### -############################################################## -# Variables -git_url="" #e.g.: https://github.com/madler/zlib.git -use_cmake=true # Choose either cmake or make -use_make=false # Choose either cmake or make -path_to_build="" #e.g.: "." (Relative path in repo) - -# Functions before and after build -pre_build_commands() { - : #e.g.: ./configure -} - -post_build_commands() { - : -} -############################################################## -####################### USER VARIABLES ####################### -############################################################## - -# Export variables so they can be used in the main script -export git_url -export use_cmake -export use_make -export path_to_build -export pre_build_commands -export post_build_commands -``` - -The script `generate_git_build.sh` interacts with these user shell scripts and can clone the repositories, build the repositories and provide the build path. -

-For not analyzing all the commits you can specify a start and end commit hash. \ No newline at end of file diff --git a/scripts/incremental-test-generation/generators/generate_git.py b/scripts/incremental-test-generation/generators/generate_git.py deleted file mode 100644 index b06b1a686c..0000000000 --- a/scripts/incremental-test-generation/generators/generate_git.py +++ /dev/null @@ -1,193 +0,0 @@ -import argparse -import os -import subprocess -import sys -from datetime import datetime -from pydriller import Repository -import yaml - -sys.path.insert(0, "..") -from util.util import * - -build_errors = 0 -checkout_errors = 0 -cil_errors = 0 -ids_errors = [] - -def generate_git(goblint_path, temp_dir, meta_path, git_info_sh_path, start_commit, end_commit): - goblint_path = os.path.expanduser(os.path.abspath(goblint_path)) - temp_dir = os.path.expanduser(os.path.abspath(temp_dir)) - temp_repo_dir = os.path.join(temp_dir, 'repo') - meta_path = os.path.expanduser(os.path.abspath(meta_path)) - git_info_sh_path = os.path.expanduser(os.path.abspath(git_info_sh_path)) - - print(SEPERATOR) - print(f'[GIT] Cloning into {temp_repo_dir}') - _clone_repo(git_info_sh_path, temp_repo_dir) - build_path = _get_build_path(git_info_sh_path, temp_repo_dir) - print(f'{COLOR_GREEN}[GIT] Cloning finished{COLOR_RESET}') - - if not os.path.exists(temp_repo_dir): - os.mkdir(temp_repo_dir) - - def get_commit_traverser(): - all_commits = list(Repository(build_path).traverse_commits()) - if start_commit == None: - start_index = 0 - else: - start_index = next((i for i, commit in enumerate(all_commits) if commit.hash == start_commit), None) - if end_commit == None: - end_index = len(all_commits) - 1 - else: - end_index = next((i for i, commit in enumerate(all_commits) if commit.hash == end_commit), None) - - if start_index is None or end_index is None: - raise ValueError("One or both commit hashes not found in the repository") - - return all_commits[start_index:end_index + 1] - - - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - index: int = yaml_data[META_N] - - num_of_commits = sum(1 for _ in get_commit_traverser()) - print(SEPERATOR) - print(f'[GIT] Start traversing {num_of_commits} commits. Including checkout, build and cil generation. This may take a while...') - if num_of_commits <= 2: - print(f'{COLOR_RED}You must traverse at least two commits to generate a test!') - sys.exit(-1) - t = 0 - #last_failed = False - for commit in get_commit_traverser(): - t += 1 - if commit.merge: - print(f"[GIT][{t}/{num_of_commits}] {COLOR_YELLOW}Skipping merge commit {commit.hash}{COLOR_RESET}, continuing with the next commit...") - last_failed = True - continue - try: - _checkout(build_path, meta_path, commit.hash) - _build_repo(git_info_sh_path, temp_repo_dir, meta_path, commit.hash) - new_path = os.path.join(temp_dir, f"p_{index}.c") - _create_cil_file(goblint_path, build_path, new_path, meta_path, commit.hash) - - #if t % 10 == 0 or last_failed: - #last_failed = False - #print(f"{COLOR_GREEN}[GIT][{t}/{num_of_commits}] Written cil file with index {index}{COLOR_RESET}, continuing with the next commits...") - print(f"{COLOR_GREEN}[{t}/{num_of_commits}] Written cil file with index [{index}] for commit {commit.hash}{COLOR_RESET}, continuing with the next commits...") - _write_meta_data(meta_path, commit.hash, index) - index += 1 - except Exception as e: - #last_failed = True - print(f"{COLOR_RED}[{t}/{num_of_commits}][FAIL] Generating cil for commit {commit.hash} failed ({e}){COLOR_RESET}, continuing with the next commit...") - print(f"{COLOR_GREEN}[FINISHED] Finished creating cil files for the commits.{COLOR_RESET}") - if build_errors > 0 or checkout_errors > 0 or cil_errors > 0: - print(f"{COLOR_RED}There were the following errors: {build_errors} build errors, {checkout_errors} checkout errors and {cil_errors} cil errors.{COLOR_RESET}") - print(f"{COLOR_RED}You can read the error messages in the {meta_path} file") - print(f"{COLOR_RED}The following commit ids resulted in errors:{COLOR_RESET} {', '.join(ids_errors)}") - return index - 1 - - -def _write_meta_data(meta_path, commit_hash, index): - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data[META_N] = index - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.GIT.value, - META_SUB_TYPE: commit_hash - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - -def _write_meta_data_failure(meta_path, commit_hash, stdout_msg, stderr_msg): - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data.setdefault(META_FAILURES, {})[commit_hash] = { - META_FAILURES_STD_OUT: stdout_msg, - META_FAILURES_STD_ERR: stderr_msg - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - -def _clone_repo(git_info_sh_path, temp_repo_path): - command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--clone"] - result = subprocess.run(command, text=True, capture_output=True) - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}Could not clone!{COLOR_RESET}") - sys.exit(-1) - -def _get_build_path(git_info_sh_path, temp_repo_path): - command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--path"] - result = subprocess.run(command, text=True, capture_output=True) - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}Could not get build path!{COLOR_RESET}") - sys.exit(-1) - build_path = os.path.normpath(result.stdout.strip()) - return build_path - -def _build_repo(git_info_sh_path, temp_repo_path, meta_path, commit_hash): - global build_errors - command = ["generators/generate_git_build.sh", git_info_sh_path, temp_repo_path, "--build"] - result = subprocess.run(command, text=True, capture_output=True) - if result.returncode != 0: - build_errors += 1 - ids_errors.append(commit_hash) - _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) - raise Exception("Could not build repo!") - -def _checkout(build_path, meta_path, commit_hash): - global checkout_errors - # Clean untracked files - clean_command = ['git', '-C', build_path, 'clean', '-f'] - clean_result = subprocess.run(clean_command, text=True, capture_output=True) - if clean_result.returncode != 0: - checkout_errors += 1 - ids_errors.append(commit_hash) - _write_meta_data_failure(meta_path, commit_hash, clean_result.stdout, clean_result.stderr) - raise Exception("Could not clean untracked files!") - - # Stash any uncommitted changes - stash_command = ['git', '-C', build_path, 'stash'] - stash_result = subprocess.run(stash_command, text=True, capture_output=True) - if stash_result.returncode != 0: - checkout_errors += 1 - ids_errors.append(commit_hash) - _write_meta_data_failure(meta_path, commit_hash, stash_result.stdout, stash_result.stderr) - raise Exception("Could not stash changes!") - - # Checkout commit - command = ['git', '-C', build_path, 'checkout', commit_hash] - result = subprocess.run(command, text=True, capture_output=True) - if result.returncode != 0: - checkout_errors += 1 - ids_errors.append(commit_hash) - _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) - raise Exception("Could not checkout repo!") - -def _create_cil_file(goblint_path, build_path, output_path, meta_path, commit_hash): - global cil_errors - result = subprocess.run([goblint_path, '--set', 'justcil', 'true', '--set', 'cil.merge.inlines', 'false', build_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if result.returncode != 0: - cil_errors += 1 - ids_errors.append(commit_hash) - _write_meta_data_failure(meta_path, commit_hash, result.stdout, result.stderr) - raise Exception("Error creating cil!") - with open(output_path, 'w') as f: - f.write(result.stdout.decode()) - -if __name__ == "__main__": - parser = argparse.ArgumentParser("Script for generating cil program files from commits") - parser.add_argument("goblint_path", help="Path to Goblint directory") - parser.add_argument("temp_dir", help="Path to the temporary directory") - parser.add_argument("meta_path", help="Path to the meta directory") - parser.add_argument("git_info_sh_path", help="Path to the Git information shell script") - parser.add_argument("--start_commit", help="Hash id of the first commit to consider", default=None) - parser.add_argument("--end_commit", help="Hash id of the last commit to consider", default=None) - - args = parser.parse_args() - - generate_git(args.goblint_path, args.temp_dir, args.meta_path, args.git_info_sh_path, args.start_commit, args.end_commit) diff --git a/scripts/incremental-test-generation/generators/generate_git_build.sh b/scripts/incremental-test-generation/generators/generate_git_build.sh deleted file mode 100755 index 0f5df72a9e..0000000000 --- a/scripts/incremental-test-generation/generators/generate_git_build.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Source the input script from the provided path -if [[ -n $1 ]]; then - source $1 -else - echo "Please give the following arguments: build_info_path output_path [--clone][--build][--path]" - echo "" - echo "The build_info sh file should look like this:" - cat generate_git_build_USER_INFO_TEMPLATE.sh - echo "" - exit 1 -fi - -# Get the output path from the provided path -if [[ -n $2 ]]; then - output_path="$(pwd)/$2" -else - echo "Please provide the relative path to the output as the second argument." - exit 1 -fi - -# Exit if any command fails -set -e - -# Check bool variables -if { $use_cmake && $use_make ; } || { ! $use_cmake && ! $use_make ; }; then - echo "Error: Either cmake or make should be used, but not both or neither." - exit 1 -fi - -# Get the name of the repo -repo_name=$(basename "$git_url" .git) - -case "$3" in - "--path") - # Export path for Mutation Generator - echo "$output_path/$repo_name/$path_to_build" - exit 0 - ;; - "--clone") - # Clone repo - rm -rf "$output_path/$repo_name" - git clone $git_url "$output_path/$repo_name" - ;; - "--build") - # Build repo - cd "$output_path/$repo_name/$path_to_build" - pre_build_commands - - if $use_cmake; then - cmake "$output_path/$repo_name/$path_to_build" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - elif $use_make; then - bear -- make - fi - - post_build_commands - ;; - *) - echo "Please provide one of the options [--clone][--build][--path]" - exit 1 - ;; -esac \ No newline at end of file diff --git a/scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh b/scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh deleted file mode 100644 index b579a32be1..0000000000 --- a/scripts/incremental-test-generation/generators/generate_git_build_USER_INFO_TEMPLATE.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -############################################################## -####################### USER VARIABLES ####################### -############################################################## -# Variables -git_url="" #e.g.: https://github.com/madler/zlib.git -use_cmake=true # Choose either cmake or make -use_make=false # Choose either cmake or make -path_to_build="" #e.g.: "." (Relative path in repo) - -# Functions before and after build -pre_build_commands() { - : #e.g.: ./configure -} - -post_build_commands() { - : -} -############################################################## -####################### USER VARIABLES ####################### -############################################################## - -# Export variables so they can be used in the main script -export git_url -export use_cmake -export use_make -export path_to_build -export pre_build_commands -export post_build_commands \ No newline at end of file diff --git a/scripts/incremental-test-generation/generators/generate_ml.py b/scripts/incremental-test-generation/generators/generate_ml.py deleted file mode 100644 index f9d3a2db90..0000000000 --- a/scripts/incremental-test-generation/generators/generate_ml.py +++ /dev/null @@ -1,242 +0,0 @@ -import argparse -import ast -import os -import random -import sys -import time -import openai -import yaml -from concurrent.futures import ThreadPoolExecutor -from multiprocessing import Lock - -sys.path.insert(0, "..") -from util.util import * - -SEPERATOR_EXPLANATION_START = 'EXPLANATION>' -SEPERATOR_EXPLANATION_END = ' 0 else '')) - - return index + ml_count - -def _iterative_mutation_generation(program_path, meta_path, interesting_lines, ml_16k, num_selected_lines, max_line, index, lock): - try: - time.sleep((index * 50)/1000) # Sleep depending on index to print the start messages in the right order - new_path = make_program_copy(program_path, index) - (explanation, selected_lines) = _apply_mutation(new_path, interesting_lines, ml_16k, num_selected_lines, max_line, index) - _write_meta_data(meta_path, selected_lines, explanation, index, lock) - except Exception as e: - print(f"{COLOR_RED}[{index}] Error for request {index}:{COLOR_RESET} {e}") - _write_meta_data(meta_path, [], '', index, lock, exception=e) - return index - -def _apply_mutation(new_path, interesting_lines, ml_16k, num_selected_lines, max_line, index): - # Get the original lines - with open(new_path, "r") as file: - lines = file.readlines() - - # Get code snippet - selected_lines = _select_lines(interesting_lines, num_selected_lines, max_line) - snippet = '' - for i in selected_lines: - snippet += lines[i] - - print(f"[{index}][{Generate_Type.ML.value}][REQUEST] Make request for lines [{selected_lines.start}, {selected_lines.stop}]. This may take a few seconds...") - - # Get response from gpt - response = _make_gpt_request(snippet, ml_16k) - - # Extract Explanation - explanation_start = response.find(SEPERATOR_EXPLANATION_START) + len(SEPERATOR_EXPLANATION_START) - explanation_end = response.find(SEPERATOR_EXPLANATION_END) - explanation = response[explanation_start:explanation_end].strip() - - # Extract Code lines - code_start = response.find(SEPERATOR_CODE_START) + len(SEPERATOR_CODE_START) - code_end = response.find(SEPERATOR_CODE_END) - code = response[code_start:code_end].strip() - new_code_lines = code.splitlines() - new_code_lines = [line for line in new_code_lines if line != '```' and line != '```c' and line != '``'] - - # Comment out the original lines - for i in selected_lines: - lines[i] = '// ' + lines[i] - - # Add start marker - lines.insert(selected_lines[0], f'//[ML][START] {explanation.replace("/n", "")}\n') - - # Insert new lines of code - for i, new_line in enumerate(new_code_lines, start=selected_lines[0] + 1): - lines.insert(i, new_line + '\n') - - # Add end marker - lines.insert(selected_lines[0] + len(new_code_lines) + 1, '//[ML][END]\n') - - # Write the updated lines back to the file - with open(new_path, "w") as file: - file.writelines(lines) - - explanation_lines = explanation.splitlines() - limited_explanation = "\n".join(explanation_lines[:4]) - print(f'{COLOR_GREEN}[{index}] Finished request:{COLOR_RESET} {limited_explanation}') - - return (explanation, selected_lines) - - -def _write_meta_data(meta_path, selected_lines, explanation, index, lock, exception=None): - lock.acquire() - global error_counter - try: - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - if exception == None: - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.ML.value, - META_SUB_TYPE: explanation, - META_LINES: f'[{selected_lines.start}, {selected_lines.stop}]' - } - else: - error_counter += 1 - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.ML.value, - META_EXCEPTION: str(exception) - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - finally: - lock.release() - -def _make_gpt_request(snippet, ml_16k): - prompt = f''' - You are a developer for C helping me with my following question. I want to understand the typical process of code evolution by looking at how developers make changes over time for testing an incremental analysis of the static c analyzer Goblint. - - The following mutations are already generated by me. So please do not generate programs that can be generated by this mutations: Removal of function bodies, Inversion of if statements, Switching <= with < and >= with >, Replacing constants unequal 0 with 1, Replace pthread calls with function calls, Switching && with ||. Please do not consider these mutations as examples how your code changes should look like. Just try to prevent doing things that could be done with these mutations. - - Below is an snippet from a C file which represents a part of the finished program. My question is how a previous version of this code could have looked like before some typical code changes done by developers. Can you generate me such a previous version? - - The code you generate should be a self-contained snippet that could directly replace the provided excerpt in the original, complete program. It should preserve the overall functionality of the program and must not cause any compilation errors when reintegrated into the larger code base. Please consider the dependencies and interactions with other parts of the program when generating the previous version of the code. Your generated code should be able to interact correctly with the rest of the program just like the original excerpt does. You do not have to add import statements or function declarations or closing brackets when these are cut off in the snippet, but when they are in the snippet you need to add them to preserve the whole program. - - Use these keywords (\"{SEPERATOR_EXPLANATION_START}\", \"{SEPERATOR_EXPLANATION_END}\", \"{SEPERATOR_CODE_START}\", \"{SEPERATOR_CODE_END}\") to structure you answer. You answer should have the following structure for better identifying the different parts of the response: {SEPERATOR_EXPLANATION_START} (Explain what you have changed in one or two sentences) {SEPERATOR_EXPLANATION_END} {SEPERATOR_CODE_START} (the previous version of the code) {SEPERATOR_CODE_END} - - ```c - {snippet} - ``` - ''' - - if ml_16k: - model = "gpt-3.5-turbo-16k" - else: - model = "gpt-3.5-turbo" - - response = openai.ChatCompletion.create( - model=model, - n = 1, - messages=[ - {"role": "user", "content": prompt}, - ] - ).choices[0].message['content'] - - return response - -def _reformat_interesting_lines(num_selected_lines, interesting_lines, max_line): - for i in range(len(interesting_lines)): - interesting_lines[i] = int(interesting_lines[i]) - # Adjust for line array starting with 0 but in input first line is 1 - interesting_lines[i] -= 1 - # When line + num_selected_lines is greater then max_line correct the line downwards - if interesting_lines[i] + num_selected_lines > (max_line): - interesting_lines[i] = (max_line) - num_selected_lines - return interesting_lines - -def _select_lines(interesting_lines, num_selected_lines, max_line): - if interesting_lines == []: - selected_line = random.randint(0, (max_line) - num_selected_lines) - else: - selected_line = random.choice(interesting_lines) - return range(selected_line, selected_line + num_selected_lines) - -def validate_interesting_lines(intersting_lines_string: str, program_path): - if program_path != None: - with open(program_path, "r") as file: - max_line = len(file.readlines()) - else: - max_line = None - - try: - intersting_lines = ast.literal_eval(intersting_lines_string) - except SyntaxError: - print(f"{COLOR_RED}The format \"{intersting_lines_string}\" is incorrect! Please use a format like this: \"[1, 42, 99]\"{COLOR_RESET}") - return None - except Exception as e: - print(f"{COLOR_RED}An unexpected error occurred reading the input \"{intersting_lines_string}\":{COLOR_RESET} {e}") - return None - - if max_line != None: - for line in intersting_lines: - if line <= 0: - print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be greater zero!{COLOR_RESET}") - return None - if line > max_line: - print(f"{COLOR_RED}All lines in \"{intersting_lines_string}\" should be below the maximum line {max_line}!{COLOR_RESET}") - return None - - return intersting_lines - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate mutations with ML.") - parser.add_argument("program", help="Path to the C program") - parser.add_argument("apikey", help="Path to the api key") - parser.add_argument("meta_path", help="Path to the meta_file") - parser.add_argument("ml_count", help="How many different programs should be generated with ML") - parser.add_argument("num_selected_lines", help="How many lines to consider") - parser.add_argument("interesting_lines", help="Which parts are interesting (All: [], Specify: \"[1, 42, 99]\")") - parser.add_argument('-m16', '--model-16k', action='store_true', help='Run with the 16k model instead of the 4k') - - args = parser.parse_args() - - interesting_lines = validate_interesting_lines(args.interesting_lines, args.program) - if interesting_lines == None: - print(f'{COLOR_RED}Stopped program execution{COLOR_RESET}') - sys.exit(-1) - - generate_ml(args.program, args.apikey, args.meta_path, int(args.ml_count), int(args.num_selected_lines), interesting_lines, args.model_16k) \ No newline at end of file diff --git a/scripts/incremental-test-generation/generators/generate_mutations.py b/scripts/incremental-test-generation/generators/generate_mutations.py deleted file mode 100644 index fa6cfba434..0000000000 --- a/scripts/incremental-test-generation/generators/generate_mutations.py +++ /dev/null @@ -1,207 +0,0 @@ -# Generate all possible mutations of a program and -# Assume there is a meta.yaml file with content "n: >=0" - -import argparse -import json -import os -import re -import subprocess -import sys -import yaml -sys.path.insert(0, "..") -from util.util import * - -def generate_mutations(program_path, clang_tidy_path, meta_path, mutations): - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - index: int = yaml_data[META_N] - - if mutations.rfb: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rfb_s, index) - if mutations.uoi: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.uoi_s, index) - if mutations.ror: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.ror_s, index) - if mutations.cr: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.cr_s, index) - if mutations.rt: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.rt_s, index) - if mutations.lcr: - index = _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutations.lcr_s, index) - - return index - -def _iterative_mutation_generation(program_path, clang_tidy_path, meta_path, mutation_name, index): - print(SEPERATOR) - print(f"[{Generate_Type.MUTATION.value}] {mutation_name}") - lineGroups = _get_line_groups(clang_tidy_path, mutation_name, program_path, index) - for lines in lineGroups: - index += 1 - new_path = make_program_copy(program_path, index) - if mutation_name == Mutations().rt_s: - # When Remove Thread create wrapper an then apply the mutations - if len(lines) != 1: - # Needed to prevent conflicts on generating wrappers - print(f"{COLOR_RED}ERROR When applying remove_thread there always should be exactly one line{COLOR_RESET}") - function_name = _get_thread_function_name(clang_tidy_path, lines, new_path, index) - _wrap_thread_function(clang_tidy_path, new_path, function_name, index) - _apply_mutation(clang_tidy_path, mutation_name, lines, new_path, index) - _write_meta_data(meta_path, index, mutation_name, lines) - return index - -def _get_line_groups(clang_tidy_path, mutation_name, program_path, index): - command = [ - clang_tidy_path, - "-checks=-*,readability-" + mutation_name, - program_path, - "--" - ] - - result = subprocess.run(command, text=True, capture_output=True) - print(f"[MUTATION][CHECK] Check mutation {mutation_name}") - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") - sys.exit(-1) - - line_groups = [] - pattern = r":(\d+):.*\[readability-" + mutation_name + r"\]" - macro_pattern = r"\[MACRO\]\[(.*?)\]" - macro_lines = {} - - for line in result.stdout.splitlines(): - match = re.search(pattern, line) - if match: - macro_match = re.search(macro_pattern, line) - if macro_match: - macro_name = macro_match.group(1) - line_number = int(match.group(1)) - if macro_name not in macro_lines: - macro_lines[macro_name] = [line_number] - else: - macro_lines[macro_name].append(line_number) - else: - line_groups.append([int(match.group(1))]) - - for macro_name, lines in macro_lines.items(): - line_groups.append(lines) - - # Remove duplicate line groups - line_groups = [list(x) for x in set(tuple(x) for x in line_groups)] - - print(f"[MUTATION][CHECK RESULT] Mutation {mutation_name} can be applied to lines {line_groups}") - return sorted(line_groups, key=lambda x: x[0]) - -def _apply_mutation(clang_tidy_path, mutation_name, lines, program_path, index): - lines_mapped = [[x,x] for x in lines] - line_filter = [{"name": program_path, "lines": lines_mapped}] - line_filter_json = json.dumps(line_filter) - command = [ - clang_tidy_path, - "-checks=-*,readability-" + mutation_name, - "-fix", - "--fix-errors", - "-line-filter=" + line_filter_json, - program_path, - "--" - ] - - result = subprocess.run(command, text=True, capture_output=True) - if result.returncode == 0: - print(f"{COLOR_GREEN}[{index}] Finished mutation:{COLOR_RESET} {mutation_name} on lines {lines}") - else: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") - sys.exit(-1) - -def _get_thread_function_name(clang_tidy_path, lines, program_path, index): - lines_mapped = [[x,x] for x in lines] - line_filter = [{"name": program_path, "lines": lines_mapped}] - line_filter_json = json.dumps(line_filter) - command = [ - clang_tidy_path, - "-checks=-*,readability-" + Mutations().rt_s, - "-line-filter=" + line_filter_json, - program_path, - "--" - ] - result = subprocess.run(command, text=True, capture_output=True) - print(f"[{index}][WRAP] Check function name for wrapping thread function") - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") - sys.exit(-1) - - function_name_pattern = r"\[FUNCTION_NAME\]\[(.*?)\]" - function_name = None - - for line in result.stdout.splitlines(): - function_name_match = re.search(function_name_pattern, line) - if function_name_match: - function_name = function_name_match.group(1) - break - - print(f"[{index}][WRAP RESULT] Found the thread function name {function_name}") - return function_name - -def _wrap_thread_function(clang_tidy_path, program_path, function_name, index): - if function_name == None: - print(f"{COLOR_YELLOW}[{index}][WRAP FIX] No function name was provided. Hope the program will compile without wrapping{COLOR_RESET}") - return - - check_options = {"CheckOptions": {"readability-remove-thread-wrapper.WrapFunctionName": function_name}} - check_options_json = json.dumps(check_options) - command = [ - clang_tidy_path, - "-checks=-*,readability-remove-thread-wrapper", - "-config=" + check_options_json, - "-fix", - program_path, - "--" - ] - result = subprocess.run(command, text=True, capture_output=True) - print(f"[{index}][WRAP FIX] Apply the wrapping of {function_name}") - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - print(f"{COLOR_RED}ERROR Running Clang{COLOR_RESET}") - sys.exit(-1) - -def _write_meta_data(meta_path, index, mutation_name, lines): - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data[META_N] = index - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.MUTATION.value, - META_SUB_TYPE: mutation_name, - META_LINES: lines - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - -def add_mutation_options(parser): - parser.add_argument("-rfb", "--remove-function-body", action="store_true", help="Option for \"remove function body\" mutation") - parser.add_argument("-uoi", "--unary-operator-inversion", action="store_true", help="Option for \"unary operator inversion\" mutation") - parser.add_argument("-ror", "--relational-operator-replacement", action="store_true", help="Option for \"relational operator replacement\" mutation") - parser.add_argument("-cr", "--constant-replacement", action="store_true", help="Option for \"constant replacement\" mutation") - parser.add_argument("-rt", "--remove-thread", action="store_true", help="Option for \"remove thread\" mutation") - parser.add_argument("-lcr", "--logical-connector-replacement", action="store_true", help="Option for \"logical connector replacement\" mutation") - -def get_mutations_from_args(args): - return Mutations(args.remove_function_body, args.unary_operator_inversion, - args.relational_operator_replacement, args.constant_replacement, - args.remove_thread, args.logical_connector_replacement) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate all possible mutations of a program.") - parser.add_argument("program", help="Path to the C program") - parser.add_argument("clang_tidy", help="Path to the modified clang-tidy executable") - parser.add_argument("meta", help="Path to the meta data file") - add_mutation_options(parser) - - args = parser.parse_args() - mutations = get_mutations_from_args(args) - generate_mutations(args.program, args.clang_tidy, args.meta, mutations) diff --git a/scripts/incremental-test-generation/sample-files/.gitignore b/scripts/incremental-test-generation/sample-files/.gitignore deleted file mode 100644 index 530234efac..0000000000 --- a/scripts/incremental-test-generation/sample-files/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**/ diff --git a/scripts/incremental-test-generation/sample-files/REPOSITORIES.md b/scripts/incremental-test-generation/sample-files/REPOSITORIES.md deleted file mode 100644 index c3ecdad6e2..0000000000 --- a/scripts/incremental-test-generation/sample-files/REPOSITORIES.md +++ /dev/null @@ -1,3 +0,0 @@ -# CMake Repositories -## zlib -`git clone https://github.com/madler/zlib.git` \ No newline at end of file diff --git a/scripts/incremental-test-generation/sample-files/comparisons.c b/scripts/incremental-test-generation/sample-files/comparisons.c deleted file mode 100644 index ad5cd2cb77..0000000000 --- a/scripts/incremental-test-generation/sample-files/comparisons.c +++ /dev/null @@ -1,40 +0,0 @@ -#include - -int main() { - int a = 10; - int b = 20; - - if (a < b) { - printf("a is less than b\n"); - } - - if (a <= b) { - printf("a is less than or equal to b\n"); - } - - if (a > b) { - printf("a is greater than b\n"); - } - - if (a >= b) { - printf("a is greater than or equal to b\n"); - } - - if (a == b) { - printf("a is equal to b\n"); - } - - if (a != b) { - printf("a is not equal to b\n"); - } - - if (a && b) { - printf("Both a and b are non-zero\n"); - } - - if (a || b) { - printf("Either a or b is non-zero\n"); - } - - return 0; -} diff --git a/scripts/incremental-test-generation/sample-files/constants.c b/scripts/incremental-test-generation/sample-files/constants.c deleted file mode 100644 index c092c1ebeb..0000000000 --- a/scripts/incremental-test-generation/sample-files/constants.c +++ /dev/null @@ -1,11 +0,0 @@ -#define MY_MACRO_5 5 -#define MY_MACRO_0 0 - -int main() { - int a = 42; - int b = 0; - int c = MY_MACRO_5; - int d = MY_MACRO_0; - - return 0; -} \ No newline at end of file diff --git a/scripts/incremental-test-generation/sample-files/functions.c b/scripts/incremental-test-generation/sample-files/functions.c deleted file mode 100644 index c96a2eec9e..0000000000 --- a/scripts/incremental-test-generation/sample-files/functions.c +++ /dev/null @@ -1,40 +0,0 @@ -#include - -// Function to calculate the sum of two numbers -int sum(int a, int b) { - return a + b; -} - -// Function to calculate the difference of two numbers -int difference(int a, int b) { - return a - b; -} - -// Function to calculate the product of two numbers -int product(int a, int b) { - return a * b; -} - -// Function to calculate the quotient of two numbers -float divide(int a, int b) { - if (b != 0) { - return (float)a / b; - } else { - printf("Error: Division by zero is not allowed.\n"); - return 0; - } -} - -int main() { - int num1, num2; - - printf("Enter two numbers: "); - scanf("%d %d", &num1, &num2); - - printf("Sum: %d\n", sum(num1, num2)); - printf("Difference: %d\n", difference(num1, num2)); - printf("Product: %d\n", product(num1, num2)); - printf("Quotient: %.2f\n", divide(num1, num2)); - - return 0; -} diff --git a/scripts/incremental-test-generation/sample-files/large.c b/scripts/incremental-test-generation/sample-files/large.c deleted file mode 100644 index b474747866..0000000000 --- a/scripts/incremental-test-generation/sample-files/large.c +++ /dev/null @@ -1,762 +0,0 @@ -#include -#include -#include - -int main() { - int x; - srand(time(NULL)); - x = rand() % 250; // Generiere eine zufällige Zahl zwischen 0 und 249 - - printf("%d\n", x); - - if (x == 0) { - printf("x ist 0\n"); - } - if (x == 1) { - printf("x ist 1\n"); - } - if (x == 2) { - printf("x ist 2\n"); - } - if (x == 3) { - printf("x ist 3\n"); - } - if (x == 4) { - printf("x ist 4\n"); - } - if (x == 5) { - printf("x ist 5\n"); - } - if (x == 6) { - printf("x ist 6\n"); - } - if (x == 7) { - printf("x ist 7\n"); - }if (x == 8) { - printf("x ist 8\n"); - } - if (x == 9) { - printf("x ist 9\n"); - } - if (x == 10) { - printf("x ist 10\n"); - } - if (x == 11) { - printf("x ist 11\n"); - } - if (x == 12) { - printf("x ist 12\n"); - } - if (x == 13) { - printf("x ist 13\n"); - } - if (x == 14) { - printf("x ist 14\n"); - } - if (x == 15) { - printf("x ist 15\n"); - } - if (x == 16) { - printf("x ist 16\n"); - } - if (x == 17) { - printf("x ist 17\n"); - } - if (x == 18) { - printf("x ist 18\n"); - } - if (x == 19) { - printf("x ist 19\n"); - } - if (x == 20) { - printf("x ist 20\n"); - } - if (x == 21) { - printf("x ist 21\n"); - } - if (x == 22) { - printf("x ist 22\n"); - } - if (x == 23) { - printf("x ist 23\n"); - } - if (x == 24) { - printf("x ist 24\n"); - } - if (x == 25) { - printf("x ist 25\n"); - } - if (x == 26) { - printf("x ist 26\n"); - } - if (x == 27) { - printf("x ist 27\n"); - } - if (x == 28) { - printf("x ist 28\n"); - } - if (x == 29) { - printf("x ist 29\n"); - } - if (x == 30) { - printf("x ist 30\n"); - } - if (x == 31) { - printf("x ist 31\n"); - } - if (x == 32) { - printf("x ist 32\n"); - } - if (x == 33) { - printf("x ist 33\n"); - } - if (x == 34) { - printf("x ist 34\n"); - } - if (x == 35) { - printf("x ist 35\n"); - } - if (x == 36) { - printf("x ist 36\n"); - } - if (x == 37) { - printf("x ist 37\n"); - } - if (x == 38) { - printf("x ist 38\n"); - } - if (x == 39) { - printf("x ist 39\n"); - } - if (x == 40) { - printf("x ist 40\n"); - } - if (x == 41) { - printf("x ist 41\n"); - } - if (x == 42) { - printf("x ist 42\n"); - } - if (x == 43) { - printf("x ist 43\n"); - } - if (x == 44) { - printf("x ist 44\n"); - } - if (x == 45) { - printf("x ist 45\n"); - } - if (x == 46) { - printf("x ist 46\n"); - } - if (x == 47) { - printf("x ist 47\n"); - } - if (x == 48) { - printf("x ist 48\n"); - } - if (x == 49) { - printf("x ist 49\n"); - } - if (x == 50) { - printf("x ist 50\n"); - } - if (x == 51) { - printf("x ist 51\n"); - } - if (x == 52) { - printf("x ist 52\n"); - } - if (x == 53) { - printf("x ist 53\n"); - } - if (x == 54) { - printf("x ist 54\n"); - } - if (x == 55) { - printf("x ist 55\n"); - } - if (x == 56) { - printf("x ist 56\n"); - } - if (x == 57) { - printf("x ist 57\n"); - } - if (x == 58) { - printf("x ist 58\n"); - } - if (x == 59) { - printf("x ist 59\n"); - } - if (x == 60) { - printf("x ist 60\n"); - } - if (x == 61) { - printf("x ist 61\n"); - } - if (x == 62) { - printf("x ist 62\n"); - } - if (x == 63) { - printf("x ist 63\n"); - } - if (x == 64) { - printf("x ist 64\n"); - } - if (x == 65) { - printf("x ist 65\n"); - } - if (x == 66) { - printf("x ist 66\n"); - } - if (x == 67) { - printf("x ist 67\n"); - } - if (x == 68) { - printf("x ist 68\n"); - } - if (x == 69) { - printf("x ist 69\n"); - } - if (x == 70) { - printf("x ist 70\n"); - } - if (x == 71) { - printf("x ist 71\n"); - } - if (x == 72) { - printf("x ist 72\n"); - } - if (x == 73) { - printf("x ist 73\n"); - } - if (x == 74) { - printf("x ist 74\n"); - } - if (x == 75) { - printf("x ist 75\n"); - } - if (x == 76) { - printf("x ist 76\n"); - } - if (x == 77) { - printf("x ist 77\n"); - } - if (x == 78) { - printf("xist 78\n"); - } - if (x == 79) { - printf("x ist 79\n"); - } - if (x == 80) { - printf("x ist 80\n"); - } - if (x == 81) { - printf("x ist 81\n"); - } - if (x == 82) { - printf("x ist 82\n"); - } - if (x == 83) { - printf("x ist 83\n"); - } - if (x == 84) { - printf("x ist 84\n"); - } - if (x == 85) { - printf("x ist 85\n"); - } - if (x == 86) { - printf("x ist 86\n"); - } - if (x == 87) { - printf("x ist 87\n"); - } - if (x == 88) { - printf("x ist 88\n"); - } - if (x == 89) { - printf("x ist 89\n"); - } - if (x == 90) { - printf("x ist 90\n"); - } - if (x == 91) { - printf("x ist 91\n"); - } - if (x == 92) { - printf("x ist 92\n"); - } - if (x == 93) { - printf("x ist 93\n"); - } - if (x == 94) { - printf("x ist 94\n"); - } - if (x == 95) { - printf("x ist 95\n"); - } - if (x == 96) { - printf("x ist 96\n"); - } - if (x == 97) { - printf("x ist 97\n"); - } - if (x == 98) { - printf("x ist 98\n"); - } - if (x == 99) { - printf("x ist 99\n"); - } - if (x == 100) { - printf("x ist 100\n"); - } - if (x == 101) { - printf("x ist 101\n"); - } - if (x == 102) { - printf("x ist 102\n"); - } - if (x == 103) { - printf("x ist 103\n"); - } - if (x == 104) { - printf("x ist 104\n"); - } - if (x == 105) { - printf("x ist 105\n"); - } - if (x == 106) { - printf("x ist 106\n"); - } - if (x == 107) { - printf("x ist 107\n"); - } - if (x == 108) { - printf("x ist 108\n"); - } - if (x == 109) { - printf("x ist 109\n"); - } - if (x == 110) { - printf("x ist 110\n"); - } - if (x == 111) { - printf("x ist 111\n"); - } - if (x == 112) { - printf("x ist 112\n"); - } - if (x == 113) { - printf("x ist 113\n"); - } - if (x == 114) { - printf("x ist 114\n"); - } - if (x == 115) { - printf("x ist 115\n"); - } - if (x == 116) { - printf("x ist 116\n"); - } - if (x == 117) { - printf("x ist 117\n"); - } - if (x == 118) { - printf("x ist 118\n"); - } - if (x == 119) { - printf("x ist 119\n"); - } - if (x == 120) { - printf("x ist 120\n"); - } - if (x == 121) { - printf("x ist 121\n"); - } - if (x == 122) { - printf("x ist 122\n"); - } - if (x == 123) { - printf("x ist 123\n"); - } - if (x == 124) { - printf("x ist 124\n"); - } - if (x == 125) { - printf("x ist 125\n"); - } - if (x == 126) { - printf("x ist 126\n"); - } - if (x == 127) { - printf("x ist 127\n"); - } - if (x == 128) { - printf("x ist 128\n"); - } - if (x == 129) { - printf("x ist 129\n"); - } - if (x == 130) { - printf("x ist 130\n"); - } - if (x == 131) { - printf("x ist 131\n"); - } - if (x == 132) { - printf("x ist 132\n"); - } - if (x == 133) { - printf("x ist 133\n"); - } - if (x == 134) { - printf("x ist 134\n"); - } - if (x == 135) { - printf("x ist 135\n"); - } - if (x == 136) { - printf("x ist 136\n"); - } - if (x == 137) { - printf("x ist 137\n"); - } - if (x == 138) { - printf("x ist 138\n"); - } - if (x == 139) { - printf("x ist 139\n"); - } - if (x == 140) { - printf("x ist 140\n"); - } - if (x == 141) { - printf("x ist 141\n"); - } - if (x == 142) { - printf("x ist 142\n"); - } - if (x == 143) { - printf("x ist 143\n"); - } - if (x == 144) { - printf("x ist 144\n"); - } - if (x == 145) { - printf("x ist 145\n"); - } - if (x == 146) { - printf("x ist 146\n"); - } - if (x == 147) { - printf("x ist 147\n"); - } - if (x == 148) { - printf("x ist 148\n"); - } - if (x == 149) { - printf("x ist 149\n"); - } - if (x == 150) { - printf("x ist 150\n"); - } - if (x == 151) { - printf("x ist 151\n"); - } - if (x == 152) { - printf("x ist 152\n"); - } - if (x == 153) { - printf("x ist 153\n"); - } - if (x == 154) { - printf("x ist 154\n"); - } - if (x == 155) { - printf("x ist 155\n"); - } - if (x == 156) { - printf("x ist 156\n"); - } - if (x == 157) { - printf("x ist 157\n"); - } - if (x == 158) { - printf("x ist 158\n"); - } - if (x == 159) { - printf("x ist 159\n"); - } - if (x == 160) { - printf("x ist 160\n"); - } - if (x == 161) { - printf("x ist 161\n"); - } - if (x == 162) { - printf("x ist 162\n"); - } - if (x == 163) { - printf("x ist 163\n"); - } - if (x == 164) { - printf("x ist 164\n"); - } - if (x == 165) { - printf("x ist 165\n"); - } - if (x == 166) { - printf("x ist 166\n"); - } - if (x == 167) { - printf("x ist 167\n"); - } - if (x == 168) { - printf("x ist 168\n"); - } - if (x == 169) { - printf("x ist 169\n"); - } - if (x == 170) { - printf("x ist 170\n"); - } - if (x == 171) { - printf("x ist 171\n"); - } - if (x == 172) { - printf("x ist 172\n"); - } - if (x == 173) { - printf("x ist 173\n"); - } - if (x == 174) { - printf("x ist 174\n"); - } - if (x == 175) { - printf("x ist 175\n"); - } - if (x == 176) { - printf("x ist 176\n"); - } - if (x == 177) { - printf("x ist 177\n"); - } - if (x == 178) { - printf("x ist 178\n"); - } - if (x == 179) { - printf("x ist 179\n"); - } - if (x == 180) { - printf("x ist 180\n"); - } - if (x == 181) { - printf("x ist 181\n"); - } - if (x == 182) { - printf("x ist 182\n"); - } - if (x == 183) { - printf("x ist 183\n"); - } - if (x == 184) { - printf("x ist 184\n"); - } - if (x == 185) { - printf("x ist 185\n"); - } - if (x == 186) { - printf("x ist 186\n"); - } - if (x == 187) { - printf("x ist 187\n"); - } - if (x == 188) { - printf("x ist 188\n"); - } - if (x == 189) { - printf("x ist 189\n"); - } - if (x == 190) { - printf("x ist 190\n"); - } - if (x == 191) { - printf("x ist 191\n"); - } - if (x == 192) { - printf("x ist 192\n"); - } - if (x == 193) { - printf("x ist 193\n"); - } - if (x == 194) { - printf("x ist 194\n"); - } - if (x == 195) { - printf("x ist 195\n"); - } - if (x == 196) { - printf("x ist 196\n"); - } - if (x == 197) { - printf("x ist 197\n"); - } - if (x == 198) { - printf("x ist 198\n"); - } - if (x == 199) { - printf("x ist 199\n"); - } - if (x == 200) { - printf("x ist 200\n"); - } - if (x == 201) { - printf("x ist 201\n"); - } - if (x == 202) { - printf("x ist 202\n"); - } - if (x == 203) { - printf("x ist 203\n"); - } - if (x == 204) { - printf("x ist 204\n"); - } - if (x == 205) { - printf("x ist 205\n"); - } - if (x == 206) { - printf("x ist 206\n"); - } - if (x == 207) { - printf("x ist 207\n"); - } - if (x == 208) { - printf("x ist 208\n"); - } - if (x == 209) { - printf("x ist 209\n"); - } - if (x == 210) { - printf("x ist 210\n"); - } - if (x == 211) { - printf("x ist 211\n"); - } - if (x == 212) { - printf("x ist 212\n"); - } - if (x == 213) { - printf("x ist 213\n"); - } - if (x == 214) { - printf("x ist 214\n"); - } - if (x == 215) { - printf("x ist 215\n");} - if (x == 216) { - printf("x ist 216\n"); - } - if (x == 217) { - printf("x ist 217\n"); - } - if (x == 218) { - printf("x ist 218\n"); - } - if (x == 219) { - printf("x ist 219\n"); - } - if (x == 220) { - printf("x ist 220\n"); - } - if (x == 221) { - printf("x ist 221\n"); - } - if (x == 222) { - printf("x ist 222\n"); - } - if (x == 223) { - printf("x ist 223\n"); - } - if (x == 224) { - printf("x ist 224\n"); - } - if (x == 225) { - printf("x ist 225\n"); - } - if (x == 226) { - printf("x ist 226\n"); - } - if (x == 227) { - printf("x ist 227\n"); - } - if (x == 228) { - printf("x ist 228\n"); - } - if (x == 229) { - printf("x ist 229\n"); - } - if (x == 230) { - printf("x ist 230\n"); - } - if (x == 231) { - printf("x ist 231\n"); - } - if (x == 232) { - printf("x ist 232\n"); - } - if (x == 233) { - printf("x ist 233\n"); - } - if (x == 234) { - printf("x ist 234\n"); - } - if (x == 235) { - printf("x ist 235\n"); - } - if (x == 236) { - printf("x ist 236\n"); - } - if (x == 237) { - printf("x ist 237\n"); - } - if (x == 238) { - printf("x ist 238\n"); - } - if (x == 239) { - printf("x ist 239\n"); - } - if (x == 240) { - printf("x ist 240\n"); - } - if (x == 241) { - printf("x ist 241\n"); - } - if (x == 242) { - printf("x ist 242\n"); - } - if (x == 243) { - printf("x ist 243\n"); - } - if (x == 244) { - printf("x ist 244\n"); - } - if (x == 245) { - printf("x ist 245\n"); - } - if (x == 246) { - printf("x ist 246\n"); - } - if (x == 247) { - printf("x ist 247\n"); - } - if (x == 248) { - printf("x ist 248\n"); - } - if (x == 249) { - printf("x ist 249\n"); - } - - return 0; -} \ No newline at end of file diff --git a/scripts/incremental-test-generation/sample-files/minimalTestNothing.c b/scripts/incremental-test-generation/sample-files/minimalTestNothing.c deleted file mode 100644 index d4db2f965f..0000000000 --- a/scripts/incremental-test-generation/sample-files/minimalTestNothing.c +++ /dev/null @@ -1,38 +0,0 @@ -// Run with: -// python3 RUN_CLI.py -m -dp -er -i ../sample-files/minimalTestNothing.c -rfb - -#include -#include -#include - -int main() { - int a = 10; - int b = 20; - - if (a > b) { - // Is never reached so there is registered nothing! - printf("a is greater than b\n"); - } - - if (a < b) { - // Is reached so there no problem - printf("a is greater than b\n"); - } - - // Try with random inputs -> No problem - srand(time(NULL)); - int r1 = rand(); - int r2 = rand(); - - if (r1 > r2) { - // No knowledge about outcome so no problem - printf("a is greater than b\n"); - } - - if (r1 < r2) { - // No knowledge about outcome so no problem - printf("a is greater than b\n"); - } - - return 0; -} diff --git a/scripts/incremental-test-generation/sample-files/pthread.c b/scripts/incremental-test-generation/sample-files/pthread.c deleted file mode 100644 index b47776dbf0..0000000000 --- a/scripts/incremental-test-generation/sample-files/pthread.c +++ /dev/null @@ -1,124 +0,0 @@ -#include -#include - -void* thread_function(void* arg) { - printf("Thread function\n"); - return NULL; -} - -void cleanup_function(void* arg) { - printf("Cleanup function\n"); -} - -int main() { - pthread_t thread; - pthread_attr_t attr; - pthread_cond_t cond; - pthread_condattr_t cond_attr; - pthread_key_t key; - pthread_mutex_t mutex; - pthread_mutexattr_t mutex_attr; - pthread_once_t once_control; - pthread_rwlock_t rwlock; - pthread_rwlockattr_t rwlock_attr; - struct sched_param param; - - // Attribute functions - pthread_attr_destroy(&attr); - pthread_attr_getdetachstate(&attr, NULL); - pthread_attr_getguardsize(&attr, NULL); - pthread_attr_getinheritsched(&attr, NULL); - pthread_attr_getschedparam(&attr, ¶m); - pthread_attr_getschedpolicy(&attr, NULL); - pthread_attr_getscope(&attr, NULL); - pthread_attr_getstackaddr(&attr, NULL); - pthread_attr_getstacksize(&attr, NULL); - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, 0); - pthread_attr_setguardsize(&attr, 0); - pthread_attr_setinheritsched(&attr, 0); - pthread_attr_setschedparam(&attr, ¶m); - pthread_attr_setschedpolicy(&attr, 0); - pthread_attr_setscope(&attr, 0); - pthread_attr_setstackaddr(&attr, NULL); - pthread_attr_setstacksize(&attr, 0); - - // Cancellation functions - pthread_cancel(thread); - pthread_setcancelstate(0, NULL); - pthread_setcanceltype(0, NULL); - pthread_testcancel(); - - // Condition variable functions - pthread_cond_broadcast(&cond); - pthread_cond_destroy(&cond); - pthread_cond_init(&cond, &cond_attr); - pthread_cond_signal(&cond); - pthread_cond_timedwait(&cond, &mutex, NULL); - pthread_cond_wait(&cond, &mutex); - - // Condition attribute functions - pthread_condattr_destroy(&cond_attr); - pthread_condattr_getpshared(&cond_attr, NULL); - pthread_condattr_init(&cond_attr); - pthread_condattr_setpshared(&cond_attr, 0); - - // Creation and termination functions - pthread_create(&thread, &attr, thread_function, NULL); - pthread_detach(thread); - pthread_equal(thread, thread); - pthread_exit(NULL); - pthread_join(thread, NULL); - - // Thread-specific data functions - pthread_key_create(&key, cleanup_function); - pthread_key_delete(key); - pthread_getspecific(key); - pthread_setspecific(key, NULL); - - // Mutex functions - pthread_mutex_destroy(&mutex); - pthread_mutex_getprioceiling(&mutex, NULL); - pthread_mutex_init(&mutex, &mutex_attr); - pthread_mutex_lock(&mutex); - pthread_mutex_setprioceiling(&mutex, 0, NULL); - pthread_mutex_trylock(&mutex); - pthread_mutex_unlock(&mutex); - - // Mutex attribute functions - pthread_mutexattr_destroy(&mutex_attr); - pthread_mutexattr_getprioceiling(&mutex_attr, NULL); - pthread_mutexattr_getprotocol(&mutex_attr, NULL); - pthread_mutexattr_getpshared(&mutex_attr, NULL); - pthread_mutexattr_gettype(&mutex_attr, NULL); - pthread_mutexattr_init(&mutex_attr); - pthread_mutexattr_setprioceiling(&mutex_attr, 0); - pthread_mutexattr_setprotocol(&mutex_attr, 0); - pthread_mutexattr_setpshared(&mutex_attr, 0); - pthread_mutexattr_settype(&mutex_attr, 0); - - // One-time initialization - pthread_once(&once_control, (void (*)(void))thread_function); - - // Read-write lock functions - pthread_rwlock_destroy(&rwlock); - pthread_rwlock_init(&rwlock, &rwlock_attr); - pthread_rwlock_rdlock(&rwlock); - pthread_rwlock_tryrdlock(&rwlock); - pthread_rwlock_trywrlock(&rwlock); - pthread_rwlock_unlock(&rwlock); - pthread_rwlock_wrlock(&rwlock); - - // Read-write lock attribute functions - pthread_rwlockattr_destroy(&rwlock_attr); - pthread_rwlockattr_getpshared(&rwlock_attr, NULL); - pthread_rwlockattr_init(&rwlock_attr); - pthread_rwlockattr_setpshared(&rwlock_attr, 0); - - // Other functions - pthread_self(); - pthread_getschedparam(thread, NULL, ¶m); - pthread_setschedparam(thread, 0, ¶m); - - return 0; -} diff --git a/scripts/incremental-test-generation/sample-files/threads.c b/scripts/incremental-test-generation/sample-files/threads.c deleted file mode 100644 index 0148d12a0d..0000000000 --- a/scripts/incremental-test-generation/sample-files/threads.c +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include - -#define NUM_THREADS 5 - -void *threadFunction(void *arg) { - int threadID = *(int *)arg; - printf("Thread %d is running.\n", threadID); - // Perform some task in the thread - printf("Thread %d completed.\n", threadID); - pthread_exit(NULL); -} - -int main() { - pthread_t threads[NUM_THREADS]; - int threadArgs[NUM_THREADS]; - int i, result; - - // Create threads - for (i = 0; i < NUM_THREADS; i++) { - threadArgs[i] = i; - result = pthread_create(&threads[i], NULL, threadFunction, &threadArgs[i]); - if (result != 0) { - printf("Error creating thread %d. Exiting program.\n", i); - return -1; - } - } - - // Wait for threads to finish - for (i = 0; i < NUM_THREADS; i++) { - result = pthread_join(threads[i], NULL); - if (result != 0) { - printf("Error joining thread %d. Exiting program.\n", i); - return -1; - } - } - - printf("All threads have completed. Exiting program.\n"); - - return 0; -} diff --git a/scripts/incremental-test-generation/sample-files/zlib.sh b/scripts/incremental-test-generation/sample-files/zlib.sh deleted file mode 100755 index b5192ef281..0000000000 --- a/scripts/incremental-test-generation/sample-files/zlib.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -############################################################## -####################### USER VARIABLES ####################### -############################################################## -# Variables -git_url="https://github.com/madler/zlib.git" -use_cmake=false -use_make=true -path_to_build="." - -# Functions before and after build -pre_build_commands() { - ./configure -} - -post_build_commands() { - : -} -############################################################## -####################### USER VARIABLES ####################### -############################################################## - -# Export variables so they can be used in the main script -export git_url -export use_cmake -export use_make -export path_to_build -export pre_build_commands -export post_build_commands \ No newline at end of file diff --git a/scripts/incremental-test-generation/util/.gitignore b/scripts/incremental-test-generation/util/.gitignore deleted file mode 100644 index 4ecb4bd38f..0000000000 --- a/scripts/incremental-test-generation/util/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -__pycache__ -__init__.py diff --git a/scripts/incremental-test-generation/util/README.md b/scripts/incremental-test-generation/util/README.md deleted file mode 100644 index be92ae3f16..0000000000 --- a/scripts/incremental-test-generation/util/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Util -This directory contains all the utility files required to generate incremental tests from the generated mutated files. Each Python script is briefly described below, and they are ordered according to their typical usage. Please use the `meta.yaml` files from the previous script calls. - -# generate_programs.py -This script generates an empty `meta.yaml` file and, depending on the passed options, calls the different [generators](../generators/README.md) to create mutated files in the `temp_dir` (`p_42.c`). It then adds the `___goblint_check()` function along with corresponding comments by calling `add_check.py` and `add_check_comments.py`. - -# add_check.py -This script calls Goblint to add the `___goblint_check()` function to the C files in the `temp_dir` (`p_42_check.c`). - -# add_check_comments.py -This script adds the corresponding `//UNKNOWN //SUCCESS` or `//SUCCESS` comments to all the Goblint checks (`___goblint_check()`) in the `temp_dir` (`p_42_check_unknown.c` / `p_42_check_success.c`). - -# generate_tests.py -This script generates the test files for each mutation. A test consists of a source `.c` file, a `.patch` file containing the changes from the mutation, and a `.json` file with the Goblint options. Normally, a patch is the difference between the original program provided by the user (`p_0_check_success.c`) and the mutated program (`p_42_check_unknown.c`). For precision tests, the mutated program is only annotated with success Goblint checks (`p_42_check_success.c`). For mutations generated by a Git project, the patch is formed from the previous file to the current file. diff --git a/scripts/incremental-test-generation/util/add_check.py b/scripts/incremental-test-generation/util/add_check.py deleted file mode 100644 index b1ebc16099..0000000000 --- a/scripts/incremental-test-generation/util/add_check.py +++ /dev/null @@ -1,67 +0,0 @@ -# Takes a file and generates the goblint checks -# Stores the file with an additional "_c" -# When there is a compilation error the process writes [META_EXCEPTION] into the meta data file for the given index - -import argparse -import subprocess -import sys -import yaml -sys.path.insert(0, "..") -from util.util import * - -def add_check(file_path: str, index: int, goblint_path: str, meta_path: str): - command = [ - goblint_path, - "--enable", - "trans.goblint-check", - "--set", - "trans.activated", - '["assert"]', - "--set", - "trans.output", - file_path.rsplit('.', 1)[0] + '_check.c', - file_path - ] - - result = subprocess.run(command, text=True, capture_output=True) - - compiling = result.returncode == 0 - - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data["p_" + str(index)][META_COMPILING] = compiling - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - - if not compiling: - print(f"{COLOR_RED}Error compiling program with index {index}.{COLOR_RESET}") - if index == 0 and not yaml_data["p_0"][META_TYPE] == Generate_Type.GIT.value: - print(f"{COLOR_RED}The original program did not compile. Stopping program!{COLOR_RESET}") - sys.exit(-1) - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - yaml_data[f"p_{index}"] = { - META_TYPE: Generate_Type.ML.value, - META_EXCEPTION: result.stderr, - META_COMPILING: False - } - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - return False - else: - return True - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate __goblint_check() for a C file. When not compiling write [NOT COMPILING] in the meta file') - - # Add the required arguments - parser.add_argument('file', help='Path to the C file') - parser.add_argument('index', help='Index of the file (needed for meta data)') - parser.add_argument('goblint', help='Path to the Goblint executable') - parser.add_argument('meta', help='Path to the meta data file') - - # Parse the command-line arguments - args = parser.parse_args() - - # Call the process_file function with the provided arguments - add_check(args.file, args.index, args.goblint, args.meta) diff --git a/scripts/incremental-test-generation/util/add_check_comments.py b/scripts/incremental-test-generation/util/add_check_comments.py deleted file mode 100644 index 275573c356..0000000000 --- a/scripts/incremental-test-generation/util/add_check_comments.py +++ /dev/null @@ -1,40 +0,0 @@ -# Adds "//UNDEFINED" or "//SUCCESS" to the Goblint checks "__goblint_check(exp);". -# Stores the file with the same name when adding "//SUCCESS". -# Stores the file with the appendix _u when adding "//UNKNOWN". -# Run with ´python3 script.py example.txt -u´ or ´python3 script.py example.txt -s´. - -import argparse -import re - -def add_check_comments(file_path: str, unknown_instead_of_success: bool): - with open(file_path, 'r') as file: - lines = file.readlines() - modified_lines = [] - - for line in lines: - if '__goblint_check(' in line: - match = re.search(r'(__goblint_check\(.*?\);)', line) - if match: - modified_line = match.group(1) - if unknown_instead_of_success: - modified_line += ' //UNKNOWN //SUCCESS' - else: - modified_line += ' //SUCCESS' - line = line.replace(match.group(1), modified_line) - modified_lines.append(line) - - new_file_name = file_path.rsplit('.', 1)[0] + ('_unknown.c' if unknown_instead_of_success else '_success.c') - with open(new_file_name, 'w') as new_file: - new_file.writelines(modified_lines) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("file", help="Path to the C file") - parser.add_argument("-u", "--undefined", action="store_true", help="Option for //UNDEFINED") - parser.add_argument("-s", "--success", action="store_true", help="ption for //SUCCESS") - args = parser.parse_args() - - if not (args.undefined or args.success): - parser.error("Error: Invalid option. Provide -u for \"//UNDEFINED\" or -s for \"//SUCCESS\".") - - add_check_comments(args.file, args.undefined) diff --git a/scripts/incremental-test-generation/util/generate_programs.py b/scripts/incremental-test-generation/util/generate_programs.py deleted file mode 100644 index 728e2c9608..0000000000 --- a/scripts/incremental-test-generation/util/generate_programs.py +++ /dev/null @@ -1,124 +0,0 @@ -import argparse -import os -import shutil -import sys -sys.path.insert(0, "..") -from util.util import * -from util.add_check import add_check -from util.add_check_comments import add_check_comments -from generators.generate_mutations import * -from generators.generate_ml import * -from generators.generate_git import * - -generate_type_source = "SOURCE" - -def generate_programs(source_path, temp_dir, clang_tidy_path, goblint_path, apikey_path, mutations, enable_mutations, enable_ml, enable_git, ml_count, ml_select, ml_interesting, ml_16k, git_start, git_end): - # Clean working directory - if os.path.isdir(temp_dir): - shutil.rmtree(temp_dir) - os.makedirs(temp_dir) - # Create Meta file - meta_path = os.path.join(temp_dir, META_FILENAME) - with open(meta_path, 'w') as outfile: - yaml.dump({'n': 0, 'p_0': {META_TYPE: generate_type_source}}, outfile) - # Copy the source program into the temp dir - program_path = os.path.join(temp_dir, 'p.c' if not enable_git else 'p.sh') - shutil.copy2(source_path, program_path) - program_0_path = os.path.join(temp_dir, 'p_0.c') - shutil.copy2(source_path, program_0_path) - - index = 0 - if enable_mutations: - index = generate_mutations(program_path, clang_tidy_path, meta_path, mutations) - - if enable_ml: - index = generate_ml(program_path, apikey_path, meta_path, ml_count, ml_select, ml_interesting, ml_16k) - - if enable_git: - index = generate_git(goblint_path, temp_dir, meta_path, program_path, git_start, git_end) - - # Add checks with comments - print(SEPERATOR) - if enable_git: - print('Generating goblint checks. This may take a while...') - for i in range(index + 1): - print(f"\r[{i}/{index}] Generating goblint checks...", end='') - sys.stdout.flush() - file_path = os.path.join(temp_dir, f"p_{i}.c") - compiling = add_check(file_path, i, goblint_path, meta_path) - if not compiling: - continue - file_path = os.path.join(temp_dir, f"p_{i}_check.c") - if i == 0 or enable_git: - add_check_comments(file_path, unknown_instead_of_success=True) - add_check_comments(file_path, unknown_instead_of_success=False) - print(f"\r{COLOR_GREEN}Generating goblint checks [DONE]{SPACE}{COLOR_RESET}") - - # Check how many and which files were not compiling - print(SEPERATOR) - print("Check if the files compiled...", end='') - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - failed_count = 0 - failed_compilation_keys = [] - for key, value in yaml_data.items(): - if isinstance(value, dict) and META_COMPILING in value and value[META_COMPILING] is False: - failed_count += 1 - failed_compilation_keys.append(key) - if failed_count == 0: - print(f"\r{COLOR_GREEN}All files compiled succesfully{COLOR_RESET}") - else: - print(f"\r{COLOR_RED}There were {failed_count} files not compiling (stderr written to {temp_dir}/meta.yaml):{COLOR_RESET} {failed_compilation_keys}") - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate programs in the working directory') - parser.add_argument('source_path', help='Path to the original program or git sh file provided by the user') - parser.add_argument('temp_dir', help='Path to the working directory') - parser.add_argument('clang_tidy_path', help='Path to the modified clang-tidy executable') - parser.add_argument('goblint_path', help='Path to the goblint executable') - parser.add_argument('--apikey-path', help='Path to the API') - parser.add_argument('--enable-mutations', action='store_true', help='Enable Mutations. When no mutation is selected all are activated.') - parser.add_argument('--enable-ml', action='store_true', help='Enable ML') - parser.add_argument('--enable-git', action='store_true', help='Enable Git') - parser.add_argument('--ml-count', type=int, default=DEFAULT_ML_COUNT, help='Number of ML programs to generate') - parser.add_argument('--ml-select', type=int, default=DEFAULT_ML_SELECT, help='Number of selected lines for ML') - parser.add_argument('--ml-interesting', default="[]", help='Lines to randomly choose the start line for selection (Defaul are all lines)') - parser.add_argument('--ml-16k', action='store_true', help='Use the 16k mode for ml') - parser.add_argument('--git-start', help='The starting commit hash for git generation') - parser.add_argument('--git-end', help='The ending commit hash for git generation') - - # Add mutation options - add_mutation_options(parser) - - args = parser.parse_args() - - # At least one generator has to be enabled - if not args.enable_mutations and not args.enable_ml and not args.enable_git: - parser.error("At least one generator has to be enabled (--enable_mutations, --enable-ml, --enable-git)") - - # If using git, only git can be used - if args.enable_git and (args.enable_ml or args.enable_mutations): - parser.error("--enable-git cannot be used with --enable-ml or --enable-mutations") - - # If all mutation options are false, set all to true - mutations = get_mutations_from_args(args) - non_str_attributes = [attr for attr in vars(mutations) if not attr.endswith('_s')] - if all(getattr(mutations, attr) is False for attr in non_str_attributes): - mutations = Mutations(True, True, True, True, True, True) - - # Check required parameters for optional features - if args.enable_ml and not args.apikey_path: - parser.error("--enable-ml requires --apikey-path") - - # Check ml intersting string - if args.ml_interesting != "[]" and validate_interesting_lines(args.ml_interesting, None) == None: - sys.exit(-1) - - # Check git commit hashes - git_start_commit = args.git_start - git_end_commit = args.git_end - if (git_start_commit == None and git_end_commit != None) or (git_start_commit != None and git_end_commit == None): - parser.error('[ERROR] Give a git start commit hash AND a end commit hash') - - generate_programs(args.source_path, args.temp_dir, args.clang_tidy_path, args.goblint_path, args.apikey_path, mutations, args.enable_mutations, args.enable_ml, args.enable_git, args.ml_count, args.ml_select, args.ml_interesting, args.ml_16k, args.git_start, args.git_end) - diff --git a/scripts/incremental-test-generation/util/generate_tests.py b/scripts/incremental-test-generation/util/generate_tests.py deleted file mode 100644 index 985db64b85..0000000000 --- a/scripts/incremental-test-generation/util/generate_tests.py +++ /dev/null @@ -1,182 +0,0 @@ -import argparse -import json -import os -import re -import shutil -import subprocess -import questionary -import yaml -import sys -sys.path.insert(0, "..") -from util.util import * - -def generate_tests(temp_dir, target_dir, goblint_config, precision_test, temp_name): - # Check the name of the target_dir - directory_name = os.path.basename(target_dir) - if not temp_name and not check_test_name(directory_name): - sys.exit(-1) - - if os.path.exists(target_dir): - print(f'{COLOR_RED}The test directory {target_dir} already exists.{COLOR_RESET}') - if questionary.confirm('Replace the directory?', default=True).ask(): - shutil.rmtree(target_dir) - else: - sys.exit(-1) - os.makedirs(target_dir) - - # Read the meta.yaml - meta_path = os.path.join(temp_dir,META_FILENAME) - with open(meta_path, 'r') as file: - yaml_data = yaml.safe_load(file) - n = yaml_data[META_N] - - # loop threw all generated mutations - original_target_dir = target_dir - test_paths = [target_dir] - current_test_num = 1 # Skip num 0 to align program index with test number - current_dir_num = int(directory_name[:2]) - unchanged_count = 0 - compiling_programs = [] - if temp_name and int(directory_name[:3]) != 100: - print(f'{COLOR_RED}[ERROR] The directory number for temp files must be 100 but was {directory_name}{COLOR_RESET}') - sys.exit(-1) - elif temp_name: - current_dir_num = 100 - original_target_dir = os.path.join(os.path.dirname(target_dir), '99' + os.path.basename(target_dir)[3:]) - for i in range(n + 1): - if current_test_num > 99: - current_dir_num += 1 - - # When temporary files let the files go over 99 to rename them later - if not temp_name and current_dir_num > 99: - print(f'{COLOR_RED}[ERROR] The directory number 100 is out of range. Consider starting with a lower than {directory_name} {COLOR_RESET}') - sys.exit(-1) - - group_name = re.match(r'\d+-(.*)', directory_name).group(1) - target_dir = os.path.join(os.path.dirname(target_dir), f'{current_dir_num:02}-{group_name}') - test_paths.append(target_dir) - - if os.path.exists(target_dir): - print(f'{COLOR_RED}The test directory {target_dir} already exists.{COLOR_RESET}') - if questionary.confirm('Replace the directory?', default=True).ask(): - shutil.rmtree(target_dir) - else: - sys.exit(-1) - os.mkdir(target_dir) - - current_test_num = 0 - - current_program_id = f'p_{i}' - compilation_success = yaml_data[current_program_id][META_COMPILING] - if compilation_success: - compiling_programs.append(i) - else: - print(f"\rGenerating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Not compiling){COLOR_RESET}") - continue - if META_EXCEPTION in yaml_data[current_program_id]: - print(f"\rGenerating test files [{i}/{n}] {COLOR_YELLOW}Skipped {i} (Exception occured during generation){COLOR_RESET}") - continue - type = yaml_data[current_program_id][META_TYPE] - if type == Generate_Type.SOURCE.value or (type == Generate_Type.GIT.value and i == 0): - continue - if (i-1) % 9 == 0: - print(f"\rGenerating test files [{i}/{n}]", end='') - sub_type = yaml_data[current_program_id][META_SUB_TYPE] - if type == Generate_Type.MUTATION.value or type == Generate_Type.GIT.value: - test_name = f'{_format_number(current_test_num)}-{type}_{sub_type}_{_format_number(i)}' - else: - test_name = f'{_format_number(current_test_num)}-{type}_{_format_number(i)}' - - # Select depending on generator the start and end file of at test - if type == Generate_Type.MUTATION.value or type == Generate_Type.ML.value: - source_program_id = 'p_0' - start_program = os.path.join(temp_dir, current_program_id + '_check_success.c') - end_program = os.path.join(temp_dir, source_program_id + '_check_unknown.c') - end_program_precison = os.path.join(temp_dir, source_program_id + '_check_success.c') - elif type ==Generate_Type.GIT.value: - # If it's the first compiling program skip it. - if i == compiling_programs[0]: - print(f"\rGenerating test files [{i}/{n}] {COLOR_BLUE}Skipped {i} as the source file for the first test{COLOR_RESET}") - continue - - # Find the index of the previous compiling program. - previous_program_index = compiling_programs.index(i) - 1 - previous_program_id = f'p_{compiling_programs[previous_program_index]}' - - start_program = os.path.join(temp_dir, previous_program_id + '_check_success.c') - end_program = os.path.join(temp_dir, current_program_id + '_check_unknown.c') - end_program_precison = os.path.join(temp_dir, current_program_id + '_check_success.c') - else: - print(f'\n{COLOR_RED}[ERROR] Trying to generate tests from unknown generator type{COLOR_RESET}') - sys.exit(-1) - - # Copy mutated code as the original code - shutil.copy2(start_program, os.path.join(target_dir, test_name + '.c')) - # Create a patch file - patch_path = os.path.join(target_dir, test_name + '.patch') - command = 'diff -u {} {} > {}'.format( - os.path.join(target_dir, test_name + '.c'), - end_program_precison if precision_test else end_program, - patch_path - ) - result = subprocess.run(command, shell=True) - if temp_name: - # For patch files keep the 99 for running inplace after renaming folder - _fix_patch_file(patch_path, os.path.basename(original_target_dir), test_name + '.c') - else: - _fix_patch_file(patch_path, os.path.basename(target_dir), test_name + '.c') - if result.returncode in [0, 1]: - if result.returncode == 0: - print(f"\r{COLOR_YELLOW}[WARNING] There were no changes in the patch for test {i}{COLOR_RESET}") - unchanged_count += 1 - yaml_data[current_program_id][META_DIFF] = False - else: - yaml_data[current_program_id][META_DIFF] = True - else: - raise Exception("Command failed with return code: {}".format(result.returncode)) - if goblint_config == None: - # Create a empty config file - data = {} - with open(os.path.join(target_dir, test_name + '.json'), 'w') as f: - json.dump(data, f) - else: - # Copy config file - shutil.copy2(os.path.abspath(os.path.expanduser(goblint_config)), os.path.join(target_dir, test_name + '.json')) - - current_test_num += 1 - - print(f"\r{COLOR_GREEN}Generating test files [DONE]{SPACE}{COLOR_RESET}") - if unchanged_count > 0: - print(f'{COLOR_YELLOW}There were {unchanged_count} patch files with no changes.{COLOR_RESET}') - - with open(meta_path, 'w') as file: - yaml.safe_dump(yaml_data, file) - - # Return the generated directories - return test_paths - -def _fix_patch_file(patch_file, folder_name, file_name): - with open(patch_file, 'r') as file: - lines = file.readlines() - - with open(patch_file, 'w') as file: - for line in lines: - if line.startswith('---') or line.startswith('+++'): - line = line.split(' ')[0] + " " + "tests/incremental/" + folder_name + "/" + file_name + "\n" - file.write(line) - -def _format_number(n): - return str(n).zfill(2) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate Test files in the target directory based on the working directory') - parser.add_argument('temp_dir', help='Path to the working directory') - parser.add_argument('target_dir', help='Path to the target directory') - parser.add_argument('-p', '--precision-test', action='store_true', help='Generate tests for precision') - parser.add_argument('-c', '--goblint-config', help='Optional path to the goblint config file used for the tests (using no option creates an empty one)') - parser.add_argument('-t', '--temp-name', action='store_true', help='Store name in special format for running the tests and removing them directly again') - - args = parser.parse_args() - - generate_tests(args.temp_dir, args.target_dir, args.goblint_config, args.precision_test, args.temp_name) diff --git a/scripts/incremental-test-generation/util/run_tests.py b/scripts/incremental-test-generation/util/run_tests.py deleted file mode 100644 index 57a15d8f18..0000000000 --- a/scripts/incremental-test-generation/util/run_tests.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -import os -import shutil -import subprocess -import re -import sys - -import questionary -sys.path.insert(0, "..") -from util.util import * - -def run_tests(program_path, test_dir, goblint_repo_dir, cfg): - # When the directory has a starting number >99 rename it for in place running of the tests - match = re.match(r'(\d+)-(.*)', os.path.basename(test_dir)) - if match: - number, text = match.groups() - number = int(number) - if number > 99: - number = 99 - new_name = f'{number}-{text}' - os.rename(test_dir, os.path.join(os.path.dirname(test_dir), new_name)) - test_dir = os.path.join(os.path.dirname(test_dir), new_name) - else: - print(f"{COLOR_RED}[ERROR] The test directory had not the format number-text{COLOR_RESET}") - - - # Check the name of the test_dir - test_dir_name = os.path.basename(test_dir) - if test_dir_name != "99-temp": - print(f"{COLOR_RED}[ERROR] The test directory name has to be \'99-temp\'{COLOR_RESET}") - sys.exit(-1) - - incremental_tests_dir_abs = os.path.abspath(os.path.join(goblint_repo_dir, "tests", "incremental", test_dir_name)) - if os.path.exists(incremental_tests_dir_abs): - print(f'{COLOR_RED}The test directory {incremental_tests_dir_abs} already exists.{COLOR_RESET}') - if questionary.confirm('Replace the directory?', default=True).ask(): - shutil.rmtree(incremental_tests_dir_abs) - else: - sys.exit(-1) - shutil.copytree(test_dir, incremental_tests_dir_abs) - - ruby_path_abs = os.path.abspath(os.path.join(goblint_repo_dir, "scripts", "update_suite.rb")) - params = get_params_from_file(program_path) - if params != "": - print(f"\n{COLOR_YELLOW}Using parameters from input file:{COLOR_RESET} {params}") - original_dir = os.getcwd() - os.chdir(goblint_repo_dir) - command = f"{ruby_path_abs} group temp -p \"{params}\" -i" - if cfg: - command += " -c" - process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - - # Print the output including carriage returns - line = '' - while process.poll() is None: - char = process.stdout.read(1).decode('utf-8') - if char == '\r' or char == '\n': - sys.stdout.write('\r' + line) - sys.stdout.flush() - if char == '\n': - print() - line = '' - else: - line += char - if line: - sys.stdout.write('\r' + line + '\n') - sys.stdout.flush() - - process.wait() - - shutil.rmtree(incremental_tests_dir_abs) - shutil.rmtree(test_dir) - os.chdir(original_dir) - -def get_params_from_file(filename): - param_pattern = re.compile(r"\s*//.*PARAM\s*:\s*(.*)") - - with open(filename, 'r') as f: - for line in f: - match = param_pattern.match(line) - if match: - params = match.group(1).strip() - return params - - return "" - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Run the tests in the specified test directory with the ruby script from Goblint') - parser.add_argument('program_path', help='Path to the input file of the user') - parser.add_argument('test_dir', help='Path to the directory with the tests (WARNING Will be removed!)') - parser.add_argument('goblint_repo_dir', help='Path to the Goblint repository') - parser.add_argument('-c', '--cfg', action='store_true', help='Run with fine-grained cfg-based change detection') - - args = parser.parse_args() - - run_tests(args.program_path, args.test_dir, args.goblint_repo_dir, args.cfg) \ No newline at end of file diff --git a/scripts/incremental-test-generation/util/util.py b/scripts/incremental-test-generation/util/util.py deleted file mode 100644 index 9cc37cc406..0000000000 --- a/scripts/incremental-test-generation/util/util.py +++ /dev/null @@ -1,75 +0,0 @@ -from enum import Enum -import re -import shutil - -class Mutations: - def __init__(self, rfb=False, uoi=False, ror=False, cr=False, rt=False, lcr=False): - self.rfb = rfb - self.rfb_s = "remove-function-body" - self.uoi = uoi - self.uoi_s = "unary-operator-inversion" - self.ror = ror - self.ror_s = "relational-operator-replacement" - self.cr = cr - self.cr_s = "constant-replacement" - self.rt = rt - self.rt_s = "remove-thread" - self.lcr = lcr - self.lcr_s = "logical-connector-replacement" - -class Generate_Type(Enum): - SOURCE = 'SOURCE' - MUTATION = 'MUTATION' - ML = 'ML' - GIT = 'GIT' - -COLOR_RED = '\033[31m' -COLOR_BLUE = '\033[34m' -COLOR_GREEN = '\033[32m' -COLOR_YELLOW = '\033[33m' -COLOR_RESET = '\033[0m' - -SEPERATOR = f"{COLOR_BLUE}------------------------------------------------------------------------------------------------------------------{COLOR_RESET}" -SPACE = ' ' * 20 - -META_FILENAME = 'meta.yaml' -META_N = 'n' -META_COMPILING = 'compilation' -META_EXCEPTION = 'exception' -META_DIFF = 'diff' -META_TYPE = 'type' -META_SUB_TYPE = 'sub_type' -META_LINES = 'lines' -META_FAILURES = 'failures' -META_FAILURES_STD_OUT = 'stdout' -META_FAILURES_STD_ERR = 'stderr' - -CONFIG_FILENAME = 'config.yaml' -CONFIG_GOBLINT = "goblint-path" -CONFIG_LLVM = "llvm-path" -CONFIG_LAST_INPUT_MUTATION = "last-input-mutation" -CONFIG_LAST_INPUT_GIT = "last-input-git" - -APIKEY_FILENAME = 'api-key.yaml' -APIKEY_APIKEY = 'api-key' -APIKEY_ORGANISATION = 'organisation' - -DEFAULT_ML_COUNT = 5 -DEFAULT_ML_SELECT = 50 -ML_WORKERS = 5 - -def make_program_copy(program_path, index): - new_path = program_path.rsplit('.', 1)[0] + '_' + str(index) + '.c' - shutil.copy2(program_path, new_path) - return new_path - -def check_test_name(directoryName): - if directoryName is None or not isinstance(directoryName, str): - print(f"{COLOR_RED}[ERROR] Target Directory name {directoryName} is not a string{COLOR_RESET}") - return False - - pattern = r"\d{2}-\w+" - if not re.match(pattern, directoryName): - print(f"{COLOR_RED}[ERROR] Target Directory name {directoryName} is not of the format 01-Name (\d{{2}}-\w+){COLOR_RESET}") - return False - return True \ No newline at end of file From 49d71b17eef067f043204c2d139067d7509aea83 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 19 Jun 2023 13:20:42 +0200 Subject: [PATCH 101/131] Add gitignore for temp test files --- tests/incremental/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/incremental/.gitignore diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore new file mode 100644 index 0000000000..f9c88fa39b --- /dev/null +++ b/tests/incremental/.gitignore @@ -0,0 +1,2 @@ +# Do not consider temporary files for the running of mutation generator +99-temp \ No newline at end of file From 7a96981aa7d8ea028e240dad2173fc41fd83b3aa Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 21 Jun 2023 00:25:06 +0200 Subject: [PATCH 102/131] Read always the PARMs from the first line --- scripts/update_suite.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 01e609fab4..16b20f6a3f 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -543,10 +543,9 @@ def run () if incremental then config_path = File.expand_path(f[0..-3] + ".json", grouppath) params = if cfg then "--conf #{config_path} --set incremental.compare cfg" else "--conf #{config_path}" end - else - lines[0] =~ /PARAM: (.*)$/ - if $1 then params = $1 else params = "" end end + lines[0] =~ /PARAM: (.*)$/ + if $1 then params << " #{$1}" else params << "" end # always enable debugging so that the warnings would work params << " --set dbg.debug true" p = if incremental then From 6633ada9be4993794a4ca72ebde931fc91bb7e22 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 21 Jun 2023 00:31:18 +0200 Subject: [PATCH 103/131] Revert passing params with -p --- scripts/update_suite.rb | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 16b20f6a3f..649e5b4b67 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -66,10 +66,6 @@ def clearline cfg = ARGV.last == "-c" && ARGV.pop incremental = (ARGV.last == "-i" && ARGV.pop) || cfg report = ARGV.last == "-r" && ARGV.pop -p_index = ARGV.index("-p") # Comand line parameters for Goblint (pass before other otions) -user_params = p_index.nil? || p_index == ARGV.length - 1 ? "" : ARGV[p_index + 1] -ARGV.delete_at(p_index) unless p_index.nil? -ARGV.delete_at(p_index) unless p_index.nil? || p_index >= ARGV.length only = ARGV[0] unless ARGV[0].nil? if marshal || witness || incremental then sequential = true @@ -138,7 +134,7 @@ def initialize(project, tests, tests_line, todo) def report filename = File.basename(p.path) system($highlighter.call(filename, orgfile)) - `#{$goblint} #{filename} --set justcil true #{p.params} #{p.user_params} >#{cilfile} 2> /dev/null` + `#{$goblint} #{filename} --set justcil true #{p.params} >#{cilfile} 2> /dev/null` p.size = `wc -l #{cilfile}`.split[0] end @@ -266,7 +262,7 @@ def to_html class Project attr_reader :id, :name, :group, :path, :params, :testset, :html_heading - attr_accessor :size, :testset, :user_params + attr_accessor :size, :testset def initialize(id, name, group, path, params) @id = id @name = name @@ -374,7 +370,7 @@ def run_testset (testset, cmd, starttime) def run filename = File.basename(@path) - cmd = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set goblint-dir .goblint-#{@id.sub('/','-')} 2>#{@testset.statsfile}" + cmd = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set goblint-dir .goblint-#{@id.sub('/','-')} 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd, starttime) end @@ -440,8 +436,8 @@ def create_test_set(lines) def run filename = File.basename(@path) - cmd = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --enable incremental.save --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-save 2>#{@testset.statsfile}" - cmd_incr = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset_incr.warnfile} --enable dbg.timing.enabled --enable incremental.load --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-load 2>#{@testset_incr.statsfile}" + cmd = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --enable incremental.save --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-save 2>#{@testset.statsfile}" + cmd_incr = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset_incr.warnfile} --enable dbg.timing.enabled --enable incremental.load --set goblint-dir .goblint-#{@id.sub('/','-')}-incr-load 2>#{@testset_incr.statsfile}" starttime = Time.now run_testset(@testset_incr, cmd, starttime) # apply patch @@ -484,8 +480,8 @@ def create_test_set(lines) end def run () filename = File.basename(@path) - cmd1 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set save_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-save 2>#{@testset.statsfile}" - cmd2 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --conf run/config.json --set save_run '' --set load_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-load 2>#{@testset.statsfile}" + cmd1 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --set save_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-save 2>#{@testset.statsfile}" + cmd2 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --enable dbg.timing.enabled --conf run/config.json --set save_run '' --set load_run run --set goblint-dir .goblint-#{@id.sub('/','-')}-run-load 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd1, starttime) run_testset(@testset, cmd2, starttime) @@ -500,8 +496,8 @@ def create_test_set(lines) end def run () filename = File.basename(@path) - cmd1 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile}0 --enable dbg.debug --set dbg.timing.enabled true --enable witness.yaml.enabled --set goblint-dir .goblint-#{@id.sub('/','-')}-witness1 2>#{@testset.statsfile}0" - cmd2 = "#{$goblint} #{filename} #{@params} #{user_params} #{ENV['gobopt']} 1>#{@testset.warnfile} --set ana.activated[+] unassume --enable dbg.debug --set dbg.timing.enabled true --set witness.yaml.unassume witness.yml --set goblint-dir .goblint-#{@id.sub('/','-')}-witness2 2>#{@testset.statsfile}" + cmd1 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile}0 --enable dbg.debug --set dbg.timing.enabled true --enable witness.yaml.enabled --set goblint-dir .goblint-#{@id.sub('/','-')}-witness1 2>#{@testset.statsfile}0" + cmd2 = "#{$goblint} #{filename} #{@params} #{ENV['gobopt']} 1>#{@testset.warnfile} --set ana.activated[+] unassume --enable dbg.debug --set dbg.timing.enabled true --set witness.yaml.unassume witness.yml --set goblint-dir .goblint-#{@id.sub('/','-')}-witness2 2>#{@testset.statsfile}" starttime = Time.now run_testset(@testset, cmd1, starttime) starttime = Time.now @@ -561,7 +557,6 @@ def run () else Project.new(id, testname, groupname, path, params) end - p.user_params = user_params p.create_test_set(lines) projects << p From 8ea76186c633d795babc37965a990b3ffe382e32 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 21 Jun 2023 13:51:29 +0200 Subject: [PATCH 104/131] =?UTF-8?q?Schreibe=20keine=20assertions=20f=C3=BC?= =?UTF-8?q?r=20dead=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/transform/evalAssert.ml | 39 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index 8265c72a5f..a3541b0a06 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -34,7 +34,7 @@ module EvalAssert = struct let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) - class visitor (ask: ?node:Node.t -> Cil.location -> Queries.ask) = object(self) + class visitor (q: Transform.queries) = object(self) inherit nopCilVisitor val full = GobConfig.get_bool "witness.invariant.full" (* TODO: handle witness.invariant.loop-head *) @@ -53,22 +53,25 @@ module EvalAssert = struct in let make_assert ~node loc lval = - let lvals = match lval with - | None -> CilLval.Set.top () - | Some lval -> CilLval.(Set.singleton lval) - in - let context = {Invariant.default_context with lvals} in - match (ask ~node loc).f (Queries.Invariant context) with - | `Lifted e -> - let es = WitnessUtil.InvariantExp.process_exp e in - let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in - if surroundByAtomic && not (goblintCheck ()) then - let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in - let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in - abegin :: (asserts @ [aend]) - else - asserts - | _ -> [] + if q.must_be_dead node then + [] + else + let lvals = match lval with + | None -> CilLval.Set.top () + | Some lval -> CilLval.(Set.singleton lval) + in + let context = {Invariant.default_context with lvals} in + match (q.ask ~node loc).f (Queries.Invariant context) with + | `Lifted e -> + let es = WitnessUtil.InvariantExp.process_exp e in + let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in + if surroundByAtomic && not (goblintCheck ()) then + let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in + let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in + abegin :: (asserts @ [aend]) + else + asserts + | _ -> [] in let instrument_instructions il s = @@ -143,7 +146,7 @@ module EvalAssert = struct end let transform (q : Transform.queries) file = - visitCilFile (new visitor q.ask) file; + visitCilFile (new visitor q) file; (* Add function declarations before function definitions. This way, asserts may reference functions defined later. *) From cc64c2d010949200654e64ba1023af8ed3227e1b Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 23 Jun 2023 10:40:55 +0200 Subject: [PATCH 105/131] Do not interpret cil __SC_TRACE as RACE --- scripts/update_suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 649e5b4b67..2d0ac784a8 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -286,8 +286,8 @@ def parse_tests (lines) next if obj =~ /^\s*\/\// || obj =~ /^\s*\/\*([^*]|\*+[^*\/])*\*\/$/ todo << i if obj =~ /TODO|SKIP/ tests_line[i] = obj - if obj =~ /RACE/ then - tests[i] = if obj =~ /NORACE/ then "norace" else "race" end + if obj =~ /(?<=\/|\b)RACE/ # do not match RACE with a preceding car unequal '/'' or ' '. Prevents interpreting _SC_TRACE from cil as race + tests[i] = if obj =~ /(?<=\/|\b)NORACE/ then "norace" else "race" end elsif obj =~ /DEADLOCK/ then tests[i] = if obj =~ /NODEADLOCK/ then "nodeadlock" else "deadlock" end elsif obj =~ /WARN/ then From 9001024aee350e8effd16c6479e84ceaff252f93 Mon Sep 17 00:00:00 2001 From: J2000A Date: Mon, 26 Jun 2023 12:25:24 +0200 Subject: [PATCH 106/131] Only consider annotations not part of a word --- scripts/update_suite.rb | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 2d949c2ed4..7a7c2f764b 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -282,24 +282,30 @@ def parse_tests (lines) i = $1.to_i - 1 end next if obj =~ /^\s*\/\// || obj =~ /^\s*\/\*([^*]|\*+[^*\/])*\*\/$/ - todo << i if obj =~ /TODO|SKIP/ + todo << i if obj =~ /(\b|\/)(TODO|SKIP)/ tests_line[i] = obj - if obj =~ /(?<=\/|\b)RACE/ # do not match RACE with a preceding car unequal '/'' or ' '. Prevents interpreting _SC_TRACE from cil as race - tests[i] = if obj =~ /(?<=\/|\b)NORACE/ then "norace" else "race" end - elsif obj =~ /DEADLOCK/ then - tests[i] = if obj =~ /NODEADLOCK/ then "nodeadlock" else "deadlock" end - elsif obj =~ /WARN/ then - tests[i] = if obj =~ /NOWARN/ then "nowarn" else "warn" end - elsif obj =~ /SUCCESS/ then + if obj =~ /(\b|\/)RACE/ then + tests[i] = "race" + elsif obj =~ /(\b|\/)NORACE/ then + tests[i] = "norace" + elsif obj =~ /(\b|\/)DEADLOCK/ then + tests[i] = "deadlock" + elsif obj =~ /(\b|\/)NODEADLOCK/ then + tests[i] = "nodeadlock" + elsif obj =~ /(\b|\/)WARN/ then + tests[i] = "warn" + elsif obj =~ /NOWARN/ then + tests[i] = "nowarn" + elsif obj =~ /(\b|\/)SUCCESS/ then tests[i] = "success" - elsif obj =~ /FAIL/ then + elsif obj =~ /(\b|\/)FAIL/ then tests[i] = "fail" - elsif obj =~ /UNKNOWN/ then + elsif obj =~ /(\b|\/)UNKNOWN/ then tests[i] = "unknown" - elsif obj =~ /(assert|__goblint_check).*\(/ then - if obj =~ /FAIL/ then + elsif obj =~ /(\b|\/)(assert|__goblint_check).*\(/ then + if obj =~ /(\b|\/)FAIL/ then tests[i] = "fail" - elsif obj =~ /UNKNOWN/ then + elsif obj =~ /(\b|\/)UNKNOWN/ then tests[i] = "unknown" else tests[i] = "assert" From 764482c1fce42d80f12027f33b7ed6872fbb405b Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 27 Jun 2023 20:58:28 +0200 Subject: [PATCH 107/131] Add nowarn annotation --- docs/developer-guide/testing.md | 1 + scripts/update_suite.rb | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 0854e4f33d..877cc161c4 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -53,6 +53,7 @@ Comments at the end of other lines indicate the behavior on that line: | `DEADLOCK` | Deadlock warning | Deadlock is possible | Soundness | | `NOWARN` | No warning | — | Precision | | `WARN` | Some warning | — | Soundness | +| `NOFAIL` | Unknown or Success | — | Incremental analysis | #### Other Other useful constructs are the following: diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 7a7c2f764b..67c2d8013a 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -203,6 +203,8 @@ def compare_warnings check.call warnings[idx] != "race" when "nodeadlock" check.call warnings[idx] != "deadlock" + when "nofail" + check.call(warnings[idx] == "success" || warnings[idx] == "unknown") end end end @@ -284,7 +286,9 @@ def parse_tests (lines) next if obj =~ /^\s*\/\// || obj =~ /^\s*\/\*([^*]|\*+[^*\/])*\*\/$/ todo << i if obj =~ /(\b|\/)(TODO|SKIP)/ tests_line[i] = obj - if obj =~ /(\b|\/)RACE/ then + if obj =~ /(\b|\/)NOFAIL/ then + tests[i] = "nofail" + elsif obj =~ /(\b|\/)RACE/ then tests[i] = "race" elsif obj =~ /(\b|\/)NORACE/ then tests[i] = "norace" From bbbfaf7446818726c60155cdb36c850088540b96 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 27 Jun 2023 21:26:48 +0200 Subject: [PATCH 108/131] Update documentation --- docs/developer-guide/testing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 877cc161c4..4eab0c52b4 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -137,6 +137,9 @@ git diff --no-prefix relative/path/to/test.c relative/path/to/test.json > relati The comparison input and the metadata in the patch headers are not necessary and can be removed. +### Generate incremental tests +The Mutation Generator in the bench repository (`bench/incremental-test-generation/`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-test-generation/README.md`). + ## Unit tests ### Running From 075c1a8eb1e54ad7aa99d0be2a6c5cb03b80def0 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 27 Jun 2023 22:59:24 +0200 Subject: [PATCH 109/131] In case of exception print content of warnfile --- scripts/update_suite.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 67c2d8013a..1755fc2257 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -353,6 +353,7 @@ def run_testset (testset, cmd, starttime) lastline = (File.readlines testset.warnfile).last() filename = File.basename(@path) puts lastline.strip().sub filename, relpath(@path).to_s unless lastline.nil? + puts "Content of the warnfile: \n #{File.read(testset.warnfile)}" puts stats[0..9].itemize elsif status == 3 then warn = File.readlines testset.warnfile From b4c84b0240addd8adfc6e07306d91de4c302ebd7 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 28 Jun 2023 20:33:07 +0200 Subject: [PATCH 110/131] Add warn and nothing to results for NOFAIL --- scripts/update_suite.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 1755fc2257..61ec9af0d4 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -204,7 +204,12 @@ def compare_warnings when "nodeadlock" check.call warnings[idx] != "deadlock" when "nofail" - check.call(warnings[idx] == "success" || warnings[idx] == "unknown") + if warnings[idx] then + check.call(warnings[idx] == "success" || warnings[idx] == "unknown" || warnings[idx] == "warn") + else + # When nothing ignore it + @ignored += 1 + end end end end From 63fb6bf7cf74f7f33d1aee54701102af4caf9857 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Wed, 28 Jun 2023 20:48:31 +0200 Subject: [PATCH 111/131] change accepted results to not fail --- scripts/update_suite.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 61ec9af0d4..bec3b024b5 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -205,7 +205,7 @@ def compare_warnings check.call warnings[idx] != "deadlock" when "nofail" if warnings[idx] then - check.call(warnings[idx] == "success" || warnings[idx] == "unknown" || warnings[idx] == "warn") + check.call(warnings[idx] != "fail") else # When nothing ignore it @ignored += 1 From 7e2491ab44759308a96c9d60f5579aec9964858b Mon Sep 17 00:00:00 2001 From: Jonas August Date: Thu, 29 Jun 2023 08:29:30 +0200 Subject: [PATCH 112/131] Fix depreciated option --- tests/regression/46-apron2/26-autotune.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regression/46-apron2/26-autotune.c b/tests/regression/46-apron2/26-autotune.c index 96c5c85278..ec8f55fe19 100644 --- a/tests/regression/46-apron2/26-autotune.c +++ b/tests/regression/46-apron2/26-autotune.c @@ -1,4 +1,4 @@ -//SKIP PARAM: --enable ana.int.interval --sets sem.int.signed_overflow assume_none --set ana.activated[+] apron --enable ana.autotune.enabled +//SKIP PARAM: --enable ana.int.interval --set sem.int.signed_overflow assume_none --set ana.activated[+] apron --enable ana.autotune.enabled // Check that autotuner disables context for apron as well for recursive calls #include From 52ee2f873d388ed6dbce89c2bbfd656791ce3363 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 29 Jun 2023 16:47:58 +0200 Subject: [PATCH 113/131] Check files with linter --- scripts/update_suite.rb | 5 ++--- src/transform/evalAssert.ml | 39 +++++++++++++++++------------------- tests/incremental/.gitignore | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index bec3b024b5..5e6c7f562f 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -303,7 +303,7 @@ def parse_tests (lines) tests[i] = "nodeadlock" elsif obj =~ /(\b|\/)WARN/ then tests[i] = "warn" - elsif obj =~ /NOWARN/ then + elsif obj =~ /(\b|\/)NOWARN/ then tests[i] = "nowarn" elsif obj =~ /(\b|\/)SUCCESS/ then tests[i] = "success" @@ -358,7 +358,7 @@ def run_testset (testset, cmd, starttime) lastline = (File.readlines testset.warnfile).last() filename = File.basename(@path) puts lastline.strip().sub filename, relpath(@path).to_s unless lastline.nil? - puts "Content of the warnfile: \n #{File.read(testset.warnfile)}" + puts "Content of the warn-file: \n #{File.read(testset.warnfile)}" puts stats[0..9].itemize elsif status == 3 then warn = File.readlines testset.warnfile @@ -550,7 +550,6 @@ def run () end lines[0] =~ /PARAM: (.*)$/ if $1 then params << " #{$1}" else params << "" end - # always enable debugging so that the warnings would work params << " --set warn.debug true" p = if incremental then patch = f[0..-3] + ".patch" diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index 93cb845a9d..fdce18e2fa 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -35,7 +35,7 @@ module EvalAssert = struct let atomicEnd = makeVarinfo true "__VERIFIER_atomic_end" (TVoid []) - class visitor (q: Transform.queries) = object(self) + class visitor (ask: ?node:Node.t -> Cil.location -> Queries.ask) = object(self) inherit nopCilVisitor val full = GobConfig.get_bool "witness.invariant.full" (* TODO: handle witness.invariant.loop-head *) @@ -54,25 +54,22 @@ module EvalAssert = struct in let make_assert ~node loc lval = - if q.must_be_dead node then (* Has currently no affect. Maybe the results are not avaiable yet?! *) - [] - else - let lvals = match lval with - | None -> Lval.Set.top () - | Some lval -> Lval.(Set.singleton lval) - in - let context = {Invariant.default_context with lvals} in - match (q.ask ~node loc).f (Queries.Invariant context) with - | `Lifted e -> - let es = WitnessUtil.InvariantExp.process_exp e in - let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in - if surroundByAtomic && not (goblintCheck ()) then - let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in - let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in - abegin :: (asserts @ [aend]) - else - asserts - | _ -> [] + let lvals = match lval with + | None -> Lval.Set.top () + | Some lval -> Lval.(Set.singleton lval) + in + let context = {Invariant.default_context with lvals} in + match (q.ask ~node loc).f (Queries.Invariant context) with + | `Lifted e -> + let es = WitnessUtil.InvariantExp.process_exp e in + let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in + if surroundByAtomic && not (goblintCheck ()) then + let abegin = (cInstr ("%v:__VERIFIER_atomic_begin();") loc [("__VERIFIER_atomic_begin", Fv atomicBegin)]) in + let aend = (cInstr ("%v:__VERIFIER_atomic_end();") loc [("__VERIFIER_atomic_end", Fv atomicEnd)]) in + abegin :: (asserts @ [aend]) + else + asserts + | _ -> [] in let instrument_instructions il s = @@ -147,7 +144,7 @@ module EvalAssert = struct end let transform (q : Transform.queries) file = - visitCilFile (new visitor q) file; + visitCilFile (new visitor q.ask) file; (* Add function declarations before function definitions. This way, asserts may reference functions defined later. *) diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore index f9c88fa39b..51459b1a01 100644 --- a/tests/incremental/.gitignore +++ b/tests/incremental/.gitignore @@ -1,2 +1,2 @@ # Do not consider temporary files for the running of mutation generator -99-temp \ No newline at end of file +99-temp From 3ba91494f46af3a577d7ab729c5df01fbd0b6408 Mon Sep 17 00:00:00 2001 From: J2000A Date: Thu, 29 Jun 2023 16:50:31 +0200 Subject: [PATCH 114/131] Fix --- src/transform/evalAssert.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index fdce18e2fa..96ce12b2d8 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -59,7 +59,7 @@ module EvalAssert = struct | Some lval -> Lval.(Set.singleton lval) in let context = {Invariant.default_context with lvals} in - match (q.ask ~node loc).f (Queries.Invariant context) with + match (ask ~node loc).f (Queries.Invariant context) with | `Lifted e -> let es = WitnessUtil.InvariantExp.process_exp e in let asserts = List.map (fun e -> cInstr ("%v:assert (%e:exp);") loc [("assert", Fv (ass ())); ("exp", Fe e)]) es in From e11b1353f72f7010b1fe466505dc656007ad9a9a Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 7 Jul 2023 19:18:20 +0200 Subject: [PATCH 115/131] Fix << for nil error --- scripts/update_suite.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 5e6c7f562f..0802890667 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -547,6 +547,8 @@ def run () if incremental then config_path = File.expand_path(f[0..-3] + ".json", grouppath) params = if cfg then "--conf #{config_path} --set incremental.compare cfg" else "--conf #{config_path}" end + else + params = "" end lines[0] =~ /PARAM: (.*)$/ if $1 then params << " #{$1}" else params << "" end From 4bc17292fa43421ee51dab3bd44e266e2149ddc4 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 7 Jul 2023 19:19:51 +0200 Subject: [PATCH 116/131] Revive removed comment --- scripts/update_suite.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 0802890667..ad62578728 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -552,6 +552,7 @@ def run () end lines[0] =~ /PARAM: (.*)$/ if $1 then params << " #{$1}" else params << "" end + # always enable debugging so that the warnings would work params << " --set warn.debug true" p = if incremental then patch = f[0..-3] + ".patch" From 1c5871f5638e1bc792c18f12bf3134b51a0717d0 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 11 Jul 2023 18:35:11 +0200 Subject: [PATCH 117/131] Add annotation not imprecise --- scripts/update_suite.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index ad62578728..61976184ff 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -210,6 +210,13 @@ def compare_warnings # When nothing ignore it @ignored += 1 end + when "notinprecise" + if warnings[idx] then + check.call(warnings[idx] != "unknown") + else + # When nothing ignore it + @ignored += 1 + end end end end @@ -293,6 +300,8 @@ def parse_tests (lines) tests_line[i] = obj if obj =~ /(\b|\/)NOFAIL/ then tests[i] = "nofail" + elsif obj =~ /(\b|\/)NOTINPRECISE/ then + tests[i] = "notinprecise" elsif obj =~ /(\b|\/)RACE/ then tests[i] = "race" elsif obj =~ /(\b|\/)NORACE/ then From 184e9004dba25788d4bb7a9a0bd4157ae72e4217 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 11 Jul 2023 20:39:10 +0200 Subject: [PATCH 118/131] Add annotation not imprecise to documentation --- docs/developer-guide/testing.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 4eab0c52b4..19e0e11a58 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -53,7 +53,8 @@ Comments at the end of other lines indicate the behavior on that line: | `DEADLOCK` | Deadlock warning | Deadlock is possible | Soundness | | `NOWARN` | No warning | — | Precision | | `WARN` | Some warning | — | Soundness | -| `NOFAIL` | Unknown or Success | — | Incremental analysis | +| `NOFAIL` | Unknown or Success | Everything except fail | Incremental analysis | +| `NOTINPRECISE` | Success or Fail | Everything except unknown | Incremental analysis precision | #### Other Other useful constructs are the following: From 5de6d6c77722fc70a65522c636844b23f3157f29 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 11 Jul 2023 20:49:26 +0200 Subject: [PATCH 119/131] Rephrase documentation --- docs/developer-guide/testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 19e0e11a58..5cce5714f8 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -53,8 +53,8 @@ Comments at the end of other lines indicate the behavior on that line: | `DEADLOCK` | Deadlock warning | Deadlock is possible | Soundness | | `NOWARN` | No warning | — | Precision | | `WARN` | Some warning | — | Soundness | -| `NOFAIL` | Unknown or Success | Everything except fail | Incremental analysis | -| `NOTINPRECISE` | Success or Fail | Everything except unknown | Incremental analysis precision | +| `NOFAIL` | Assertion is unknown
or succeeds | Everything except fail | Incremental analysis | +| `NOTINPRECISE` | Assertion succeeds
or fails | Everything except unknown | Incremental analysis
precision | #### Other Other useful constructs are the following: From 757b85c0ce3c88179a23aaeb5657c8b1776554df Mon Sep 17 00:00:00 2001 From: J2000A Date: Tue, 18 Jul 2023 20:40:23 +0200 Subject: [PATCH 120/131] Remove ignoring of nothing --- scripts/update_suite.rb | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 61976184ff..9ceea3c0dd 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -204,19 +204,9 @@ def compare_warnings when "nodeadlock" check.call warnings[idx] != "deadlock" when "nofail" - if warnings[idx] then - check.call(warnings[idx] != "fail") - else - # When nothing ignore it - @ignored += 1 - end + check.call(warnings[idx] != "fail") when "notinprecise" - if warnings[idx] then - check.call(warnings[idx] != "unknown") - else - # When nothing ignore it - @ignored += 1 - end + check.call(warnings[idx] != "unknown") end end end From fddcfb1fc8bee3802d2d53016d8f26f073c11393 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 09:04:41 +0200 Subject: [PATCH 121/131] Change MAYFAIL to MAY FAIL --- tests/regression/58-base-mm-tid/15-branched-thread-creation.c | 2 +- .../67-interval-sets-two/16-branched-thread-creation.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regression/58-base-mm-tid/15-branched-thread-creation.c b/tests/regression/58-base-mm-tid/15-branched-thread-creation.c index 3292182adc..abe59039ef 100644 --- a/tests/regression/58-base-mm-tid/15-branched-thread-creation.c +++ b/tests/regression/58-base-mm-tid/15-branched-thread-creation.c @@ -41,7 +41,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAYFAIL + __goblint_check(g==h); //MAY FAIL pthread_mutex_unlock(&mutex); } diff --git a/tests/regression/67-interval-sets-two/16-branched-thread-creation.c b/tests/regression/67-interval-sets-two/16-branched-thread-creation.c index c309ec8ffd..25369b4396 100644 --- a/tests/regression/67-interval-sets-two/16-branched-thread-creation.c +++ b/tests/regression/67-interval-sets-two/16-branched-thread-creation.c @@ -41,7 +41,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAYFAIL + __goblint_check(g==h); //MAY FAIL pthread_mutex_unlock(&mutex); } From 1c5c35521f0be555f5c6f92a08004397cf09a928 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 09:24:17 +0200 Subject: [PATCH 122/131] Update name to IATT --- tests/incremental/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore index 51459b1a01..c9d22a0a58 100644 --- a/tests/incremental/.gitignore +++ b/tests/incremental/.gitignore @@ -1,2 +1,2 @@ -# Do not consider temporary files for the running of mutation generator +# Do not consider temporary files for the running of the "Incremental Analysis Test Toolchain" 99-temp From 8aabe2d5ad133c866307aef61de322cdc682dba0 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 09:30:37 +0200 Subject: [PATCH 123/131] Update name to IATT --- docs/developer-guide/testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 5cce5714f8..7fff1c9981 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -138,8 +138,8 @@ git diff --no-prefix relative/path/to/test.c relative/path/to/test.json > relati The comparison input and the metadata in the patch headers are not necessary and can be removed. -### Generate incremental tests -The Mutation Generator in the bench repository (`bench/incremental-test-generation/`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-test-generation/README.md`). +### Incremental Analysis Test Toolchain - IATT +The "Incremental Analysis Test Toolchain" in the bench repository (`bench/incremental-test-generation/`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-test-generation/README.md`). ## Unit tests From d3835f2fa1e136c8469e021711ef32d208544c55 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 09:32:35 +0200 Subject: [PATCH 124/131] Update paths in readme --- docs/developer-guide/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 7fff1c9981..6aac58961e 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -139,7 +139,7 @@ git diff --no-prefix relative/path/to/test.c relative/path/to/test.json > relati The comparison input and the metadata in the patch headers are not necessary and can be removed. ### Incremental Analysis Test Toolchain - IATT -The "Incremental Analysis Test Toolchain" in the bench repository (`bench/incremental-test-generation/`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-test-generation/README.md`). +The "Incremental Analysis Test Toolchain" in the bench repository (`bench/incremental-analysis-test-toolchain`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-analysis-test-toolchain/README.md`). ## Unit tests From cb0ddb62e3ec76029f654fc300aab07820287b3c Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 10:03:38 +0200 Subject: [PATCH 125/131] Change MAYFAIL to MAY FAIL --- tests/regression/36-apron/86-branched-thread-creation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regression/36-apron/86-branched-thread-creation.c b/tests/regression/36-apron/86-branched-thread-creation.c index cac0a881a6..7d2a662035 100644 --- a/tests/regression/36-apron/86-branched-thread-creation.c +++ b/tests/regression/36-apron/86-branched-thread-creation.c @@ -40,7 +40,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAYFAIL + __goblint_check(g==h); //MAY FAIL pthread_mutex_unlock(&mutex); } From 558b68e791a8807e3f64de4b3a5c068b901ad8c1 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 12:12:07 +0200 Subject: [PATCH 126/131] Correct description --- src/util/options.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/options.schema.json b/src/util/options.schema.json index 893dd18e1b..2f79a1e509 100644 --- a/src/util/options.schema.json +++ b/src/util/options.schema.json @@ -1495,7 +1495,7 @@ }, "goblint-check" : { "title": "trans.goblint-check", - "description": "Write __Goblint_Check(exp) in the file instead of __VERIFIER_assert(exp).", + "description": "Write __goblint_check(exp) in the file instead of __VERIFIER_assert(exp).", "type": "boolean", "default": false } From 840d587038b077188c74c327cdb1f3769f0eacd2 Mon Sep 17 00:00:00 2001 From: Jonas August Date: Tue, 25 Jul 2023 19:34:44 +0200 Subject: [PATCH 127/131] Update Name to TAIA --- docs/developer-guide/testing.md | 4 ++-- tests/incremental/.gitignore | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 6aac58961e..3757a83a00 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -138,8 +138,8 @@ git diff --no-prefix relative/path/to/test.c relative/path/to/test.json > relati The comparison input and the metadata in the patch headers are not necessary and can be removed. -### Incremental Analysis Test Toolchain - IATT -The "Incremental Analysis Test Toolchain" in the bench repository (`bench/incremental-analysis-test-toolchain`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-analysis-test-toolchain/README.md`). +### Test Automation for Incremental Analysis - TAIA +The "Test Automation for Incremental Analysis" in the bench repository (`bench/incremental-analysis-test-toolchain`) enables you to generate and run incremental tests based on one single c file as input. You find more details in the readme file in the repository (`bench/incremental-analysis-test-toolchain/README.md`). ## Unit tests diff --git a/tests/incremental/.gitignore b/tests/incremental/.gitignore index c9d22a0a58..76d849fa56 100644 --- a/tests/incremental/.gitignore +++ b/tests/incremental/.gitignore @@ -1,2 +1,2 @@ -# Do not consider temporary files for the running of the "Incremental Analysis Test Toolchain" +# Do not consider temporary files for the running of the "Test Automation for Incremental Analysis" 99-temp From 9bf295917c019339bbe409eab35ffc3440a16025 Mon Sep 17 00:00:00 2001 From: J2000A Date: Fri, 4 Aug 2023 09:32:34 +0200 Subject: [PATCH 128/131] Ignore extern definitions --- scripts/update_suite.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 9ceea3c0dd..1d9257a696 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -311,12 +311,14 @@ def parse_tests (lines) elsif obj =~ /(\b|\/)UNKNOWN/ then tests[i] = "unknown" elsif obj =~ /(\b|\/)(assert|__goblint_check).*\(/ then - if obj =~ /(\b|\/)FAIL/ then - tests[i] = "fail" - elsif obj =~ /(\b|\/)UNKNOWN/ then - tests[i] = "unknown" - else - tests[i] = "assert" + unless obj =~ /^\s*extern\b/ + if obj =~ /(\b|\/)FAIL/ then + tests[i] = "fail" + elsif obj =~ /(\b|\/)UNKNOWN/ then + tests[i] = "unknown" + else + tests[i] = "assert" + end end end end From 692e8a0e7c5d7257e1bd2b7b9bbee53b384ee705 Mon Sep 17 00:00:00 2001 From: stilscher <66023521+stilscher@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:24:31 +0100 Subject: [PATCH 129/131] remove notinprecise annotation --- docs/developer-guide/testing.md | 1 - scripts/update_suite.rb | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/developer-guide/testing.md b/docs/developer-guide/testing.md index 3757a83a00..87c97cc5d1 100644 --- a/docs/developer-guide/testing.md +++ b/docs/developer-guide/testing.md @@ -54,7 +54,6 @@ Comments at the end of other lines indicate the behavior on that line: | `NOWARN` | No warning | — | Precision | | `WARN` | Some warning | — | Soundness | | `NOFAIL` | Assertion is unknown
or succeeds | Everything except fail | Incremental analysis | -| `NOTINPRECISE` | Assertion succeeds
or fails | Everything except unknown | Incremental analysis
precision | #### Other Other useful constructs are the following: diff --git a/scripts/update_suite.rb b/scripts/update_suite.rb index 1d9257a696..07f06601f7 100755 --- a/scripts/update_suite.rb +++ b/scripts/update_suite.rb @@ -204,9 +204,7 @@ def compare_warnings when "nodeadlock" check.call warnings[idx] != "deadlock" when "nofail" - check.call(warnings[idx] != "fail") - when "notinprecise" - check.call(warnings[idx] != "unknown") + check.call(warnings[idx] != "fail") end end end @@ -289,9 +287,7 @@ def parse_tests (lines) todo << i if obj =~ /(\b|\/)(TODO|SKIP)/ tests_line[i] = obj if obj =~ /(\b|\/)NOFAIL/ then - tests[i] = "nofail" - elsif obj =~ /(\b|\/)NOTINPRECISE/ then - tests[i] = "notinprecise" + tests[i] = "nofail" elsif obj =~ /(\b|\/)RACE/ then tests[i] = "race" elsif obj =~ /(\b|\/)NORACE/ then From 474010b3d685d3fcf0d1d5534b24e6bc2bb33d7b Mon Sep 17 00:00:00 2001 From: stilscher <66023521+stilscher@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:26:31 +0100 Subject: [PATCH 130/131] change may fail to fail annotation --- tests/regression/36-apron/86-branched-thread-creation.c | 2 +- tests/regression/58-base-mm-tid/15-branched-thread-creation.c | 2 +- .../67-interval-sets-two/16-branched-thread-creation.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/regression/36-apron/86-branched-thread-creation.c b/tests/regression/36-apron/86-branched-thread-creation.c index 7d2a662035..91a5411e8f 100644 --- a/tests/regression/36-apron/86-branched-thread-creation.c +++ b/tests/regression/36-apron/86-branched-thread-creation.c @@ -40,7 +40,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAY FAIL + __goblint_check(g==h); //FAIL pthread_mutex_unlock(&mutex); } diff --git a/tests/regression/58-base-mm-tid/15-branched-thread-creation.c b/tests/regression/58-base-mm-tid/15-branched-thread-creation.c index abe59039ef..c7bb43519f 100644 --- a/tests/regression/58-base-mm-tid/15-branched-thread-creation.c +++ b/tests/regression/58-base-mm-tid/15-branched-thread-creation.c @@ -41,7 +41,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAY FAIL + __goblint_check(g==h); //FAIL pthread_mutex_unlock(&mutex); } diff --git a/tests/regression/67-interval-sets-two/16-branched-thread-creation.c b/tests/regression/67-interval-sets-two/16-branched-thread-creation.c index 25369b4396..d97f800621 100644 --- a/tests/regression/67-interval-sets-two/16-branched-thread-creation.c +++ b/tests/regression/67-interval-sets-two/16-branched-thread-creation.c @@ -41,7 +41,7 @@ int main(void) { if(!mt) { pthread_mutex_lock(&mutex); - __goblint_check(g==h); //MAY FAIL + __goblint_check(g==h); //FAIL pthread_mutex_unlock(&mutex); } From 40a9f80958d3fdc981c98a7ca0c73bb46cfaf120 Mon Sep 17 00:00:00 2001 From: stilscher <66023521+stilscher@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:37:44 +0100 Subject: [PATCH 131/131] rename option --- src/transform/evalAssert.ml | 2 +- src/util/options.schema.json | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/transform/evalAssert.ml b/src/transform/evalAssert.ml index 96ce12b2d8..a8d8e5d087 100644 --- a/src/transform/evalAssert.ml +++ b/src/transform/evalAssert.ml @@ -27,7 +27,7 @@ module EvalAssert = struct let surroundByAtomic = true (* variable for generating __goblint_check(exp) instead of __VERIFIER_assert(exp) *) - let goblintCheck () = GobConfig.get_bool "trans.goblint-check" + let goblintCheck () = GobConfig.get_bool "trans.assert.goblint-check" (* Cannot use Cilfacade.name_fundecs as assert() is external and has no fundec *) let ass () = if goblintCheck () then makeVarinfo true "__goblint_check" (TVoid []) else makeVarinfo true "__VERIFIER_assert" (TVoid []) diff --git a/src/util/options.schema.json b/src/util/options.schema.json index 2f79a1e509..c2b895d608 100644 --- a/src/util/options.schema.json +++ b/src/util/options.schema.json @@ -1493,11 +1493,19 @@ "type":"string", "default": "transformed.c" }, - "goblint-check" : { - "title": "trans.goblint-check", - "description": "Write __goblint_check(exp) in the file instead of __VERIFIER_assert(exp).", - "type": "boolean", - "default": false + "assert" : { + "title": "trans.assert", + "type": "object", + "properties": { + "goblint-check": { + "title": "trans.assert.goblint-check", + "description": + "Write __goblint_check(exp) in the file instead of __VERIFIER_assert(exp).", + "type": "boolean", + "default": false + } + }, + "additionalProperties": false } }, "additionalProperties": false