Skip to content

Commit

Permalink
added better logging and resuming from given last path
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cornul11
Cornul11 committed Feb 26, 2024
1 parent 560709c commit 56b184d
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 33 deletions.
2 changes: 1 addition & 1 deletion src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/CommandExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void run() {
String evaluationDirectory = options.getEvaluationDirectory();
if (evaluationDirectory != null) {
JarEvaluator jarEvaluator = new JarEvaluator(signatureDao, evaluationDirectory);
Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = jarEvaluator.inferLibrariesFromJars();
Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = jarEvaluator.inferLibrariesFromJars(options.getLastPath());
jarEvaluator.evaluate(inferredLibrariesMap);
} else {
System.out.println("Evaluation directory is required for EVALUATION_MODE");
Expand Down
100 changes: 68 additions & 32 deletions src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/JarEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,51 +40,81 @@ public void evaluate(Map<String, List<JarEvaluator.InferredLibrary>> inferredLib
}
}

public Map<String, List<JarEvaluator.InferredLibrary>> inferLibrariesFromJars() {
public Map<String, List<JarEvaluator.InferredLibrary>> inferLibrariesFromJars(String resumePath) {
boolean resumeProcessing = (resumePath == null || resumePath.isEmpty());
Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = new HashMap<>();

File[] projectFolders = getProjectFolders();
if (projectFolders == null) {
logger.warn("No projects found in {}", evaluationDirectory);
return inferredLibrariesMap;
}


try {
Map<String, List<InferredLibrary>> loadedLibraries = loadInferredLibraries();
if (!loadedLibraries.isEmpty()) {
logger.info("Loaded inferred libraries from file");
logger.info("Loaded {} inferred libraries from file out of {}", loadedLibraries.size(), projectFolders.length);
return loadedLibraries;
}
} catch (IOException e) {
logger.warn("Failed to load inferred libraries from file. Re-inferring them", e);
}

Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = new HashMap<>();

File[] projectFolders = getProjectFolders();
if (projectFolders == null) return inferredLibrariesMap;
int total = projectFolders.length;
logger.info("Inferring libraries from {} projects", total);

int current = 0;
int failedCount = 0;
for (File projectFolder : projectFolders) {
String projectName = projectFolder.getName();
String jarPath = Paths.get(projectFolder.getAbsolutePath(), "target", projectName + "-1.0-SNAPSHOT.jar").toString();
Path jarFilePath = Path.of(jarPath);

if (!resumeProcessing) {
if (!jarPath.equals(resumePath)) {
current++;
continue;
}
resumeProcessing = true;
}

current++;

Path jarFilePath = Path.of(jarPath);
boolean processedSuccessfully = true;
if (!jarFilePath.toFile().exists()) {
logger.info("Skipping " + jarPath + " because it does not exist");
continue;
processedSuccessfully = false;
} else {
List<SignatureDAOImpl.LibraryCandidate> libraryCandidates = jarSignatureMapper.inferJarFileMultithreadedProcess(jarFilePath);
if (libraryCandidates == null) {
logger.warn("No class file signatures were retrieved from {}", jarFilePath);
processedSuccessfully = false;
} else {
List<JarEvaluator.InferredLibrary> candidateLibraries = libraryCandidates.stream()
.map(libraryCandidate -> new JarEvaluator.InferredLibrary(libraryCandidate, jarPath))
.collect(Collectors.toList());

inferredLibrariesMap.put(jarPath, candidateLibraries);

try {
storeInferredLibraries(projectName, candidateLibraries);
} catch (IOException e) {
logger.error("Failed to store inferred libraries to file", e);
}
}
}

List<SignatureDAOImpl.LibraryCandidate> libraryCandidates = jarSignatureMapper.inferJarFileMultithreadedProcess(jarFilePath);
if (libraryCandidates == null) {
logger.warn("No class file signatures were retrieved from {}", jarFilePath);
continue;
if (!processedSuccessfully) {
failedCount++;
}

List<JarEvaluator.InferredLibrary> candidateLibraries = libraryCandidates.stream()
.map(libraryCandidate -> new JarEvaluator.InferredLibrary(libraryCandidate, jarPath))
.collect(Collectors.toList());


inferredLibrariesMap.put(jarPath, candidateLibraries);
double percentageCompleted = ((double) current / total) * 100;
logger.info("Processed {} ({} failed) out of {} ({}% completed)",
current, failedCount, total,
String.format("%.2f", percentageCompleted));
}

try {
storeInferredLibraries(inferredLibrariesMap);
} catch (IOException e) {
logger.error("Failed to store inferred libraries to file", e);
}
return inferredLibrariesMap;
}

Expand Down Expand Up @@ -146,14 +176,14 @@ private List<JarEvaluator.InferredLibrary> filterLibrariesByThreshold(List<JarEv
return filteredList;
}

private void storeInferredLibraries(Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap) throws IOException {
Path filePath = Paths.get(evaluationDirectory, "evaluation", "inferredLibraries.json");
private void storeInferredLibraries(String projectName, List<JarEvaluator.InferredLibrary> inferredLibraries) throws IOException {
Path filePath = Paths.get(evaluationDirectory, "evaluation", projectName + "_inferredLibraries.json");

if (!filePath.getParent().toFile().exists()) {
filePath.getParent().toFile().mkdirs();
}

objectMapper.writerWithDefaultPrettyPrinter().writeValue(filePath.toFile(), inferredLibrariesMap);
objectMapper.writerWithDefaultPrettyPrinter().writeValue(filePath.toFile(), inferredLibraries);
logger.info("Stored inferred libraries to {}", filePath);
}

Expand Down Expand Up @@ -249,13 +279,19 @@ public void setAlternativeVersions(List<String> alternativeVersions) {
}

private Map<String, List<InferredLibrary>> loadInferredLibraries() throws IOException {
Path filePath = Paths.get(evaluationDirectory, "evaluation", "inferredLibraries.json");
if (filePath.toFile().exists()) {
logger.info("Loading inferred libraries from {}", filePath);
return objectMapper.readValue(filePath.toFile(), new TypeReference<>() {
});
Map<String, List<JarEvaluator.InferredLibrary>> allInferredLibraries = new HashMap<>();
File evaluationDir = new File(evaluationDirectory, "evaluation");

File[] files = evaluationDir.listFiles((dir, name) -> name.endsWith("_inferredLibraries.json"));
if (files != null) {
for (File file : files) {
String projectName = file.getName().replace("_inferredLibraries.json", "");
List<JarEvaluator.InferredLibrary> inferredLibraries = objectMapper.readValue(file, new TypeReference<>() {});
allInferredLibraries.put(projectName, inferredLibraries);
logger.info("Loaded inferred libraries {} from {}", projectName, file.getPath());
}
}
return new HashMap<>();
return allInferredLibraries;
}

private File[] getProjectFolders() {
Expand Down

0 comments on commit 56b184d

Please sign in to comment.