Address #518 and silence output for normal conditions

phaag · Apr 2, 2024 · f462e2b · f462e2b
1 parent b064481
commit f462e2b
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 39 deletions.
diff --git a/src/include/nfdump.h b/src/include/nfdump.h
@@ -132,6 +132,6 @@ typedef struct stat_record_s {
 // allocate space for this number of workers
 #define MAXWORKERS 64
 // If number of CPUs can not be determined
-#define DEFAULTWORKERS 4
+#define DEFAULTWORKERS 2
 
 #endif  //_NFDUMP_H
diff --git a/src/libnffile/barrier.c b/src/libnffile/barrier.c
@@ -42,33 +42,44 @@
 #include <unistd.h>
 
 #include "barrier.h"
+#include "nfconf.h"
 #include "nfdump.h"
 #include "util.h"
 
 // get decent number of workers depending
 // on the number of cores online
 uint32_t GetNumWorkers(uint32_t requested) {
-    long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN);
+    // get conf value for maxworkers
+    int confMaxWorkers = ConfGetValue("maxworkers");
+    if (confMaxWorkers == 0) confMaxWorkers = DEFAULTWORKERS;
 
+    long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN);
     if (CoresOnline < 0) {
         LogError("sysconf(_SC_NPROCESSORS_ONLN) error in %s line %d: %s", __FILE__, __LINE__, strerror(errno));
         CoresOnline = 1;
     }
 
-    uint32_t numWorkers = DEFAULTWORKERS;
-    if (requested && requested < CoresOnline) {
-        numWorkers = requested;
-    } else {
-        numWorkers = CoresOnline;
+    // no more than cores online
+    if (requested && (requested > CoresOnline)) {
+        LogError("Number of workers should not be greater than number of cores online. %d is > %d", requested, CoresOnline);
     }
 
-    // limit to MAXWORKERS
-    if (numWorkers > MAXWORKERS) numWorkers = DEFAULTWORKERS;
-
-    LogInfo("CoresOnline: %lld, Requested: %u, Set workers to: %u", CoresOnline, requested, numWorkers);
+    // try to find optimal number of workers
+    // if nothing requested, use default, unless we are on a high number of cores
+    // system, so double the workers
+    if (requested == 0) {
+        requested = confMaxWorkers;
+        if ((2 * requested) < CoresOnline) requested = 2 * requested;
+    }
+    if (requested > CoresOnline) requested = CoresOnline;
 
-    return numWorkers;
+    // no more than internal array limit
+    if (requested > MAXWORKERS) {
+        LogError("Number of workers is limited to %s", MAXWORKERS);
+        requested = MAXWORKERS;
+    }
 
+    return requested;
 }  // End of GetNumWorkers
 
 // initialize barrier for numWorkers + 1 controller

diff --git a/src/libnffile/nffile.c b/src/libnffile/nffile.c
@@ -74,8 +74,8 @@
 #include "lz4.h"
 #include "lz4hc.h"
 #endif
+#include "barrier.h"
 #include "minilzo.h"
-#include "nfconf.h"
 #include "nfdump.h"
 #include "nffileV2.h"
 #include "util.h"
@@ -169,32 +169,7 @@ int Init_nffile(int workers, queue_t *fileList) {
 
     atomic_init(&blocksInUse, 0);
 
-    // get conf value for maxworkers
-    int confMaxWorkers = ConfGetValue("maxworkers");
-    if (confMaxWorkers == 0) confMaxWorkers = DEFAULTWORKERS;
-
-    long CoresOnline = sysconf(_SC_NPROCESSORS_ONLN);
-    if (CoresOnline < 0) {
-        LogError("sysconf(_SC_NPROCESSORS_ONLN) error in %s line %d: %s", __FILE__, __LINE__, strerror(errno));
-        CoresOnline = 1;
-    }
-
-    // no more than cores online
-    if (workers && (workers > CoresOnline)) {
-        LogError("Number of workers should not be greater than number of cores online. %d is > %d", workers, CoresOnline);
-    }
-
-    // set to default if not set
-    if (workers == 0) workers = confMaxWorkers;
-    if (workers > CoresOnline) workers = CoresOnline;
-
-    // no more than internal array limit
-    if (workers > MAXWORKERS) {
-        LogError("Number of workers is limited to %s", MAXWORKERS);
-        workers = MAXWORKERS;
-    }
-
-    NumWorkers = workers;
+    NumWorkers = GetNumWorkers(workers);
     return 1;
 
 }  // End of Init_nffile