Include file size and modified date of inputs in tmp file hash calcul…

…ation #372
soedinglab · Nov 12, 2020 · 45c4de7 · 45c4de7
1 parent cc47254
commit 45c4de7
Show file tree

Hide file tree

Showing 21 changed files with 64 additions and 25 deletions.
diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp
@@ -2289,18 +2289,57 @@ std::vector<MMseqsParameter*> Parameters::combineList(const std::vector<MMseqsPa
     return retVec;
 }
 
-size_t Parameters::hashParameter(const std::vector<std::string> &filenames, const std::vector<MMseqsParameter*> &par){
+size_t Parameters::hashParameter(const std::vector<DbType> &dbtypes, const std::vector<std::string> &filenames, const std::vector<MMseqsParameter*> &par){
     std::string hashString;
     hashString.reserve(1024);
+
+    struct stat stat_buf;
+    bool stopAfterVariadic = false;
     for (size_t i = 0; i < filenames.size(); ++i){
         hashString.append(filenames[i]);
-        hashString.append(" ");
+        if (stopAfterVariadic == false && i < dbtypes.size()) {
+            const DbType& type = dbtypes[i];
+            if (type.accessMode != DbType::ACCESS_MODE_INPUT) {
+                continue;
+            }
+
+            if (type.specialType & DbType::VARIADIC) {
+                stopAfterVariadic = true;
+            }
+
+            if (filenames[i] == "stdin") {
+                continue;
+            }
+
+            if (::stat(filenames[i].c_str(), &stat_buf) == 0) {
+                hashString.append(SSTR(stat_buf.st_size));
+#ifdef __APPLE__
+                hashString.append(SSTR(stat_buf.st_mtimespec.tv_sec));
+#else
+                hashString.append(SSTR(stat_buf.st_mtime));
+#endif
+                continue;
+            }
+
+            std::string index(filenames[i]);
+            index.append(".index");
+            if (::stat(index.c_str(), &stat_buf) == 0) {
+                hashString.append(SSTR(stat_buf.st_size));
+#ifdef __APPLE__
+                hashString.append(SSTR(stat_buf.st_mtimespec.tv_sec));
+#else
+                hashString.append(SSTR(stat_buf.st_mtime));
+#endif
+                continue;
+            }
+        }
     }
     hashString.append(createParameterString(par));
     hashString.append(version);
     for (int i = 0; i < restArgc; ++i) {
         hashString.append(restArgv[i]);
     }
+    Debug(Debug::ERROR) << hashString << "\n";
     return Util::hash(hashString.c_str(), hashString.size());
 }
 

diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h
@@ -1040,7 +1040,7 @@ class Parameters {
     std::vector<MMseqsParameter*> combineList(const std::vector<MMseqsParameter*> &par1,
                                              const std::vector<MMseqsParameter*> &par2);
 
-    size_t hashParameter(const std::vector<std::string> &filenames, const std::vector<MMseqsParameter*> &par);
+    size_t hashParameter(const std::vector<DbType> &dbtypes, const std::vector<std::string> &filenames, const std::vector<MMseqsParameter*> &par);
 
     std::string createParameterString(const std::vector<MMseqsParameter*> &vector, bool wasSet = false);
 

diff --git a/src/multihit/MultiHitDb.cpp b/src/multihit/MultiHitDb.cpp
@@ -28,7 +28,7 @@ int multihitdb(int argc, const char **argv, const Command &command) {
             Debug(Debug::INFO) << "Created dir " << tmpDir << "\n";
         }
     }
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.multihitdb));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.multihitdb));
     if(par.reuseLatest == true){
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest" );
     }

diff --git a/src/multihit/MultiHitSearch.cpp b/src/multihit/MultiHitSearch.cpp
@@ -56,7 +56,7 @@ int multihitsearch(int argc, const char **argv, const Command &command) {
             Debug(Debug::INFO) << "Created dir " << par.db4 << "\n";
         }
     }
-    size_t hash = par.hashParameter(par.filenames, par.multihitsearch);
+    size_t hash = par.hashParameter(command.databases, par.filenames, par.multihitsearch);
     std::string tmpDir = par.db4 + "/" + SSTR(hash);
     if (FileUtil::directoryExists(tmpDir.c_str()) == false) {
         if (FileUtil::makeDir(tmpDir.c_str()) == false) {

diff --git a/src/workflow/Cluster.cpp b/src/workflow/Cluster.cpp
@@ -130,7 +130,7 @@ int clusteringworkflow(int argc, const char **argv, const Command& command) {
     setClusterAutomagicParameters(par);
 
     std::string tmpDir = par.db3;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.clusterworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.clusterworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/ClusterUpdate.cpp b/src/workflow/ClusterUpdate.cpp
@@ -68,7 +68,7 @@ int clusterupdate(int argc, const char **argv, const Command& command) {
     par.maxAccept = maxAccept;
 
     std::string tmpDir = par.db6;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.clusterUpdate));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.clusterUpdate));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/CreateIndex.cpp b/src/workflow/CreateIndex.cpp
@@ -11,7 +11,7 @@
 #include <cassert>
 #include <climits>
 
-int createindex(Parameters &par, std::string indexerModule, std::string flag) {
+int createindex(Parameters &par, const Command &command, const std::string &indexerModule, const std::string &flag) {
     bool sensitivity = false;
     // only set kmerScore  to INT_MAX if -s was used
     for (size_t i = 0; i < par.createindex.size(); i++) {
@@ -30,7 +30,7 @@ int createindex(Parameters &par, std::string indexerModule, std::string flag) {
     }
 
     std::string tmpDir = par.db2;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.createindex));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.createindex));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }
@@ -100,7 +100,7 @@ int createlinindex(int argc, const char **argv, const Command& command) {
                             << "Please provide the parameter --search-type 2 (translated) or 3 (nucleotide)\n";
         return EXIT_FAILURE;
     }
-    return createindex(par, "kmerindexdb", (isNucl == false) ? "" : (par.searchType == Parameters::SEARCH_TYPE_TRANSLATED||
+    return createindex(par, command, "kmerindexdb", (isNucl == false) ? "" : (par.searchType == Parameters::SEARCH_TYPE_TRANSLATED||
                                                                                            par.searchType == Parameters::SEARCH_TYPE_TRANS_NUCL_ALN) ? "TRANSLATED" : "LIN_NUCL");
 }
 
@@ -169,6 +169,6 @@ int createindex(int argc, const char **argv, const Command& command) {
                             << "Please provide the parameter --search-type 2 (translated) or 3 (nucleotide)\n";
         return EXIT_FAILURE;
     }
-    return createindex(par, "indexdb",  (isNucl == false) ? "" : (par.searchType == Parameters::SEARCH_TYPE_TRANSLATED||
+    return createindex(par, command, "indexdb",  (isNucl == false) ? "" : (par.searchType == Parameters::SEARCH_TYPE_TRANSLATED||
                                                                   par.searchType == Parameters::SEARCH_TYPE_TRANS_NUCL_ALN) ? "TRANSLATED" : "NUCL");
 }
diff --git a/src/workflow/Databases.cpp b/src/workflow/Databases.cpp
@@ -247,7 +247,7 @@ int databases(int argc, const char **argv, const Command &command) {
     }
     par.printParameters(command.cmd, argc, argv, par.databases);
     std::string tmpDir = par.db3;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.databases));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.databases));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/EasyCluster.cpp b/src/workflow/EasyCluster.cpp
@@ -52,7 +52,7 @@ int easycluster(int argc, const char **argv, const Command &command) {
     setEasyClusterMustPassAlong(&par);
 
     std::string tmpDir = par.filenames.back();
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/EasyLinclust.cpp b/src/workflow/EasyLinclust.cpp
@@ -57,7 +57,7 @@ int easylinclust(int argc, const char **argv, const Command &command) {
     setEasyLinclustMustPassAlong(&par);
 
     std::string tmpDir = par.filenames.back();
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/EasyRbh.cpp b/src/workflow/EasyRbh.cpp
@@ -65,7 +65,7 @@ int easyrbh(int argc, const char **argv, const Command &command) {
     }
 
     std::string tmpDir = par.filenames.back();
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/EasySearch.cpp b/src/workflow/EasySearch.cpp
@@ -84,7 +84,7 @@ int doeasysearch(int argc, const char **argv, const Command &command, bool linse
     }
 
     std::string tmpDir = par.filenames.back();
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/EasyTaxonomy.cpp b/src/workflow/EasyTaxonomy.cpp
@@ -68,7 +68,7 @@ int easytaxonomy(int argc, const char **argv, const Command& command) {
     setEasyTaxonomyMustPassAlong(&par);
 
     std::string tmpDir = par.filenames.back();
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Enrich.cpp b/src/workflow/Enrich.cpp
@@ -19,7 +19,7 @@ int enrich(int argc, const char **argv, const Command &command) {
     par.parseParameters(argc, argv, command, true, 0, 0);
 
     std::string tmpDir = par.db6;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.enrichworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.enrichworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Linclust.cpp b/src/workflow/Linclust.cpp
@@ -33,7 +33,7 @@ int linclust(int argc, const char **argv, const Command& command) {
     par.parseParameters(argc, argv, command, true, 0, 0);
 
     std::string tmpDir = par.db3;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.linclustworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.linclustworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Linsearch.cpp b/src/workflow/Linsearch.cpp
@@ -103,7 +103,7 @@ int linsearch(int argc, const char **argv, const Command &command) {
     par.printParameters(command.cmd, argc, argv, par.searchworkflow);
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.linsearchworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.linsearchworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Map.cpp b/src/workflow/Map.cpp
@@ -42,7 +42,7 @@ int map(int argc, const char **argv, const Command &command) {
     par.parseParameters(argc, argv, command, true, 0, 0);
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.mapworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.mapworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Rbh.cpp b/src/workflow/Rbh.cpp
@@ -39,7 +39,7 @@ int rbh(int argc, const char **argv, const Command &command) {
 
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.searchworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.searchworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Search.cpp b/src/workflow/Search.cpp
@@ -282,7 +282,7 @@ int search(int argc, const char **argv, const Command& command) {
     par.printParameters(command.cmd, argc, argv, par.searchworkflow);
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.searchworkflow));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.searchworkflow));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/TaxPerContig.cpp b/src/workflow/TaxPerContig.cpp
@@ -51,7 +51,7 @@ int taxpercontig(int argc, const char **argv, const Command& command) {
     setTaxPerContigMustPassAlong(&par);
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, *command.params));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, *command.params));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }

diff --git a/src/workflow/Taxonomy.cpp b/src/workflow/Taxonomy.cpp
@@ -56,7 +56,7 @@ int taxonomy(int argc, const char **argv, const Command& command) {
     setTaxonomyMustPassAlong(&par);
 
     std::string tmpDir = par.db4;
-    std::string hash = SSTR(par.hashParameter(par.filenames, par.taxonomy));
+    std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.taxonomy));
     if (par.reuseLatest) {
         hash = FileUtil::getHashFromSymLink(tmpDir + "/latest");
     }