Convert_tokenizer support

src/pull_module/optimum_export.cpp

@@ -51,7 +51,7 @@ std::string OptimumDownloader::getExportCmdEmbeddings() {
     std::ostringstream oss;
     // clang-format off
     oss << this->OPTIMUM_CLI_EXPORT_COMMAND;
-    oss << "--disable-convert-tokenizer --task feature-extraction --library sentence_transformers";
+    oss << "--task feature-extraction --library sentence_transformers";
     oss << " --model " << this->sourceModel << " --trust-remote-code ";
     oss << " --weight-format " << this->exportSettings.precision;
     oss << " " << this->downloadPath;
@@ -64,7 +64,7 @@ std::string OptimumDownloader::getExportCmdRerank() {
     std::ostringstream oss;
     // clang-format off
     oss << this->OPTIMUM_CLI_EXPORT_COMMAND;
-    oss << "--disable-convert-tokenizer --model " << this->sourceModel;
+    oss << "--model " << this->sourceModel;
     oss << " --trust-remote-code ";
     oss << " --weight-format " << this->exportSettings.precision;
     oss << " --task text-classification ";
@@ -114,12 +114,32 @@ std::string OptimumDownloader::getExportCmd() {
     return cmd;
 }
 
-OptimumDownloader::OptimumDownloader(const ExportSettings& inExportSettings, const GraphExportType& inTask, const std::string& inSourceModel, const std::string& inDownloadPath, bool inOverwrite, const std::string& cliExportCmd, const std::string& cliCheckCmd) :
+std::string OptimumDownloader::getConvertCmd() {
+    std::ostringstream oss;
+    // clang-format off
+    oss << this->CONVERT_TOKENIZER_EXPORT_COMMAND;
+    oss << this->sourceModel;
+    oss << " --with-detokenizer -o ";
+    oss << this->downloadPath;
+    // clang-format on
+
+    return oss.str();
+}
+
+bool OptimumDownloader::checkIfDetokenizerFileIsExported() {
+    return std::filesystem::exists(FileSystem::joinPath({this->downloadPath, "openvino_detokenizer.xml"}));
+}
+
+OptimumDownloader::OptimumDownloader(const ExportSettings& inExportSettings, const GraphExportType& inTask,
+    const std::string& inSourceModel, const std::string& inDownloadPath, bool inOverwrite, const std::string& cliExportCmd,
+    const std::string& cliCheckCmd, const std::string& convertExportCmd, const std::string& convertCheckCmd) :
     IModelDownloader(inSourceModel, inDownloadPath, inOverwrite),
     exportSettings(inExportSettings),
     task(inTask),
     OPTIMUM_CLI_CHECK_COMMAND(cliCheckCmd),
-    OPTIMUM_CLI_EXPORT_COMMAND(cliExportCmd) {}
+    OPTIMUM_CLI_EXPORT_COMMAND(cliExportCmd),
+    CONVERT_TOKENIZER_CHECK_COMMAND(convertCheckCmd),
+    CONVERT_TOKENIZER_EXPORT_COMMAND(convertExportCmd) {}
 
 Status OptimumDownloader::checkRequiredToolsArePresent() {
     int retCode = -1;
@@ -131,6 +151,15 @@ Status OptimumDownloader::checkRequiredToolsArePresent() {
     }
 
     SPDLOG_DEBUG("Optimum-cli executable is present");
+
+    output = exec_cmd(this->CONVERT_TOKENIZER_CHECK_COMMAND, retCode);
+    if (retCode != 0) {
+        SPDLOG_DEBUG("Command output {}", output);
+        SPDLOG_ERROR("Trying to pull {} from HuggingFace but missing convert_tokenizer. Use the ovms package with convert_tokenizer.", this->sourceModel);
+        return StatusCode::HF_FAILED_TO_INIT_OPTIMUM_CLI;
+    }
+
+    SPDLOG_DEBUG("Convert_tokenizer executable is present");
     return StatusCode::OK;
 }
 
@@ -171,6 +200,20 @@ Status OptimumDownloader::downloadModel() {
         return StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED;
     }
 
+    if (!this->checkIfDetokenizerFileIsExported()) {
+        SPDLOG_DEBUG("Detokenizer not found in the exported model. Exporting tokenizer and detokenizer from HF model.");
+        cmd = getConvertCmd();
+        retCode = -1;
+        output = exec_cmd(cmd, retCode);
+        if (retCode != 0) {
+            SPDLOG_DEBUG("Command output {}", output);
+            SPDLOG_ERROR("convert_tokenizer command failed.");
+            return StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED;
+        }
+    } else {
+        SPDLOG_DEBUG("Detokenizer is found in the exported model directory. Convert_tokenizer command not required.");
+    }
+
     return StatusCode::OK;
 }
 

src/pull_module/optimum_export.hpp

@@ -24,17 +24,25 @@ class Status;
 
 class OptimumDownloader : public IModelDownloader {
 public:
-    OptimumDownloader(const ExportSettings& exportSettings, const GraphExportType& task, const std::string& inSourceModel, const std::string& inDownloadPath, bool inOverwrite, const std::string& cliExportCmd = "optimum-cli export openvino ", const std::string& cliCheckCmd = "optimum-cli -h");
+    OptimumDownloader(const ExportSettings& exportSettings, const GraphExportType& task, const std::string& inSourceModel,
+        const std::string& inDownloadPath, bool inOverwrite, const std::string& cliExportCmd = "optimum-cli export openvino ",
+        const std::string& cliCheckCmd = "optimum-cli -h",
+        const std::string& convertExportCmd = "convert_tokenizer ",
+        const std::string& convertCheckCmd = "convert_tokenizer -h");
     Status downloadModel() override;
 
 protected:
     ExportSettings exportSettings;
     const GraphExportType task;
     std::string OPTIMUM_CLI_CHECK_COMMAND;
     std::string OPTIMUM_CLI_EXPORT_COMMAND;
+    std::string CONVERT_TOKENIZER_CHECK_COMMAND;
+    std::string CONVERT_TOKENIZER_EXPORT_COMMAND;
 
     Status checkRequiredToolsArePresent();
+    bool checkIfDetokenizerFileIsExported();
     std::string getExportCmd();
+    std::string getConvertCmd();
     std::string getExportCmdText();
     std::string getExportCmdEmbeddings();
     std::string getExportCmdRerank();

src/status.cpp

@@ -341,6 +341,7 @@ const std::unordered_map<StatusCode, std::string> Status::statusMessageMap = {
     {StatusCode::HF_FAILED_TO_INIT_LIBGIT2, "Failed to initialize libgit2 library"},
     {StatusCode::HF_FAILED_TO_INIT_OPTIMUM_CLI, "Failed to run optimum-cli executable"},
     {StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, "Failed to run optimum-cli export command"},
+    {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"},
     {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"},
 
     {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"},

src/status.hpp

@@ -353,6 +353,7 @@ enum class StatusCode {
     HF_FAILED_TO_INIT_LIBGIT2,
     HF_FAILED_TO_INIT_OPTIMUM_CLI,
     HF_RUN_OPTIMUM_CLI_EXPORT_FAILED,
+    HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED,
     HF_GIT_CLONE_FAILED,
 
     PARTIAL_END,

src/test/pull_hf_model_test.cpp

@@ -267,10 +267,14 @@ class TestOptimumDownloader : public ovms::OptimumDownloader {
     TestOptimumDownloader(const ovms::HFSettingsImpl& inHfSettings) :
         ovms::OptimumDownloader(inHfSettings.exportSettings, inHfSettings.task, inHfSettings.sourceModel, ovms::HfDownloader::getGraphDirectory(inHfSettings.downloadPath, inHfSettings.sourceModel), inHfSettings.overwriteModels) {}
     std::string getExportCmd() { return ovms::OptimumDownloader::getExportCmd(); }
+    std::string getConvertCmd() { return ovms::OptimumDownloader::getConvertCmd(); }
     std::string getGraphDirectory() { return ovms::OptimumDownloader::getGraphDirectory(); }
     void setExportCliCheckCommand(const std::string& input) { this->OPTIMUM_CLI_CHECK_COMMAND = input; }
+    void setConvertCliCheckCommand(const std::string& input) { this->CONVERT_TOKENIZER_CHECK_COMMAND = input; }
     void setExportCliExportCommand(const std::string& input) { this->OPTIMUM_CLI_EXPORT_COMMAND = input; }
+    void setConvertCliExportCommand(const std::string& input) { this->CONVERT_TOKENIZER_EXPORT_COMMAND = input; }
     ovms::Status checkRequiredToolsArePresent() { return ovms::OptimumDownloader::checkRequiredToolsArePresent(); }
+    bool checkIfDetokenizerFileIsExported() { return ovms::OptimumDownloader::checkIfDetokenizerFileIsExported(); }
 };
 
 class TestHfDownloader : public ovms::HfDownloader {
@@ -328,24 +332,39 @@ class TestOptimumDownloaderSetup : public ::testing::Test {
     }
 };
 
+class TestOptimumDownloaderSetupWithFile : public TestOptimumDownloaderSetup {
+public:
+    ovms::HFSettingsImpl inHfSettings;
+    std::string cliMockPath;
+    std::filesystem::path file_path;
+    std::filesystem::path dir_path;
+    void TearDown() override {
+        std::filesystem::remove(file_path);
+        std::filesystem::remove_all(dir_path);
+    }
+};
+
 TEST_F(TestOptimumDownloaderSetup, Methods) {
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
     std::string expectedPath = inHfSettings.downloadPath + "/" + inHfSettings.sourceModel;
     std::string expectedCmd = "optimum-cli export openvino --model model/name --trust-remote-code  --weight-format fp64 --param --param value \\path\\to\\Download\\model\\name";
+    std::string expectedCmd2 = "convert_tokenizer model/name --with-detokenizer -o \\path\\to\\Download\\model\\name";
 #ifdef _WIN32
     std::replace(expectedPath.begin(), expectedPath.end(), '/', '\\');
 #endif
 #ifdef __linux__
     std::replace(expectedCmd.begin(), expectedCmd.end(), '\\', '/');
+    std::replace(expectedCmd2.begin(), expectedCmd2.end(), '\\', '/');
 #endif
     ASSERT_EQ(optimumDownloader->getGraphDirectory(), expectedPath);
     ASSERT_EQ(optimumDownloader->getExportCmd(), expectedCmd);
+    ASSERT_EQ(optimumDownloader->getConvertCmd(), expectedCmd2);
 }
 
 TEST_F(TestOptimumDownloaderSetup, RerankExportCmd) {
     inHfSettings.task = ovms::RERANK_GRAPH;
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
-    std::string expectedCmd = "optimum-cli export openvino --disable-convert-tokenizer --model model/name --trust-remote-code  --weight-format fp64 --task text-classification  \\path\\to\\Download\\model\\name";
+    std::string expectedCmd = "optimum-cli export openvino --model model/name --trust-remote-code  --weight-format fp64 --task text-classification  \\path\\to\\Download\\model\\name";
 #ifdef __linux__
     std::replace(expectedCmd.begin(), expectedCmd.end(), '\\', '/');
 #endif
@@ -365,13 +384,30 @@ TEST_F(TestOptimumDownloaderSetup, ImageGenExportCmd) {
 TEST_F(TestOptimumDownloaderSetup, EmbeddingsExportCmd) {
     inHfSettings.task = ovms::EMBEDDINGS_GRAPH;
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
-    std::string expectedCmd = "optimum-cli export openvino --disable-convert-tokenizer --task feature-extraction --library sentence_transformers --model model/name --trust-remote-code  --weight-format fp64 \\path\\to\\Download\\model\\name";
+    std::string expectedCmd = "optimum-cli export openvino --task feature-extraction --library sentence_transformers --model model/name --trust-remote-code  --weight-format fp64 \\path\\to\\Download\\model\\name";
 #ifdef __linux__
     std::replace(expectedCmd.begin(), expectedCmd.end(), '\\', '/');
 #endif
     ASSERT_EQ(optimumDownloader->getExportCmd(), expectedCmd);
 }
 
+TEST_F(TestOptimumDownloaderSetup, DetokenizerCheckNegative) {
+    std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
+    ASSERT_EQ(optimumDownloader->checkIfDetokenizerFileIsExported(), false);
+}
+
+TEST_F(TestOptimumDownloaderSetupWithFile, DetokenizerCheckPositive) {
+    file_path = getGenericFullPathForBazelOut("/ovms/bazel-bin/src/model/name/openvino_detokenizer.xml");
+    inHfSettings.sourceModel = "model/name";
+    inHfSettings.downloadPath = getGenericFullPathForBazelOut("/ovms/bazel-bin/src/");
+    dir_path = getGenericFullPathForBazelOut("/ovms/bazel-bin/src/model/");
+    std::filesystem::create_directories(getGenericFullPathForBazelOut("/ovms/bazel-bin/src/model/name"));
+    std::ofstream ofs(file_path);  // Creates an empty file
+    ofs.close();
+    std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
+    ASSERT_EQ(optimumDownloader->checkIfDetokenizerFileIsExported(), true);
+}
+
 TEST_F(TestOptimumDownloaderSetup, UnknownExportCmd) {
     inHfSettings.task = ovms::UNKNOWN_GRAPH;
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
@@ -386,33 +422,51 @@ TEST_F(TestOptimumDownloaderSetup, NegativeWrongPath) {
 
 TEST_F(TestOptimumDownloaderSetup, NegativeExportCommandFailed) {
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
-    optimumDownloader->setExportCliCheckCommand("ls");
-#ifdef _WIN32
-    optimumDownloader->setExportCliCheckCommand("dir");
-#endif
+    optimumDownloader->setExportCliCheckCommand("echo ");
+    optimumDownloader->setConvertCliCheckCommand("echo ");
     optimumDownloader->setExportCliExportCommand("NonExistingCommand22");
     ASSERT_EQ(optimumDownloader->downloadModel(), ovms::StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED);
 }
 
+TEST_F(TestOptimumDownloaderSetup, NegativeConvertCommandFailed) {
+    std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
+    optimumDownloader->setExportCliCheckCommand("echo ");
+    optimumDownloader->setConvertCliCheckCommand("echo ");
+    optimumDownloader->setExportCliExportCommand("echo ");
+    optimumDownloader->setConvertCliExportCommand("nonExistingCommand222");
+    ASSERT_EQ(optimumDownloader->downloadModel(), ovms::StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED);
+}
+
 TEST_F(TestOptimumDownloaderSetup, NegativeCheckOptimumExistsCommandFailed) {
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
     optimumDownloader->setExportCliCheckCommand("NonExistingCommand33");
+    optimumDownloader->setConvertCliCheckCommand("echo ");
+    ASSERT_EQ(optimumDownloader->checkRequiredToolsArePresent(), ovms::StatusCode::HF_FAILED_TO_INIT_OPTIMUM_CLI);
+}
+
+TEST_F(TestOptimumDownloaderSetup, NegativeCheckConverterExistsCommandFailed) {
+    std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
+    optimumDownloader->setExportCliCheckCommand("echo ");
+    optimumDownloader->setConvertCliCheckCommand("NonExistingCommand33");
     ASSERT_EQ(optimumDownloader->checkRequiredToolsArePresent(), ovms::StatusCode::HF_FAILED_TO_INIT_OPTIMUM_CLI);
 }
 
 TEST_F(TestOptimumDownloaderSetup, PositiveOptimumExistsCommandPassed) {
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
     cliMockPath += " -h";
     optimumDownloader->setExportCliCheckCommand(cliMockPath);
+    optimumDownloader->setConvertCliCheckCommand("echo ");
     ASSERT_EQ(optimumDownloader->checkRequiredToolsArePresent(), ovms::StatusCode::OK);
 }
 
 TEST_F(TestOptimumDownloaderSetup, PositiveOptimumExportCommandPassed) {
     std::unique_ptr<TestOptimumDownloader> optimumDownloader = std::make_unique<TestOptimumDownloader>(inHfSettings);
     std::string cliCheckCommand = cliMockPath += " -h";
     optimumDownloader->setExportCliCheckCommand(cliCheckCommand);
+    optimumDownloader->setConvertCliCheckCommand("echo ");
     cliMockPath += " export";
     optimumDownloader->setExportCliExportCommand(cliMockPath);
+    optimumDownloader->setConvertCliExportCommand("echo ");
     ASSERT_EQ(optimumDownloader->downloadModel(), ovms::StatusCode::OK);
 }