regular and literal strings (#964)

Add escaping to quoted strings, differentiate between literal and
regular strings.

The goal is to make string processing in config files as close as
possible to toml standards. This means handing escape sequences
including unicode, and differentiating between literal strings and
regular strings in files and when splitting the command line. Also
allowing variable names in the files to be quoted.

This PR gets partway there. Removes some hacks from the previous PR to
deal with unusual option names and replaces with the quoted names.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: phlptp

include/CLI/Config.hpp

@@ -26,7 +26,7 @@ namespace detail {
 
 std::string convert_arg_for_ini(const std::string &arg,
                                 char stringQuote = '"',
-                                char characterQuote = '\'',
+                                char literalQuote = '\'',
                                 bool disable_multi_line = false);
 
 /// Comma separated join, adds quotes if needed
@@ -35,7 +35,7 @@ std::string ini_join(const std::vector<std::string> &args,
                      char arrayStart = '[',
                      char arrayEnd = ']',
                      char stringQuote = '"',
-                     char characterQuote = '\'');
+                     char literalQuote = '\'');
 
 std::vector<std::string> generate_parents(const std::string &section, std::string &name, char parentSeparator);
 

include/CLI/ConfigFwd.hpp

@@ -92,8 +92,8 @@ class ConfigBase : public Config {
     char valueDelimiter = '=';
     /// the character to use around strings
     char stringQuote = '"';
-    /// the character to use around single characters
-    char characterQuote = '\'';
+    /// the character to use around single characters and literal strings
+    char literalQuote = '\'';
     /// the maximum number of layers to allow
     uint8_t maximumLayers{255};
     /// the separator used to separator parent layers
@@ -132,7 +132,7 @@ class ConfigBase : public Config {
     /// Specify the quote characters used around strings and characters
     ConfigBase *quoteCharacter(char qString, char qChar) {
         stringQuote = qString;
-        characterQuote = qChar;
+        literalQuote = qChar;
         return this;
     }
     /// Specify the maximum number of parents

include/CLI/StringTools.hpp

@@ -120,6 +120,9 @@ inline std::string trim_copy(const std::string &str) {
 /// remove quotes at the front and back of a string either '"' or '\''
 CLI11_INLINE std::string &remove_quotes(std::string &str);
 
+/// remove quotes from all elements of a string vector and process escaped components
+CLI11_INLINE void remove_quotes(std::vector<std::string> &args);
+
 /// Add a leader to the beginning of all new lines (nothing is added
 /// at the start of the first line). `"; "` would be for ini files
 ///
@@ -212,9 +215,13 @@ template <typename Callable> inline std::string find_and_modify(std::string str,
     return str;
 }
 
+/// close a sequence of characters indicated by a closure character.  Brackets allows sub sequences
+/// recognized bracket sequences include "'`[(<{  other closure characters are assumed to be literal strings
+CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char);
+
 /// Split a string '"one two" "three"' into 'one two', 'three'
 /// Quote characters can be ` ' or " or bracket characters [{(< with matching to the matching bracket
-CLI11_INLINE std::vector<std::string> split_up(std::string str, char delimiter = '\0', bool removeQuotes = true);
+CLI11_INLINE std::vector<std::string> split_up(std::string str, char delimiter = '\0');
 
 /// get the value of an environmental variable or empty string if empty
 CLI11_INLINE std::string get_environment_value(const std::string &env_name);
@@ -246,6 +253,9 @@ CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string);
 /// extract an escaped binary_string
 CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string);
 
+/// process a quoted string, remove the quotes and if appropriate handle escaped characters
+CLI11_INLINE bool process_quoted_string(std::string &str, char string_char = '\"', char literal_char = '\'');
+
 }  // namespace detail
 
 // [CLI11:string_tools_hpp:end]

include/CLI/impl/App_inl.hpp

@@ -579,6 +579,11 @@ CLI11_INLINE void App::parse(std::string commandline, bool program_name_included
     auto args = detail::split_up(std::move(commandline));
     // remove all empty strings
     args.erase(std::remove(args.begin(), args.end(), std::string{}), args.end());
+    try {
+        detail::remove_quotes(args);
+    } catch(const std::invalid_argument &arg) {
+        throw CLI::ParseError(arg.what(), CLI::ExitCodes::InvalidError);
+    }
     std::reverse(args.begin(), args.end());
     parse(std::move(args));
 }
@@ -1569,7 +1574,7 @@ CLI11_INLINE bool App::_parse_single(std::vector<std::string> &args, bool &posit
     case detail::Classifier::SHORT:
     case detail::Classifier::WINDOWS_STYLE:
         // If already parsed a subcommand, don't accept options_
-        _parse_arg(args, classifier, false);
+        retval = _parse_arg(args, classifier, false);
         break;
     case detail::Classifier::NONE:
         // Probably a positional or something for a parent (sub)command

include/CLI/impl/Config_inl.hpp

@@ -19,18 +19,19 @@
 namespace CLI {
 // [CLI11:config_inl_hpp:verbatim]
 
-static constexpr auto triple_quote = R"(""")";
+static constexpr auto multiline_literal_quote = R"(''')";
+static constexpr auto multiline_string_quote = R"(""")";
 
 namespace detail {
 
 CLI11_INLINE bool is_printable(const std::string &test_string) {
     return std::all_of(test_string.begin(), test_string.end(), [](char x) {
-        return (isprint(static_cast<unsigned char>(x)) != 0 || x == '\n');
+        return (isprint(static_cast<unsigned char>(x)) != 0 || x == '\n' || x == '\t');
     });
 }
 
 CLI11_INLINE std::string
-convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuote, bool disable_multi_line) {
+convert_arg_for_ini(const std::string &arg, char stringQuote, char literalQuote, bool disable_multi_line) {
     if(arg.empty()) {
         return std::string(2, stringQuote);
     }
@@ -53,13 +54,10 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuot
         if(isprint(static_cast<unsigned char>(arg.front())) == 0) {
             return binary_escape_string(arg);
         }
-        if(arg == "\\") {
-            return std::string(1, stringQuote) + "\\\\" + stringQuote;
-        }
         if(arg == "'") {
             return std::string(1, stringQuote) + "'" + stringQuote;
         }
-        return std::string(1, characterQuote) + arg + characterQuote;
+        return std::string(1, literalQuote) + arg + literalQuote;
     }
     // handle hex, binary or octal arguments
     if(arg.front() == '0') {
@@ -82,13 +80,10 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuot
     if(!is_printable(arg)) {
         return binary_escape_string(arg);
     }
-    if(arg.find_first_of('\n') != std::string::npos) {
-        if(disable_multi_line) {
-            return binary_escape_string(arg);
-        }
-        return std::string(triple_quote) + arg + triple_quote;
-    }
     if(detail::has_escapable_character(arg)) {
+        if(arg.size() > 100 && !disable_multi_line) {
+            return std::string(multiline_literal_quote) + arg + multiline_literal_quote;
+        }
         return std::string(1, stringQuote) + detail::add_escaped_characters(arg) + stringQuote;
     }
     return std::string(1, stringQuote) + arg + stringQuote;
@@ -99,7 +94,7 @@ CLI11_INLINE std::string ini_join(const std::vector<std::string> &args,
                                   char arrayStart,
                                   char arrayEnd,
                                   char stringQuote,
-                                  char characterQuote) {
+                                  char literalQuote) {
     bool disable_multi_line{false};
     std::string joined;
     if(args.size() > 1 && arrayStart != '\0') {
@@ -114,7 +109,7 @@ CLI11_INLINE std::string ini_join(const std::vector<std::string> &args,
                 joined.push_back(' ');
             }
         }
-        joined.append(convert_arg_for_ini(arg, stringQuote, characterQuote, disable_multi_line));
+        joined.append(convert_arg_for_ini(arg, stringQuote, literalQuote, disable_multi_line));
     }
     if(args.size() > 1 && arrayEnd != '\0') {
         joined.push_back(arrayEnd);
@@ -233,7 +228,7 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
         if(len < 3) {
             continue;
         }
-        if(line.compare(0, 3, triple_quote) == 0 || line.compare(0, 3, "'''") == 0) {
+        if(line.compare(0, 3, multiline_string_quote) == 0 || line.compare(0, 3, multiline_literal_quote) == 0) {
             inMLineComment = true;
             auto cchar = line.front();
             while(inMLineComment) {
@@ -277,29 +272,26 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
 
         // comment lines
         if(line.front() == ';' || line.front() == '#' || line.front() == commentChar) {
-            if(line.compare(2, 13, "cli11:literal") == 0) {
-                literalName = true;
-                getline(input, buffer);
-                line = detail::trim_copy(buffer);
-            } else {
-                continue;
-            }
+            continue;
+        }
+        std::size_t search_start = 0;
+        if(line.front() == stringQuote || line.front() == literalQuote || line.front() == '`') {
+            search_start = detail::close_sequence(line, 0, line.front());
         }
-
         // Find = in string, split and recombine
-        auto delimiter_pos = line.find_first_of(valueDelimiter, 1);
-        auto comment_pos = (literalName) ? std::string::npos : line.find_first_of(commentChar);
-
+        auto delimiter_pos = line.find_first_of(valueDelimiter, search_start + 1);
+        auto comment_pos = line.find_first_of(commentChar, search_start);
         if(comment_pos < delimiter_pos) {
             delimiter_pos = std::string::npos;
         }
         if(delimiter_pos != std::string::npos) {
 
             name = detail::trim_copy(line.substr(0, delimiter_pos));
             std::string item = detail::trim_copy(line.substr(delimiter_pos + 1, std::string::npos));
-            bool mlquote = (item.compare(0, 3, "'''") == 0 || item.compare(0, 3, triple_quote) == 0);
+            bool mlquote =
+                (item.compare(0, 3, multiline_literal_quote) == 0 || item.compare(0, 3, multiline_string_quote) == 0);
             if(!mlquote && comment_pos != std::string::npos && !literalName) {
-                auto citems = detail::split_up(item, commentChar, false);
+                auto citems = detail::split_up(item, commentChar);
                 item = detail::trim_copy(citems.front());
             }
             if(mlquote) {
@@ -337,6 +329,9 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
                         if(!item.empty() && item.back() == '\n') {
                             item.pop_back();
                         }
+                        if(keyChar == '\"') {
+                            item = detail::remove_escaped_characters(item);
+                        }
                     } else {
                         if(lineExtension) {
                             detail::trim(l2);
@@ -358,29 +353,27 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
                     detail::trim(multiline);
                     item += multiline;
                 }
-                items_buffer = detail::split_up(item.substr(1, item.length() - 2), aSep, false);
+                items_buffer = detail::split_up(item.substr(1, item.length() - 2), aSep);
             } else if((isDefaultArray || isINIArray) && item.find_first_of(aSep) != std::string::npos) {
-                items_buffer = detail::split_up(item, aSep, false);
+                items_buffer = detail::split_up(item, aSep);
             } else if((isDefaultArray || isINIArray) && item.find_first_of(' ') != std::string::npos) {
-                items_buffer = detail::split_up(item, '\0', false);
+                items_buffer = detail::split_up(item, '\0');
             } else {
                 items_buffer = {item};
             }
         } else {
             name = detail::trim_copy(line.substr(0, comment_pos));
             items_buffer = {"true"};
         }
-        if(name.find(parentSeparatorChar) == std::string::npos) {
-            if(!literalName) {
-                detail::remove_quotes(name);
-            }
-        }
-        // clean up quotes on the items and check for escaped strings
-        for(auto &it : items_buffer) {
-            detail::remove_quotes(it);
-            if(detail::is_binary_escaped_string(it)) {
-                it = detail::extract_binary_string(it);
+        try {
+            literalName = detail::process_quoted_string(name, stringQuote, literalQuote);
+
+            // clean up quotes on the items and check for escaped strings
+            for(auto &it : items_buffer) {
+                detail::process_quoted_string(it, stringQuote, literalQuote);
             }
+        } catch(const std::invalid_argument &ia) {
+            throw CLI::ParseError(ia.what(), CLI::ExitCodes::InvalidError);
         }
         std::vector<std::string> parents;
         if(literalName) {
@@ -461,16 +454,17 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
                         continue;
                     }
                 }
-                std::string name = prefix + opt->get_single_name();
-                if(name == prefix) {
+                std::string single_name = opt->get_single_name();
+                if(single_name.empty()) {
                     continue;
                 }
+
                 std::string value = detail::ini_join(
-                    opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, characterQuote);
+                    opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote);
 
                 if(value.empty() && default_also) {
                     if(!opt->get_default_str().empty()) {
-                        value = detail::convert_arg_for_ini(opt->get_default_str(), stringQuote, characterQuote, false);
+                        value = detail::convert_arg_for_ini(opt->get_default_str(), stringQuote, literalQuote, false);
                     } else if(opt->get_expected_min() == 0) {
                         value = "false";
                     } else if(opt->get_run_callback_for_default()) {
@@ -479,37 +473,52 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
                 }
 
                 if(!value.empty()) {
+
                     if(!opt->get_fnames().empty()) {
                         try {
-                            value = opt->get_flag_value(name, value);
+                            value = opt->get_flag_value(single_name, value);
                         } catch(const CLI::ArgumentMismatch &) {
                             bool valid{false};
                             for(const auto &test_name : opt->get_fnames()) {
                                 try {
                                     value = opt->get_flag_value(test_name, value);
-                                    name = test_name;
+                                    single_name = test_name;
                                     valid = true;
                                 } catch(const CLI::ArgumentMismatch &) {
                                     continue;
                                 }
                             }
                             if(!valid) {
                                 value = detail::ini_join(
-                                    opt->results(), arraySeparator, arrayStart, arrayEnd, stringQuote, characterQuote);
+                                    opt->results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote);
                             }
                         }
                     }
                     if(write_description && opt->has_description()) {
                         out << '\n';
                         out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n';
                     }
-                    if(name.find_first_of(commentTest) != std::string::npos || name.compare(0, 3, triple_quote) == 0 ||
-                       name.compare(0, 3, "'''") == 0 || (name.front() == '[' && name.back() == ']') ||
-                       (name.front() == stringQuote && name.back() == stringQuote) ||
-                       (name.front() == characterQuote && name.back() == characterQuote) ||
-                       (name.front() == '`' && name.back() == '`')) {
-                        out << commentChar << " cli11:literal\n";
+                    if(single_name.find_first_of(commentTest) != std::string::npos ||
+                       single_name.compare(0, 3, multiline_string_quote) == 0 ||
+                       single_name.compare(0, 3, multiline_literal_quote) == 0 ||
+                       (single_name.front() == '[' && single_name.back() == ']') ||
+                       (single_name.find_first_of(stringQuote) != std::string::npos) ||
+                       (single_name.find_first_of(literalQuote) != std::string::npos) ||
+                       (single_name.find_first_of('`') != std::string::npos)) {
+                        if(single_name.find_first_of(literalQuote) == std::string::npos) {
+                            single_name.insert(0, 1, literalQuote);
+                            single_name.push_back(literalQuote);
+                        } else {
+                            if(detail::has_escapable_character(single_name)) {
+                                single_name = detail::add_escaped_characters(single_name);
+                            }
+                            single_name.insert(0, 1, stringQuote);
+                            single_name.push_back(stringQuote);
+                        }
                     }
+
+                    std::string name = prefix + single_name;
+
                     out << name << valueDelimiter << value << '\n';
                 }
             }

include/CLI/impl/Option_inl.hpp

@@ -609,7 +609,12 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi
             throw ArgumentMismatch::AtLeast(get_name(), static_cast<int>(num_min), original.size());
         }
         if(original.size() > num_max) {
-            throw ArgumentMismatch::AtMost(get_name(), static_cast<int>(num_max), original.size());
+            if(original.size() == 2 && num_max == 1 && original[1] == "%%" && original[0] == "{}") {
+                // this condition is a trap for the following empty indicator check on config files
+                out = original;
+            } else {
+                throw ArgumentMismatch::AtMost(get_name(), static_cast<int>(num_max), original.size());
+            }
         }
         break;
     }