From b73ec58f6b61eea7bae3ea06cd6bad2fd0e7c882 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Mon, 25 Dec 2023 06:32:57 -0800 Subject: [PATCH 01/25] add escaping to quoted strings, differentiate between literal and regular strings --- include/CLI/Config.hpp | 4 +- include/CLI/ConfigFwd.hpp | 6 +- include/CLI/StringTools.hpp | 8 +- include/CLI/impl/App_inl.hpp | 1 + include/CLI/impl/Config_inl.hpp | 109 +++++----- include/CLI/impl/StringTools_inl.hpp | 314 +++++++++++++++++++++------ tests/AppTest.cpp | 8 +- tests/ConfigFileTest.cpp | 2 +- tests/HelpersTest.cpp | 66 ++++-- 9 files changed, 372 insertions(+), 146 deletions(-) diff --git a/include/CLI/Config.hpp b/include/CLI/Config.hpp index d5c6bc347..c9809801b 100644 --- a/include/CLI/Config.hpp +++ b/include/CLI/Config.hpp @@ -26,7 +26,7 @@ namespace detail { std::string convert_arg_for_ini(const std::string &arg, char stringQuote = '"', - char characterQuote = '\'', + char literalQuote = '\'', bool disable_multi_line = false); /// Comma separated join, adds quotes if needed @@ -35,7 +35,7 @@ std::string ini_join(const std::vector &args, char arrayStart = '[', char arrayEnd = ']', char stringQuote = '"', - char characterQuote = '\''); + char literalQuote = '\''); std::vector generate_parents(const std::string §ion, std::string &name, char parentSeparator); diff --git a/include/CLI/ConfigFwd.hpp b/include/CLI/ConfigFwd.hpp index 76fc9fa7e..fe4071f7a 100644 --- a/include/CLI/ConfigFwd.hpp +++ b/include/CLI/ConfigFwd.hpp @@ -92,8 +92,8 @@ class ConfigBase : public Config { char valueDelimiter = '='; /// the character to use around strings char stringQuote = '"'; - /// the character to use around single characters - char characterQuote = '\''; + /// the character to use around single characters and literal strings + char literalQuote = '\''; /// the maximum number of layers to allow uint8_t maximumLayers{255}; /// the separator used to separator parent layers @@ -132,7 +132,7 @@ class ConfigBase : public Config { /// Specify the quote characters used around strings and characters ConfigBase *quoteCharacter(char qString, char qChar) { stringQuote = qString; - characterQuote = qChar; + literalQuote = qChar; return this; } /// Specify the maximum number of parents diff --git a/include/CLI/StringTools.hpp b/include/CLI/StringTools.hpp index 2356a70af..a5df4bf32 100644 --- a/include/CLI/StringTools.hpp +++ b/include/CLI/StringTools.hpp @@ -120,6 +120,9 @@ inline std::string trim_copy(const std::string &str) { /// remove quotes at the front and back of a string either '"' or '\'' CLI11_INLINE std::string &remove_quotes(std::string &str); +/// remove quotes from all elements of a string vector and process escaped components +CLI11_INLINE void remove_quotes(std::vector &args); + /// Add a leader to the beginning of all new lines (nothing is added /// at the start of the first line). `"; "` would be for ini files /// @@ -214,7 +217,7 @@ template inline std::string find_and_modify(std::string str, /// Split a string '"one two" "three"' into 'one two', 'three' /// Quote characters can be ` ' or " or bracket characters [{(< with matching to the matching bracket -CLI11_INLINE std::vector split_up(std::string str, char delimiter = '\0', bool removeQuotes = true); +CLI11_INLINE std::vector split_up(std::string str, char delimiter = '\0'); /// get the value of an environmental variable or empty string if empty CLI11_INLINE std::string get_environment_value(const std::string &env_name); @@ -246,6 +249,9 @@ CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string); /// extract an escaped binary_string CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string); +/// process a quoted string, remove the quotes and if appropriate handle escaped characters +CLI11_INLINE bool process_quoted_string(std::string& str, char string_char='\"', char literal_char='\''); + } // namespace detail // [CLI11:string_tools_hpp:end] diff --git a/include/CLI/impl/App_inl.hpp b/include/CLI/impl/App_inl.hpp index ee9bee336..a459e19ce 100644 --- a/include/CLI/impl/App_inl.hpp +++ b/include/CLI/impl/App_inl.hpp @@ -579,6 +579,7 @@ CLI11_INLINE void App::parse(std::string commandline, bool program_name_included auto args = detail::split_up(std::move(commandline)); // remove all empty strings args.erase(std::remove(args.begin(), args.end(), std::string{}), args.end()); + detail::remove_quotes(args); std::reverse(args.begin(), args.end()); parse(std::move(args)); } diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 2b98509cd..04c611386 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -19,18 +19,19 @@ namespace CLI { // [CLI11:config_inl_hpp:verbatim] -static constexpr auto triple_quote = R"(""")"; + static constexpr auto multiline_literal_quote = R"(''')"; + static constexpr auto multiline_string_quote = R"(""")"; namespace detail { CLI11_INLINE bool is_printable(const std::string &test_string) { return std::all_of(test_string.begin(), test_string.end(), [](char x) { - return (isprint(static_cast(x)) != 0 || x == '\n'); + return (isprint(static_cast(x)) != 0 || x == '\n'||x=='\t'); }); } CLI11_INLINE std::string -convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuote, bool disable_multi_line) { +convert_arg_for_ini(const std::string &arg, char stringQuote, char literalQuote, bool disable_multi_line) { if(arg.empty()) { return std::string(2, stringQuote); } @@ -54,12 +55,12 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuot return binary_escape_string(arg); } if(arg == "\\") { - return std::string(1, stringQuote) + "\\\\" + stringQuote; + return std::string(1, literalQuote) + "\\" + literalQuote; } if(arg == "'") { return std::string(1, stringQuote) + "'" + stringQuote; } - return std::string(1, characterQuote) + arg + characterQuote; + return std::string(1, literalQuote) + arg + literalQuote; } // handle hex, binary or octal arguments if(arg.front() == '0') { @@ -82,13 +83,11 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char characterQuot if(!is_printable(arg)) { return binary_escape_string(arg); } - if(arg.find_first_of('\n') != std::string::npos) { - if(disable_multi_line) { - return binary_escape_string(arg); - } - return std::string(triple_quote) + arg + triple_quote; - } if(detail::has_escapable_character(arg)) { + if (arg.size() > 100 && !disable_multi_line) + { + return std::string(multiline_literal_quote) + arg + multiline_literal_quote; + } return std::string(1, stringQuote) + detail::add_escaped_characters(arg) + stringQuote; } return std::string(1, stringQuote) + arg + stringQuote; @@ -99,7 +98,7 @@ CLI11_INLINE std::string ini_join(const std::vector &args, char arrayStart, char arrayEnd, char stringQuote, - char characterQuote) { + char literalQuote) { bool disable_multi_line{false}; std::string joined; if(args.size() > 1 && arrayStart != '\0') { @@ -114,7 +113,7 @@ CLI11_INLINE std::string ini_join(const std::vector &args, joined.push_back(' '); } } - joined.append(convert_arg_for_ini(arg, stringQuote, characterQuote, disable_multi_line)); + joined.append(convert_arg_for_ini(arg, stringQuote, literalQuote, disable_multi_line)); } if(args.size() > 1 && arrayEnd != '\0') { joined.push_back(arrayEnd); @@ -233,7 +232,7 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons if(len < 3) { continue; } - if(line.compare(0, 3, triple_quote) == 0 || line.compare(0, 3, "'''") == 0) { + if(line.compare(0, 3, multiline_string_quote) == 0 || line.compare(0, 3, multiline_literal_quote) == 0) { inMLineComment = true; auto cchar = line.front(); while(inMLineComment) { @@ -277,19 +276,16 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons // comment lines if(line.front() == ';' || line.front() == '#' || line.front() == commentChar) { - if(line.compare(2, 13, "cli11:literal") == 0) { - literalName = true; - getline(input, buffer); - line = detail::trim_copy(buffer); - } else { - continue; - } + continue; + } + std::size_t search_start=0; + if (line.front() == stringQuote ||line.front()==literalQuote||line.front()=='`') + { + search_start=detail::close_sequence(line,1,line.front()); } - // Find = in string, split and recombine - auto delimiter_pos = line.find_first_of(valueDelimiter, 1); - auto comment_pos = (literalName) ? std::string::npos : line.find_first_of(commentChar); - + auto delimiter_pos = line.find_first_of(valueDelimiter, search_start+1); + auto comment_pos = line.find_first_of(commentChar,search_start); if(comment_pos < delimiter_pos) { delimiter_pos = std::string::npos; } @@ -297,9 +293,9 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, delimiter_pos)); std::string item = detail::trim_copy(line.substr(delimiter_pos + 1, std::string::npos)); - bool mlquote = (item.compare(0, 3, "'''") == 0 || item.compare(0, 3, triple_quote) == 0); + bool mlquote = (item.compare(0, 3, multiline_literal_quote) == 0 || item.compare(0, 3, multiline_string_quote) == 0); if(!mlquote && comment_pos != std::string::npos && !literalName) { - auto citems = detail::split_up(item, commentChar, false); + auto citems = detail::split_up(item, commentChar); item = detail::trim_copy(citems.front()); } if(mlquote) { @@ -337,6 +333,9 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons if(!item.empty() && item.back() == '\n') { item.pop_back(); } + if (keyChar == '\"') { + item=detail::remove_escaped_characters(item); + } } else { if(lineExtension) { detail::trim(l2); @@ -358,11 +357,11 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons detail::trim(multiline); item += multiline; } - items_buffer = detail::split_up(item.substr(1, item.length() - 2), aSep, false); + items_buffer = detail::split_up(item.substr(1, item.length() - 2), aSep); } else if((isDefaultArray || isINIArray) && item.find_first_of(aSep) != std::string::npos) { - items_buffer = detail::split_up(item, aSep, false); + items_buffer = detail::split_up(item, aSep); } else if((isDefaultArray || isINIArray) && item.find_first_of(' ') != std::string::npos) { - items_buffer = detail::split_up(item, '\0', false); + items_buffer = detail::split_up(item, '\0'); } else { items_buffer = {item}; } @@ -370,17 +369,11 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, comment_pos)); items_buffer = {"true"}; } - if(name.find(parentSeparatorChar) == std::string::npos) { - if(!literalName) { - detail::remove_quotes(name); - } - } + literalName=detail::process_quoted_string(name,stringQuote,literalQuote); + // clean up quotes on the items and check for escaped strings for(auto &it : items_buffer) { - detail::remove_quotes(it); - if(detail::is_binary_escaped_string(it)) { - it = detail::extract_binary_string(it); - } + detail::process_quoted_string(it,stringQuote,literalQuote); } std::vector parents; if(literalName) { @@ -461,16 +454,36 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, continue; } } - std::string name = prefix + opt->get_single_name(); - if(name == prefix) { + std::string single_name=opt->get_single_name(); + if(single_name.empty()) { continue; } + if(single_name.find_first_of(commentTest) != std::string::npos || single_name.compare(0, 3, multiline_string_quote) == 0 || + single_name.compare(0, 3, multiline_literal_quote) == 0 || (single_name.front() == '[' && single_name.back() == ']') || + (single_name.find_first_of(stringQuote)!=std::string::npos )|| + (single_name.find_first_of(literalQuote)!=std::string::npos )|| + (single_name.find_first_of('`') != std::string::npos)) { + if (single_name.find_first_of(literalQuote) == std::string::npos) + { + single_name.insert(0,1,literalQuote); + single_name.push_back(literalQuote); + } + else + { + single_name.insert(0,1,stringQuote); + single_name.push_back(stringQuote); + } + + } + + std::string name = prefix + single_name; + std::string value = detail::ini_join( - opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, characterQuote); + opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote); if(value.empty() && default_also) { if(!opt->get_default_str().empty()) { - value = detail::convert_arg_for_ini(opt->get_default_str(), stringQuote, characterQuote, false); + value = detail::convert_arg_for_ini(opt->get_default_str(), stringQuote, literalQuote, false); } else if(opt->get_expected_min() == 0) { value = "false"; } else if(opt->get_run_callback_for_default()) { @@ -495,7 +508,7 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, } if(!valid) { value = detail::ini_join( - opt->results(), arraySeparator, arrayStart, arrayEnd, stringQuote, characterQuote); + opt->results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote); } } } @@ -503,13 +516,7 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, out << '\n'; out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n'; } - if(name.find_first_of(commentTest) != std::string::npos || name.compare(0, 3, triple_quote) == 0 || - name.compare(0, 3, "'''") == 0 || (name.front() == '[' && name.back() == ']') || - (name.front() == stringQuote && name.back() == stringQuote) || - (name.front() == characterQuote && name.back() == characterQuote) || - (name.front() == '`' && name.back() == '`')) { - out << commentChar << " cli11:literal\n"; - } + out << name << valueDelimiter << value << '\n'; } } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 40028726d..000982f08 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -61,7 +61,17 @@ CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter) { } CLI11_INLINE std::string &remove_quotes(std::string &str) { - if(str.length() > 1 && (str.front() == '"' || str.front() == '\'')) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\''|| str.front() == '`')) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +CLI11_INLINE std::string &remove_outer(std::string &str,char key) { + if(str.length() > 1 && (str.front() == key) ){ if(str.front() == str.back()) { str.pop_back(); str.erase(str.begin(), str.begin() + 1); @@ -181,9 +191,10 @@ find_member(std::string name, const std::vector names, bool ignore_ return (it != std::end(names)) ? (it - std::begin(names)) : (-1); } -static const std::string escapedChars("'\"`])>}\\"); -static const std::string bracketChars{"'\"`[(<{"}; -static const std::string matchBracketChars("'\"`])>}"); +static const std::string escapedChars("\b\t\n\f\r\"\\"); +static const std::string escapedCharsCode("btnfr\"\\"); +static const std::string bracketChars{"\"'`[(<{"}; +static const std::string matchBracketChars("\"'`])>}"); CLI11_INLINE bool has_escapable_character(const std::string &str) { return (str.find_first_of(escapedChars) != std::string::npos); @@ -193,25 +204,127 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size() + 4); for(char s : str) { - if(escapedChars.find_first_of(s) != std::string::npos) { + auto sloc=escapedChars.find_first_of(s); + if(sloc != std::string::npos) { out.push_back('\\'); + out.push_back(escapedCharsCode[sloc]); + } + else + { + out.push_back(s); } - out.push_back(s); } return out; } +CLI11_INLINE int hexConvert(char hc) +{ + int res{0}; + if(hc >= '0' && hc <= '9') { + res = (hc - '0'); + } else if(hc >= 'A' && hc <= 'F') { + res = (hc - 'A' + 10); + } else if(hc >= 'a' && hc <= 'f') { + res = (hc - 'a' + 10); + } else { + res=-1; + } + return res; +} + +CLI11_INLINE char make_char(std::uint32_t code) +{ + return static_cast(static_cast(code)); +} + +CLI11_INLINE void append_codepoint(std::string& str, std::uint32_t code) +{ + if(code < 0x80) // ascii code equivalent + { + str.push_back(static_cast(code)); + } + else if(code < 0x800) //\u0080 to \u07FF + { + // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 + str.push_back(make_char(0xC0| code >> 6)); + str.push_back(make_char(0x80|(code & 0x3F))); + } + else if(code < 0x10000) // U+0800...U+FFFF + { + if(0xD800 <= code && code <= 0xDFFF) + { + throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); + } + // 1110yyyy 10yxxxxx 10xxxxxx + str.push_back(make_char(0xE0| code >> 12)); + str.push_back(make_char(0x80|(code >> 6 & 0x3F))); + str.push_back(make_char(0x80|(code & 0x3F))); + } + else if(code < 0x110000) // U+010000 ... U+10FFFF + { + // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx + str.push_back(make_char(0xF0| code >> 18)); + str.push_back(make_char(0x80|(code >> 12 & 0x3F))); + str.push_back(make_char(0x80|(code >> 6 & 0x3F))); + str.push_back(make_char(0x80|(code & 0x3F))); + } +} + CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size()); for(auto loc = str.begin(); loc < str.end(); ++loc) { if(*loc == '\\') { - if(escapedChars.find_first_of(*(loc + 1)) != std::string::npos) { - out.push_back(*(loc + 1)); + auto ecloc=escapedCharsCode.find_first_of(*(loc+1)); + if(ecloc != std::string::npos) { + out.push_back(escapedChars[ecloc]); ++loc; - } else { - out.push_back(*loc); + } else if (*(loc+1)=='u'){ + //must have 4 hex characters + if (str.end() - loc < 5) + { + throw std::invalid_argument("unicode sequence must have 4 hex codes "+str); + } + int code{0}; + int mplier{16*16*16}; + for (int ii = 2; ii < 6; ++ii) + { + int res=hexConvert(*(loc+ii)); + if (res < 0) { + throw std::invalid_argument("unicode sequence must have 4 hex codes "+str); + } + code+=res*mplier; + mplier=mplier/16; + } + append_codepoint(out,code); + loc+=5; + }else if (*(loc+1)=='U'){ + // must have 8 hex characters + if (str.end() - loc < 9) + { + throw std::invalid_argument("unicode sequence must have 8 hex codes "+str); + } + int code{0}; + int mplier{16*16*16*16*16*16*16}; + for (int ii = 2; ii < 10; ++ii) + { + int res=hexConvert(*(loc+ii)); + if (res < 0) { + throw std::invalid_argument("unicode sequence must have 8 hex codes "+str); + } + code+=res*mplier; + mplier=mplier/16; + } + append_codepoint(out,code); + loc+=9; + } + else if (*(loc + 1) == '0') { + out.push_back('\0'); + ++loc; + } + else{ + throw std::invalid_argument(std::string("unrecognized escape sequence \\")+*(loc+1)+" in "+str); } } else { out.push_back(*loc); @@ -220,39 +333,78 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { return out; } -CLI11_INLINE std::pair close_sequence(const std::string &str, std::size_t start, char closure_char) { - std::string closures; - closures.push_back(closure_char); +CLI11_INLINE std::size_t close_string_quote(const std::string& str, std::size_t start, char closure_char) +{ + std::size_t loc=start+1; + for (loc=start+1;loc split_up(std::string str, char delimiter, bool removeQuotes) { +CLI11_INLINE std::vector split_up(std::string str, char delimiter) { auto find_ws = [delimiter](char ch) { return (delimiter == '\0') ? std::isspace(ch, std::locale()) : (ch == delimiter); @@ -260,20 +412,26 @@ CLI11_INLINE std::vector split_up(std::string str, char delimiter, trim(str); std::vector output; - bool embeddedQuote = false; - std::size_t adjust = removeQuotes ? 1 : 0; while(!str.empty()) { if(bracketChars.find_first_of(str[0]) != std::string::npos) { auto bracketLoc = bracketChars.find_first_of(str[0]); - auto closure = close_sequence(str, 0, matchBracketChars[bracketLoc]); - auto end = closure.first; - output.push_back(str.substr(adjust, end + 1 - 2 * adjust)); - if(end + 2 < str.size()) { - str = str.substr(end + 2); - } else { + auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); + if (end >= std::string::npos-1) + { + output.push_back(str.substr(0)); str.clear(); } - embeddedQuote = embeddedQuote || closure.second; + else + { + output.push_back(str.substr(0,end+1)); + if(end + 2 < str.size()) { + str = str.substr(end + 2); + } else { + str.clear(); + } + } + + } else { auto it = std::find_if(std::begin(str), std::end(str), find_ws); if(it != std::end(str)) { @@ -285,11 +443,6 @@ CLI11_INLINE std::vector split_up(std::string str, char delimiter, str.clear(); } } - // transform any embedded quotes into the regular character if the quotes are removed - if(embeddedQuote && removeQuotes) { - output.back() = remove_escaped_characters(output.back()); - embeddedQuote = false; - } trim(str); } return output; @@ -373,30 +526,12 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string if(escaped_string[loc] == '\\' && (escaped_string[loc + 1] == 'x' || escaped_string[loc + 1] == 'X')) { auto c1 = escaped_string[loc + 2]; auto c2 = escaped_string[loc + 3]; - int res{0}; - bool invalid{false}; - if(c1 >= '0' && c1 <= '9') { - res = (c1 - '0') * 16; - } else if(c1 >= 'A' && c1 <= 'F') { - res = (c1 - 'A' + 10) * 16; - } else if(c1 >= 'a' && c1 <= 'f') { - res = (c1 - 'a' + 10) * 16; - } else { - invalid = true; - } - - if(c2 >= '0' && c2 <= '9') { - res += (c2 - '0'); - } else if(c2 >= 'A' && c2 <= 'F') { - res += (c2 - 'A' + 10); - } else if(c2 >= 'a' && c2 <= 'f') { - res += (c2 - 'a' + 10); - } else { - invalid = true; - } - if(!invalid) { + + int res1=hexConvert(c1); + int res2=hexConvert(c2); + if(res1>=0 && res2>=0) { loc += 4; - outstring.push_back(static_cast(res)); + outstring.push_back(static_cast(res1*16+res2)); continue; } } @@ -406,6 +541,53 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string return outstring; } + +CLI11_INLINE void remove_quotes(std::vector &args) { + for (auto& arg : args) + { + if (arg.front() == '\"' && arg.back() == '\"') + { + remove_quotes(arg); + arg=remove_escaped_characters(arg); + } + else + { + remove_quotes(arg); + } + } +} + +CLI11_INLINE bool process_quoted_string(std::string& str, char string_char, char literal_char) +{ + if (str.size() <= 1) + { + return false; + } + if(detail::is_binary_escaped_string(str)) { + str=detail::extract_binary_string(str); + return true; + } + if (str.front() == string_char && str.back()==string_char) + { + detail::remove_outer(str,string_char); + if (str.find_first_of('\\') != std::string::npos) { + str=detail::remove_escaped_characters(str); + } + return true; + } + if (str.front() == literal_char && str.back()==literal_char) + { + detail::remove_outer(str,literal_char); + return true; + } + if (str.front() == '`' && str.back() == '`') + { + detail::remove_outer(str,literal_char); + return true; + } + return false; +} + std::string get_environment_value(const std::string &env_name) { char *buffer = nullptr; std::string ename_string; diff --git a/tests/AppTest.cpp b/tests/AppTest.cpp index 2d753527d..da8197cb4 100644 --- a/tests/AppTest.cpp +++ b/tests/AppTest.cpp @@ -414,10 +414,10 @@ TEST_CASE_METHOD(TApp, "OneStringEqualVersionSingleStringQuotedEscapedCharacters app.add_option("-s,--string", str); app.add_option("-t,--tstr", str2); app.add_option("-m,--mstr", str3); - app.parse(R"raw(--string="this is my \"quoted\" string" -t 'qst\'ring 2' -m=`"quoted\` string"`")raw"); - CHECK("this is my \"quoted\" string" == str); - CHECK("qst\'ring 2" == str2); - CHECK("\"quoted` string\"" == str3); + app.parse(R"raw(--string="this is my \n\"quoted\" string" -t 'qst\ring 2' -m=`"quoted\n string"`")raw"); + CHECK("this is my \n\"quoted\" string" == str); //escaped + CHECK("qst\\ring 2" == str2); //literal + CHECK("\"quoted\\n string\"" == str3); //double quoted literal } TEST_CASE_METHOD(TApp, "OneStringEqualVersionSingleStringQuotedMultipleWithEqual", "[app]") { diff --git a/tests/ConfigFileTest.cpp b/tests/ConfigFileTest.cpp index d59ca0eb2..710d748db 100644 --- a/tests/ConfigFileTest.cpp +++ b/tests/ConfigFileTest.cpp @@ -2713,7 +2713,7 @@ TEST_CASE_METHOD(TApp, "TomlOutputMultilineString", "[config]") { std::string desc = "flag"; app.add_option("--opt", desc); - std::string argString = "this is a very long string \n that covers multiple lines \n and should be long"; + std::string argString = "this is a very long string \n that covers multiple lines \nand should be longer than 100 characters \nto trigger the multiline string"; args = {"--opt", argString}; run(); diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index 82b972552..22ba3758b 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -165,6 +165,7 @@ TEST_CASE("String: InvalidName", "[helpers]") { CHECK(CLI::detail::valid_name_string("b@d2?")); CHECK(CLI::detail::valid_name_string("2vali?d")); CHECK_FALSE(CLI::detail::valid_name_string("!valid")); + CHECK_FALSE(CLI::detail::valid_name_string("!va\nlid")); } TEST_CASE("StringTools: Modify", "[helpers]") { @@ -300,12 +301,41 @@ TEST_CASE("StringTools: binaryStrings", "[helpers]") { CHECK(result == "\\XEM\\X7K"); } +/// these are provided for compatibility with the char8_t for C++20 that breaks stuff +std::string from_u8string(const std::string &s) { + return s; +} +std::string from_u8string(std::string &&s) { + return std::move(s); +} +#if defined(__cpp_lib_char8_t) +std::string from_u8string(const std::u8string &s) { + return std::string(s.begin(), s.end()); +} +#endif + TEST_CASE("StringTools: escapeConversion", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\\"") == "test\""); - CHECK(CLI::detail::remove_escaped_characters("test\\}") == "test}"); - CHECK(CLI::detail::remove_escaped_characters("test\\\\") == "test\\"); CHECK(CLI::detail::remove_escaped_characters("test\\\\") == "test\\"); - CHECK(CLI::detail::remove_escaped_characters("test\\k") == "test\\k"); + CHECK(CLI::detail::remove_escaped_characters("test\\b") == "test\b"); + CHECK(CLI::detail::remove_escaped_characters("test\\t") == "test\t"); + CHECK(CLI::detail::remove_escaped_characters("test\\n\\r\\t\\f") == "test\n\r\t\f"); + CHECK(CLI::detail::remove_escaped_characters("test\\r") == "test\r"); + CHECK(CLI::detail::remove_escaped_characters("test\\f") == "test\f"); + CHECK(CLI::detail::remove_escaped_characters("test\\ttest\\n") == "test\ttest\n"); + + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\m_bad"),std::invalid_argument); +} + +TEST_CASE("StringTools: unicode_literals", "[helpers]") { + + CHECK(CLI::detail::remove_escaped_characters("test\\u03C0\\u00e9") == from_u8string(u8"test\u03C0\u00E9")); + + CHECK(CLI::detail::remove_escaped_characters("test\\U0001F600\\u00E9") == from_u8string(u8"test\U0001F600\u00E9")); + + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001M600\\u00E9"),std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\u00M9"),std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\uD8E9"),std::invalid_argument); } TEST_CASE("Trim: Various", "[helpers]") { @@ -967,35 +997,35 @@ TEST_CASE("Join: Backward", "[helpers]") { } TEST_CASE("SplitUp: Simple", "[helpers]") { - std::vector oput = {"one", "two three"}; + std::vector oput = {"one", "\"two three\""}; std::string orig{R"(one "two three")"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); } TEST_CASE("SplitUp: SimpleDifferentQuotes", "[helpers]") { - std::vector oput = {"one", "two three"}; + std::vector oput = {"one", "`two three`"}; std::string orig{R"(one `two three`)"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); } TEST_CASE("SplitUp: SimpleMissingQuotes", "[helpers]") { - std::vector oput = {"one", "two three"}; + std::vector oput = {"one", "`two three"}; std::string orig{R"(one `two three)"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); } TEST_CASE("SplitUp: SimpleMissingQuotesEscaped", "[helpers]") { - std::vector oput = {"one", "two three`"}; - std::string orig{R"(one `two three\`)"}; + std::vector oput = { "one", "\"two three\\\"\""}; + std::string orig{R"(one "two three\"")"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); } TEST_CASE("SplitUp: SimpleDifferentQuotes2", "[helpers]") { - std::vector oput = {"one", "two three"}; + std::vector oput = {"one", "'two three'"}; std::string orig{R"(one 'two three')"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); @@ -1004,59 +1034,59 @@ TEST_CASE("SplitUp: SimpleDifferentQuotes2", "[helpers]") { TEST_CASE("SplitUp: Bracket1", "[helpers]") { std::vector oput = {"one", "[two, three]"}; std::string orig{"one, [two, three]"}; - std::vector result = CLI::detail::split_up(orig, ',', false); + std::vector result = CLI::detail::split_up(orig, ','); CHECK(result == oput); } TEST_CASE("SplitUp: Bracket2", "[helpers]") { std::vector oput = {"one", ""}; std::string orig{"one, "}; - std::vector result = CLI::detail::split_up(orig, ',', false); + std::vector result = CLI::detail::split_up(orig, ','); CHECK(result == oput); } TEST_CASE("SplitUp: Bracket3", "[helpers]") { std::vector oput = {"one", "(two, three)"}; std::string orig{"one, (two, three)"}; - std::vector result = CLI::detail::split_up(orig, ',', false); + std::vector result = CLI::detail::split_up(orig, ','); CHECK(result == oput); } TEST_CASE("SplitUp: Bracket4", "[helpers]") { std::vector oput = {"one", "{two, three}"}; std::string orig{"one, {two, three}"}; - std::vector result = CLI::detail::split_up(orig, ',', false); + std::vector result = CLI::detail::split_up(orig, ','); CHECK(result == oput); } TEST_CASE("SplitUp: Comment", "[helpers]") { std::vector oput = {R"(["quote1", "#"])"}; std::string orig{R"(["quote1", "#"])"}; - std::vector result = CLI::detail::split_up(orig, '#', false); + std::vector result = CLI::detail::split_up(orig, '#'); CHECK(result == oput); } TEST_CASE("SplitUp: Layered", "[helpers]") { - std::vector output = {R"(one 'two three')"}; + std::vector output = {R"("one 'two three'")"}; std::string orig{R"("one 'two three'")"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == output); } TEST_CASE("SplitUp: Spaces", "[helpers]") { - std::vector oput = {"one", " two three"}; + std::vector oput = {"one", "\" two three\""}; std::string orig{R"( one " two three" )"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); } TEST_CASE("SplitUp: BadStrings", "[helpers]") { - std::vector oput = {"one", " two three"}; + std::vector oput = {"one", "\" two three"}; std::string orig{R"( one " two three )"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); - oput = {"one", " two three"}; + oput = {"one", "' two three"}; orig = R"( one ' two three )"; result = CLI::detail::split_up(orig); CHECK(result == oput); From f6878b4ca344314c4e6a582e65c49869718deb5b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:39:56 +0000 Subject: [PATCH 02/25] style: pre-commit.ci fixes --- include/CLI/StringTools.hpp | 2 +- include/CLI/impl/Config_inl.hpp | 65 ++++---- include/CLI/impl/StringTools_inl.hpp | 233 +++++++++++---------------- tests/AppTest.cpp | 6 +- tests/ConfigFileTest.cpp | 3 +- tests/HelpersTest.cpp | 24 ++- 6 files changed, 144 insertions(+), 189 deletions(-) diff --git a/include/CLI/StringTools.hpp b/include/CLI/StringTools.hpp index a5df4bf32..4f4cfb58f 100644 --- a/include/CLI/StringTools.hpp +++ b/include/CLI/StringTools.hpp @@ -250,7 +250,7 @@ CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string); CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string); /// process a quoted string, remove the quotes and if appropriate handle escaped characters -CLI11_INLINE bool process_quoted_string(std::string& str, char string_char='\"', char literal_char='\''); +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char = '\"', char literal_char = '\''); } // namespace detail diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 04c611386..b56cbeaf5 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -19,14 +19,14 @@ namespace CLI { // [CLI11:config_inl_hpp:verbatim] - static constexpr auto multiline_literal_quote = R"(''')"; - static constexpr auto multiline_string_quote = R"(""")"; +static constexpr auto multiline_literal_quote = R"(''')"; +static constexpr auto multiline_string_quote = R"(""")"; namespace detail { CLI11_INLINE bool is_printable(const std::string &test_string) { return std::all_of(test_string.begin(), test_string.end(), [](char x) { - return (isprint(static_cast(x)) != 0 || x == '\n'||x=='\t'); + return (isprint(static_cast(x)) != 0 || x == '\n' || x == '\t'); }); } @@ -84,8 +84,7 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char literalQuote, return binary_escape_string(arg); } if(detail::has_escapable_character(arg)) { - if (arg.size() > 100 && !disable_multi_line) - { + if(arg.size() > 100 && !disable_multi_line) { return std::string(multiline_literal_quote) + arg + multiline_literal_quote; } return std::string(1, stringQuote) + detail::add_escaped_characters(arg) + stringQuote; @@ -276,16 +275,15 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons // comment lines if(line.front() == ';' || line.front() == '#' || line.front() == commentChar) { - continue; + continue; } - std::size_t search_start=0; - if (line.front() == stringQuote ||line.front()==literalQuote||line.front()=='`') - { - search_start=detail::close_sequence(line,1,line.front()); + std::size_t search_start = 0; + if(line.front() == stringQuote || line.front() == literalQuote || line.front() == '`') { + search_start = detail::close_sequence(line, 1, line.front()); } // Find = in string, split and recombine - auto delimiter_pos = line.find_first_of(valueDelimiter, search_start+1); - auto comment_pos = line.find_first_of(commentChar,search_start); + auto delimiter_pos = line.find_first_of(valueDelimiter, search_start + 1); + auto comment_pos = line.find_first_of(commentChar, search_start); if(comment_pos < delimiter_pos) { delimiter_pos = std::string::npos; } @@ -293,7 +291,8 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, delimiter_pos)); std::string item = detail::trim_copy(line.substr(delimiter_pos + 1, std::string::npos)); - bool mlquote = (item.compare(0, 3, multiline_literal_quote) == 0 || item.compare(0, 3, multiline_string_quote) == 0); + bool mlquote = + (item.compare(0, 3, multiline_literal_quote) == 0 || item.compare(0, 3, multiline_string_quote) == 0); if(!mlquote && comment_pos != std::string::npos && !literalName) { auto citems = detail::split_up(item, commentChar); item = detail::trim_copy(citems.front()); @@ -333,8 +332,8 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons if(!item.empty() && item.back() == '\n') { item.pop_back(); } - if (keyChar == '\"') { - item=detail::remove_escaped_characters(item); + if(keyChar == '\"') { + item = detail::remove_escaped_characters(item); } } else { if(lineExtension) { @@ -369,11 +368,11 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, comment_pos)); items_buffer = {"true"}; } - literalName=detail::process_quoted_string(name,stringQuote,literalQuote); - + literalName = detail::process_quoted_string(name, stringQuote, literalQuote); + // clean up quotes on the items and check for escaped strings for(auto &it : items_buffer) { - detail::process_quoted_string(it,stringQuote,literalQuote); + detail::process_quoted_string(it, stringQuote, literalQuote); } std::vector parents; if(literalName) { @@ -454,30 +453,28 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, continue; } } - std::string single_name=opt->get_single_name(); + std::string single_name = opt->get_single_name(); if(single_name.empty()) { continue; } - if(single_name.find_first_of(commentTest) != std::string::npos || single_name.compare(0, 3, multiline_string_quote) == 0 || - single_name.compare(0, 3, multiline_literal_quote) == 0 || (single_name.front() == '[' && single_name.back() == ']') || - (single_name.find_first_of(stringQuote)!=std::string::npos )|| - (single_name.find_first_of(literalQuote)!=std::string::npos )|| - (single_name.find_first_of('`') != std::string::npos)) { - if (single_name.find_first_of(literalQuote) == std::string::npos) - { - single_name.insert(0,1,literalQuote); + if(single_name.find_first_of(commentTest) != std::string::npos || + single_name.compare(0, 3, multiline_string_quote) == 0 || + single_name.compare(0, 3, multiline_literal_quote) == 0 || + (single_name.front() == '[' && single_name.back() == ']') || + (single_name.find_first_of(stringQuote) != std::string::npos) || + (single_name.find_first_of(literalQuote) != std::string::npos) || + (single_name.find_first_of('`') != std::string::npos)) { + if(single_name.find_first_of(literalQuote) == std::string::npos) { + single_name.insert(0, 1, literalQuote); single_name.push_back(literalQuote); - } - else - { - single_name.insert(0,1,stringQuote); + } else { + single_name.insert(0, 1, stringQuote); single_name.push_back(stringQuote); } - } std::string name = prefix + single_name; - + std::string value = detail::ini_join( opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote); @@ -516,7 +513,7 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, out << '\n'; out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n'; } - + out << name << valueDelimiter << value << '\n'; } } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 000982f08..fabeac7a8 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -61,7 +61,7 @@ CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter) { } CLI11_INLINE std::string &remove_quotes(std::string &str) { - if(str.length() > 1 && (str.front() == '"' || str.front() == '\''|| str.front() == '`')) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\'' || str.front() == '`')) { if(str.front() == str.back()) { str.pop_back(); str.erase(str.begin(), str.begin() + 1); @@ -70,8 +70,8 @@ CLI11_INLINE std::string &remove_quotes(std::string &str) { return str; } -CLI11_INLINE std::string &remove_outer(std::string &str,char key) { - if(str.length() > 1 && (str.front() == key) ){ +CLI11_INLINE std::string &remove_outer(std::string &str, char key) { + if(str.length() > 1 && (str.front() == key)) { if(str.front() == str.back()) { str.pop_back(); str.erase(str.begin(), str.begin() + 1); @@ -204,21 +204,18 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size() + 4); for(char s : str) { - auto sloc=escapedChars.find_first_of(s); + auto sloc = escapedChars.find_first_of(s); if(sloc != std::string::npos) { out.push_back('\\'); out.push_back(escapedCharsCode[sloc]); - } - else - { + } else { out.push_back(s); } } return out; } -CLI11_INLINE int hexConvert(char hc) -{ +CLI11_INLINE int hexConvert(char hc) { int res{0}; if(hc >= '0' && hc <= '9') { res = (hc - '0'); @@ -227,46 +224,38 @@ CLI11_INLINE int hexConvert(char hc) } else if(hc >= 'a' && hc <= 'f') { res = (hc - 'a' + 10); } else { - res=-1; + res = -1; } return res; } -CLI11_INLINE char make_char(std::uint32_t code) -{ - return static_cast(static_cast(code)); -} +CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(static_cast(code)); } -CLI11_INLINE void append_codepoint(std::string& str, std::uint32_t code) -{ - if(code < 0x80) // ascii code equivalent +CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { + if(code < 0x80) // ascii code equivalent { str.push_back(static_cast(code)); - } - else if(code < 0x800) //\u0080 to \u07FF + } else if(code < 0x800) //\u0080 to \u07FF { // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 - str.push_back(make_char(0xC0| code >> 6)); - str.push_back(make_char(0x80|(code & 0x3F))); - } - else if(code < 0x10000) // U+0800...U+FFFF + str.push_back(make_char(0xC0 | code >> 6)); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x10000) // U+0800...U+FFFF { - if(0xD800 <= code && code <= 0xDFFF) - { + if(0xD800 <= code && code <= 0xDFFF) { throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); } // 1110yyyy 10yxxxxx 10xxxxxx - str.push_back(make_char(0xE0| code >> 12)); - str.push_back(make_char(0x80|(code >> 6 & 0x3F))); - str.push_back(make_char(0x80|(code & 0x3F))); - } - else if(code < 0x110000) // U+010000 ... U+10FFFF + str.push_back(make_char(0xE0 | code >> 12)); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x110000) // U+010000 ... U+10FFFF { // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx - str.push_back(make_char(0xF0| code >> 18)); - str.push_back(make_char(0x80|(code >> 12 & 0x3F))); - str.push_back(make_char(0x80|(code >> 6 & 0x3F))); - str.push_back(make_char(0x80|(code & 0x3F))); + str.push_back(make_char(0xF0 | code >> 18)); + str.push_back(make_char(0x80 | (code >> 12 & 0x3F))); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); } } @@ -276,55 +265,49 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { out.reserve(str.size()); for(auto loc = str.begin(); loc < str.end(); ++loc) { if(*loc == '\\') { - auto ecloc=escapedCharsCode.find_first_of(*(loc+1)); + auto ecloc = escapedCharsCode.find_first_of(*(loc + 1)); if(ecloc != std::string::npos) { out.push_back(escapedChars[ecloc]); ++loc; - } else if (*(loc+1)=='u'){ - //must have 4 hex characters - if (str.end() - loc < 5) - { - throw std::invalid_argument("unicode sequence must have 4 hex codes "+str); + } else if(*(loc + 1) == 'u') { + // must have 4 hex characters + if(str.end() - loc < 5) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } int code{0}; - int mplier{16*16*16}; - for (int ii = 2; ii < 6; ++ii) - { - int res=hexConvert(*(loc+ii)); - if (res < 0) { - throw std::invalid_argument("unicode sequence must have 4 hex codes "+str); + int mplier{16 * 16 * 16}; + for(int ii = 2; ii < 6; ++ii) { + int res = hexConvert(*(loc + ii)); + if(res < 0) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } - code+=res*mplier; - mplier=mplier/16; + code += res * mplier; + mplier = mplier / 16; } - append_codepoint(out,code); - loc+=5; - }else if (*(loc+1)=='U'){ + append_codepoint(out, code); + loc += 5; + } else if(*(loc + 1) == 'U') { // must have 8 hex characters - if (str.end() - loc < 9) - { - throw std::invalid_argument("unicode sequence must have 8 hex codes "+str); + if(str.end() - loc < 9) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } int code{0}; - int mplier{16*16*16*16*16*16*16}; - for (int ii = 2; ii < 10; ++ii) - { - int res=hexConvert(*(loc+ii)); - if (res < 0) { - throw std::invalid_argument("unicode sequence must have 8 hex codes "+str); + int mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; + for(int ii = 2; ii < 10; ++ii) { + int res = hexConvert(*(loc + ii)); + if(res < 0) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } - code+=res*mplier; - mplier=mplier/16; + code += res * mplier; + mplier = mplier / 16; } - append_codepoint(out,code); - loc+=9; - } - else if (*(loc + 1) == '0') { + append_codepoint(out, code); + loc += 9; + } else if(*(loc + 1) == '0') { out.push_back('\0'); ++loc; - } - else{ - throw std::invalid_argument(std::string("unrecognized escape sequence \\")+*(loc+1)+" in "+str); + } else { + throw std::invalid_argument(std::string("unrecognized escape sequence \\") + *(loc + 1) + " in " + str); } } else { out.push_back(*loc); @@ -333,44 +316,39 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { return out; } -CLI11_INLINE std::size_t close_string_quote(const std::string& str, std::size_t start, char closure_char) -{ - std::size_t loc=start+1; - for (loc=start+1;loc split_up(std::string str, char delimiter) if(bracketChars.find_first_of(str[0]) != std::string::npos) { auto bracketLoc = bracketChars.find_first_of(str[0]); auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); - if (end >= std::string::npos-1) - { + if(end >= std::string::npos - 1) { output.push_back(str.substr(0)); str.clear(); - } - else - { - output.push_back(str.substr(0,end+1)); + } else { + output.push_back(str.substr(0, end + 1)); if(end + 2 < str.size()) { str = str.substr(end + 2); } else { str.clear(); } } - - + } else { auto it = std::find_if(std::begin(str), std::end(str), find_ws); if(it != std::end(str)) { @@ -526,12 +499,12 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string if(escaped_string[loc] == '\\' && (escaped_string[loc + 1] == 'x' || escaped_string[loc + 1] == 'X')) { auto c1 = escaped_string[loc + 2]; auto c2 = escaped_string[loc + 3]; - - int res1=hexConvert(c1); - int res2=hexConvert(c2); - if(res1>=0 && res2>=0) { + + int res1 = hexConvert(c1); + int res2 = hexConvert(c2); + if(res1 >= 0 && res2 >= 0) { loc += 4; - outstring.push_back(static_cast(res1*16+res2)); + outstring.push_back(static_cast(res1 * 16 + res2)); continue; } } @@ -541,48 +514,38 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string return outstring; } - CLI11_INLINE void remove_quotes(std::vector &args) { - for (auto& arg : args) - { - if (arg.front() == '\"' && arg.back() == '\"') - { + for(auto &arg : args) { + if(arg.front() == '\"' && arg.back() == '\"') { remove_quotes(arg); - arg=remove_escaped_characters(arg); - } - else - { + arg = remove_escaped_characters(arg); + } else { remove_quotes(arg); } } } -CLI11_INLINE bool process_quoted_string(std::string& str, char string_char, char literal_char) -{ - if (str.size() <= 1) - { +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char, char literal_char) { + if(str.size() <= 1) { return false; } if(detail::is_binary_escaped_string(str)) { - str=detail::extract_binary_string(str); + str = detail::extract_binary_string(str); return true; } - if (str.front() == string_char && str.back()==string_char) - { - detail::remove_outer(str,string_char); - if (str.find_first_of('\\') != std::string::npos) { - str=detail::remove_escaped_characters(str); + if(str.front() == string_char && str.back() == string_char) { + detail::remove_outer(str, string_char); + if(str.find_first_of('\\') != std::string::npos) { + str = detail::remove_escaped_characters(str); } return true; } - if (str.front() == literal_char && str.back()==literal_char) - { - detail::remove_outer(str,literal_char); + if(str.front() == literal_char && str.back() == literal_char) { + detail::remove_outer(str, literal_char); return true; } - if (str.front() == '`' && str.back() == '`') - { - detail::remove_outer(str,literal_char); + if(str.front() == '`' && str.back() == '`') { + detail::remove_outer(str, literal_char); return true; } return false; diff --git a/tests/AppTest.cpp b/tests/AppTest.cpp index da8197cb4..2839ff904 100644 --- a/tests/AppTest.cpp +++ b/tests/AppTest.cpp @@ -415,9 +415,9 @@ TEST_CASE_METHOD(TApp, "OneStringEqualVersionSingleStringQuotedEscapedCharacters app.add_option("-t,--tstr", str2); app.add_option("-m,--mstr", str3); app.parse(R"raw(--string="this is my \n\"quoted\" string" -t 'qst\ring 2' -m=`"quoted\n string"`")raw"); - CHECK("this is my \n\"quoted\" string" == str); //escaped - CHECK("qst\\ring 2" == str2); //literal - CHECK("\"quoted\\n string\"" == str3); //double quoted literal + CHECK("this is my \n\"quoted\" string" == str); // escaped + CHECK("qst\\ring 2" == str2); // literal + CHECK("\"quoted\\n string\"" == str3); // double quoted literal } TEST_CASE_METHOD(TApp, "OneStringEqualVersionSingleStringQuotedMultipleWithEqual", "[app]") { diff --git a/tests/ConfigFileTest.cpp b/tests/ConfigFileTest.cpp index 710d748db..20b8d7559 100644 --- a/tests/ConfigFileTest.cpp +++ b/tests/ConfigFileTest.cpp @@ -2713,7 +2713,8 @@ TEST_CASE_METHOD(TApp, "TomlOutputMultilineString", "[config]") { std::string desc = "flag"; app.add_option("--opt", desc); - std::string argString = "this is a very long string \n that covers multiple lines \nand should be longer than 100 characters \nto trigger the multiline string"; + std::string argString = "this is a very long string \n that covers multiple lines \nand should be longer than 100 " + "characters \nto trigger the multiline string"; args = {"--opt", argString}; run(); diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index 22ba3758b..7ae4803cd 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -302,16 +302,10 @@ TEST_CASE("StringTools: binaryStrings", "[helpers]") { } /// these are provided for compatibility with the char8_t for C++20 that breaks stuff -std::string from_u8string(const std::string &s) { - return s; -} -std::string from_u8string(std::string &&s) { - return std::move(s); -} +std::string from_u8string(const std::string &s) { return s; } +std::string from_u8string(std::string &&s) { return std::move(s); } #if defined(__cpp_lib_char8_t) -std::string from_u8string(const std::u8string &s) { - return std::string(s.begin(), s.end()); -} +std::string from_u8string(const std::u8string &s) { return std::string(s.begin(), s.end()); } #endif TEST_CASE("StringTools: escapeConversion", "[helpers]") { @@ -324,18 +318,18 @@ TEST_CASE("StringTools: escapeConversion", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\f") == "test\f"); CHECK(CLI::detail::remove_escaped_characters("test\\ttest\\n") == "test\ttest\n"); - CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\m_bad"),std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\m_bad"), std::invalid_argument); } TEST_CASE("StringTools: unicode_literals", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\u03C0\\u00e9") == from_u8string(u8"test\u03C0\u00E9")); - + CHECK(CLI::detail::remove_escaped_characters("test\\U0001F600\\u00E9") == from_u8string(u8"test\U0001F600\u00E9")); - CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001M600\\u00E9"),std::invalid_argument); - CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\u00M9"),std::invalid_argument); - CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\uD8E9"),std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001M600\\u00E9"), std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\u00M9"), std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\uD8E9"), std::invalid_argument); } TEST_CASE("Trim: Various", "[helpers]") { @@ -1018,7 +1012,7 @@ TEST_CASE("SplitUp: SimpleMissingQuotes", "[helpers]") { } TEST_CASE("SplitUp: SimpleMissingQuotesEscaped", "[helpers]") { - std::vector oput = { "one", "\"two three\\\"\""}; + std::vector oput = {"one", "\"two three\\\"\""}; std::string orig{R"(one "two three\"")"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); From b8c0eb0d72dacdcb52c72bfb30e96b744dfe47f9 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Mon, 25 Dec 2023 06:57:43 -0800 Subject: [PATCH 03/25] clang-tidy fixes --- include/CLI/impl/StringTools_inl.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index fabeac7a8..d86f430f5 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -215,8 +215,8 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str) { return out; } -CLI11_INLINE int hexConvert(char hc) { - int res{0}; +CLI11_INLINE std::uint32_t hexConvert(char hc) { + std::uint32_t res{0}; if(hc >= '0' && hc <= '9') { res = (hc - '0'); } else if(hc >= 'A' && hc <= 'F') { @@ -224,7 +224,7 @@ CLI11_INLINE int hexConvert(char hc) { } else if(hc >= 'a' && hc <= 'f') { res = (hc - 'a' + 10); } else { - res = -1; + res = 0xFFFF'FFFF; } return res; } @@ -274,10 +274,10 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { if(str.end() - loc < 5) { throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } - int code{0}; - int mplier{16 * 16 * 16}; + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16}; for(int ii = 2; ii < 6; ++ii) { - int res = hexConvert(*(loc + ii)); + std::uint32_t res = hexConvert(*(loc + ii)); if(res < 0) { throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } @@ -291,10 +291,10 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { if(str.end() - loc < 9) { throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } - int code{0}; - int mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; for(int ii = 2; ii < 10; ++ii) { - int res = hexConvert(*(loc + ii)); + std::uint32_t res = hexConvert(*(loc + ii)); if(res < 0) { throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } @@ -317,7 +317,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { } CLI11_INLINE std::size_t close_string_quote(const std::string &str, std::size_t start, char closure_char) { - std::size_t loc = start + 1; + std::size_t loc{0}; for(loc = start + 1; loc < str.size(); ++loc) { if(str[loc] == closure_char) { break; From 2f5001daa5ede163bc014494c97392c7da5cb145 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Mon, 25 Dec 2023 08:45:26 -0800 Subject: [PATCH 04/25] fix missing catch of argument non-processing --- include/CLI/impl/App_inl.hpp | 2 +- include/CLI/impl/StringTools_inl.hpp | 10 +++++----- tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail33 | 2 ++ 4 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail33 diff --git a/include/CLI/impl/App_inl.hpp b/include/CLI/impl/App_inl.hpp index a459e19ce..99c7c511f 100644 --- a/include/CLI/impl/App_inl.hpp +++ b/include/CLI/impl/App_inl.hpp @@ -1570,7 +1570,7 @@ CLI11_INLINE bool App::_parse_single(std::vector &args, bool &posit case detail::Classifier::SHORT: case detail::Classifier::WINDOWS_STYLE: // If already parsed a subcommand, don't accept options_ - _parse_arg(args, classifier, false); + retval=_parse_arg(args, classifier, false); break; case detail::Classifier::NONE: // Probably a positional or something for a parent (sub)command diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index d86f430f5..7ae55bcb8 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -278,7 +278,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::uint32_t mplier{16 * 16 * 16}; for(int ii = 2; ii < 6; ++ii) { std::uint32_t res = hexConvert(*(loc + ii)); - if(res < 0) { + if(res >0x0F) { throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } code += res * mplier; @@ -295,7 +295,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::uint32_t mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; for(int ii = 2; ii < 10; ++ii) { std::uint32_t res = hexConvert(*(loc + ii)); - if(res < 0) { + if(res >0x0F) { throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } code += res * mplier; @@ -500,9 +500,9 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string auto c1 = escaped_string[loc + 2]; auto c2 = escaped_string[loc + 3]; - int res1 = hexConvert(c1); - int res2 = hexConvert(c2); - if(res1 >= 0 && res2 >= 0) { + std::uint32_t res1 = hexConvert(c1); + std::uint32_t res2 = hexConvert(c2); + if (res1 <= 0x0F && res2 <= 0x0F) { loc += 4; outstring.push_back(static_cast(res1 * 16 + res2)); continue; diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index fd6af2108..e9bb5e595 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 33)); + int index = GENERATE(range(1, 34)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail33 b/tests/fuzzFail/fuzz_app_file_fail33 new file mode 100644 index 000000000..a7b45f72b --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail33 @@ -0,0 +1,2 @@ +'''-$ú +$ \ No newline at end of file From 648efb9080be699dadc208fb9413ed817ffbac89 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 16:45:47 +0000 Subject: [PATCH 05/25] style: pre-commit.ci fixes --- include/CLI/impl/App_inl.hpp | 2 +- include/CLI/impl/StringTools_inl.hpp | 6 +++--- tests/fuzzFail/fuzz_app_file_fail33 | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/CLI/impl/App_inl.hpp b/include/CLI/impl/App_inl.hpp index 99c7c511f..41fa45c11 100644 --- a/include/CLI/impl/App_inl.hpp +++ b/include/CLI/impl/App_inl.hpp @@ -1570,7 +1570,7 @@ CLI11_INLINE bool App::_parse_single(std::vector &args, bool &posit case detail::Classifier::SHORT: case detail::Classifier::WINDOWS_STYLE: // If already parsed a subcommand, don't accept options_ - retval=_parse_arg(args, classifier, false); + retval = _parse_arg(args, classifier, false); break; case detail::Classifier::NONE: // Probably a positional or something for a parent (sub)command diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 7ae55bcb8..9f66c764d 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -278,7 +278,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::uint32_t mplier{16 * 16 * 16}; for(int ii = 2; ii < 6; ++ii) { std::uint32_t res = hexConvert(*(loc + ii)); - if(res >0x0F) { + if(res > 0x0F) { throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } code += res * mplier; @@ -295,7 +295,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::uint32_t mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; for(int ii = 2; ii < 10; ++ii) { std::uint32_t res = hexConvert(*(loc + ii)); - if(res >0x0F) { + if(res > 0x0F) { throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } code += res * mplier; @@ -502,7 +502,7 @@ CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string std::uint32_t res1 = hexConvert(c1); std::uint32_t res2 = hexConvert(c2); - if (res1 <= 0x0F && res2 <= 0x0F) { + if(res1 <= 0x0F && res2 <= 0x0F) { loc += 4; outstring.push_back(static_cast(res1 * 16 + res2)); continue; diff --git a/tests/fuzzFail/fuzz_app_file_fail33 b/tests/fuzzFail/fuzz_app_file_fail33 index a7b45f72b..18e61dff6 100644 --- a/tests/fuzzFail/fuzz_app_file_fail33 +++ b/tests/fuzzFail/fuzz_app_file_fail33 @@ -1,2 +1,2 @@ '''-$ú -$ \ No newline at end of file +$ From 2eb5536a8e5f976ba523f197cc8bb50819a5186c Mon Sep 17 00:00:00 2001 From: Philip Top Date: Mon, 25 Dec 2023 12:52:50 -0800 Subject: [PATCH 06/25] add catch for invalid_argument to parse error, for invalid quoted strings --- include/CLI/impl/App_inl.hpp | 9 ++++++++- include/CLI/impl/StringTools_inl.hpp | 25 +++++++++++-------------- tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail34 | 1 + 4 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail34 diff --git a/include/CLI/impl/App_inl.hpp b/include/CLI/impl/App_inl.hpp index 41fa45c11..bdede61a2 100644 --- a/include/CLI/impl/App_inl.hpp +++ b/include/CLI/impl/App_inl.hpp @@ -579,7 +579,14 @@ CLI11_INLINE void App::parse(std::string commandline, bool program_name_included auto args = detail::split_up(std::move(commandline)); // remove all empty strings args.erase(std::remove(args.begin(), args.end(), std::string{}), args.end()); - detail::remove_quotes(args); + try + { + detail::remove_quotes(args); + } + catch (const std::invalid_argument& arg) + { + throw CLI::ParseError(arg.what(), CLI::ExitCodes::InvalidError); + } std::reverse(args.begin(), args.end()); parse(std::move(args)); } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 9f66c764d..3b96e34d8 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -216,32 +216,29 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str) { } CLI11_INLINE std::uint32_t hexConvert(char hc) { - std::uint32_t res{0}; + int hcode{ 0 }; if(hc >= '0' && hc <= '9') { - res = (hc - '0'); + hcode = (hc - '0'); } else if(hc >= 'A' && hc <= 'F') { - res = (hc - 'A' + 10); + hcode = (hc - 'A' + 10); } else if(hc >= 'a' && hc <= 'f') { - res = (hc - 'a' + 10); + hcode = (hc - 'a' + 10); } else { - res = 0xFFFF'FFFF; + hcode = -1; } - return res; + return static_cast(hcode); } CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(static_cast(code)); } CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { - if(code < 0x80) // ascii code equivalent - { + if(code < 0x80) {// ascii code equivalent str.push_back(static_cast(code)); - } else if(code < 0x800) //\u0080 to \u07FF - { + } else if(code < 0x800) { //\u0080 to \u07FF // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 str.push_back(make_char(0xC0 | code >> 6)); str.push_back(make_char(0x80 | (code & 0x3F))); - } else if(code < 0x10000) // U+0800...U+FFFF - { + } else if(code < 0x10000) {// U+0800...U+FFFF if(0xD800 <= code && code <= 0xDFFF) { throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); } @@ -249,8 +246,7 @@ CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { str.push_back(make_char(0xE0 | code >> 12)); str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); str.push_back(make_char(0x80 | (code & 0x3F))); - } else if(code < 0x110000) // U+010000 ... U+10FFFF - { + } else if(code < 0x110000) {// U+010000 ... U+10FFFF // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx str.push_back(make_char(0xF0 | code >> 18)); str.push_back(make_char(0x80 | (code >> 12 & 0x3F))); @@ -518,6 +514,7 @@ CLI11_INLINE void remove_quotes(std::vector &args) { for(auto &arg : args) { if(arg.front() == '\"' && arg.back() == '\"') { remove_quotes(arg); + //only remove escaped for string arguments not literal strings arg = remove_escaped_characters(arg); } else { remove_quotes(arg); diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index e9bb5e595..53a6a5726 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 34)); + int index = GENERATE(range(34, 35)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail34 b/tests/fuzzFail/fuzz_app_file_fail34 new file mode 100644 index 000000000..6f0f81517 --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail34 @@ -0,0 +1 @@ +" (\\\,"‚ãã \ No newline at end of file From 073a3f3265ae064878485125af69fc8c0e8e39b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 21:21:48 +0000 Subject: [PATCH 07/25] style: pre-commit.ci fixes --- include/CLI/impl/App_inl.hpp | 7 ++----- include/CLI/impl/StringTools_inl.hpp | 12 ++++++------ tests/fuzzFail/fuzz_app_file_fail34 | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/include/CLI/impl/App_inl.hpp b/include/CLI/impl/App_inl.hpp index bdede61a2..f258f7308 100644 --- a/include/CLI/impl/App_inl.hpp +++ b/include/CLI/impl/App_inl.hpp @@ -579,12 +579,9 @@ CLI11_INLINE void App::parse(std::string commandline, bool program_name_included auto args = detail::split_up(std::move(commandline)); // remove all empty strings args.erase(std::remove(args.begin(), args.end(), std::string{}), args.end()); - try - { + try { detail::remove_quotes(args); - } - catch (const std::invalid_argument& arg) - { + } catch(const std::invalid_argument &arg) { throw CLI::ParseError(arg.what(), CLI::ExitCodes::InvalidError); } std::reverse(args.begin(), args.end()); diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 3b96e34d8..d6eba5af8 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -216,7 +216,7 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str) { } CLI11_INLINE std::uint32_t hexConvert(char hc) { - int hcode{ 0 }; + int hcode{0}; if(hc >= '0' && hc <= '9') { hcode = (hc - '0'); } else if(hc >= 'A' && hc <= 'F') { @@ -232,13 +232,13 @@ CLI11_INLINE std::uint32_t hexConvert(char hc) { CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(static_cast(code)); } CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { - if(code < 0x80) {// ascii code equivalent + if(code < 0x80) { // ascii code equivalent str.push_back(static_cast(code)); - } else if(code < 0x800) { //\u0080 to \u07FF + } else if(code < 0x800) { //\u0080 to \u07FF // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 str.push_back(make_char(0xC0 | code >> 6)); str.push_back(make_char(0x80 | (code & 0x3F))); - } else if(code < 0x10000) {// U+0800...U+FFFF + } else if(code < 0x10000) { // U+0800...U+FFFF if(0xD800 <= code && code <= 0xDFFF) { throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); } @@ -246,7 +246,7 @@ CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { str.push_back(make_char(0xE0 | code >> 12)); str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); str.push_back(make_char(0x80 | (code & 0x3F))); - } else if(code < 0x110000) {// U+010000 ... U+10FFFF + } else if(code < 0x110000) { // U+010000 ... U+10FFFF // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx str.push_back(make_char(0xF0 | code >> 18)); str.push_back(make_char(0x80 | (code >> 12 & 0x3F))); @@ -514,7 +514,7 @@ CLI11_INLINE void remove_quotes(std::vector &args) { for(auto &arg : args) { if(arg.front() == '\"' && arg.back() == '\"') { remove_quotes(arg); - //only remove escaped for string arguments not literal strings + // only remove escaped for string arguments not literal strings arg = remove_escaped_characters(arg); } else { remove_quotes(arg); diff --git a/tests/fuzzFail/fuzz_app_file_fail34 b/tests/fuzzFail/fuzz_app_file_fail34 index 6f0f81517..297cbdccd 100644 --- a/tests/fuzzFail/fuzz_app_file_fail34 +++ b/tests/fuzzFail/fuzz_app_file_fail34 @@ -1 +1 @@ -" (\\\,"‚ãã \ No newline at end of file +" (\\\,"‚ãã From 2b7ea018df536c0b86e7a8aeada3ceee24f0a1ac Mon Sep 17 00:00:00 2001 From: Philip Top Date: Tue, 26 Dec 2023 06:34:22 -0800 Subject: [PATCH 08/25] add escape sequences for config option names if needed --- include/CLI/impl/Config_inl.hpp | 3 +++ tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail35 | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail35 diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index b56cbeaf5..bebaeddfd 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -468,6 +468,9 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, single_name.insert(0, 1, literalQuote); single_name.push_back(literalQuote); } else { + if (detail::has_escapable_character(single_name)) { + single_name=detail::add_escaped_characters(single_name); + } single_name.insert(0, 1, stringQuote); single_name.push_back(stringQuote); } diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index 53a6a5726..ebbef9b2f 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(34, 35)); + int index = GENERATE(range(35, 36)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail35 b/tests/fuzzFail/fuzz_app_file_fail35 new file mode 100644 index 000000000..282c98a9e --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail35 @@ -0,0 +1 @@ +'^^^^^^^\^^^^^^''''''@''i¦ \ No newline at end of file From 03e74e1a3c72d9a54f86843e3638e8cd4e05c75d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:34:58 +0000 Subject: [PATCH 09/25] style: pre-commit.ci fixes --- include/CLI/impl/Config_inl.hpp | 4 ++-- tests/fuzzFail/fuzz_app_file_fail35 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index bebaeddfd..343ad1e33 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -468,8 +468,8 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, single_name.insert(0, 1, literalQuote); single_name.push_back(literalQuote); } else { - if (detail::has_escapable_character(single_name)) { - single_name=detail::add_escaped_characters(single_name); + if(detail::has_escapable_character(single_name)) { + single_name = detail::add_escaped_characters(single_name); } single_name.insert(0, 1, stringQuote); single_name.push_back(stringQuote); diff --git a/tests/fuzzFail/fuzz_app_file_fail35 b/tests/fuzzFail/fuzz_app_file_fail35 index 282c98a9e..d9b5aa7c4 100644 --- a/tests/fuzzFail/fuzz_app_file_fail35 +++ b/tests/fuzzFail/fuzz_app_file_fail35 @@ -1 +1 @@ -'^^^^^^^\^^^^^^''''''@''i¦ \ No newline at end of file +'^^^^^^^\^^^^^^''''''@''i¦ From 1fcd88d4b35270275af8a9e1b3723057d2016498 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Tue, 26 Dec 2023 07:09:08 -0800 Subject: [PATCH 10/25] fix cpplint --- include/CLI/impl/StringTools_inl.hpp | 2 +- tests/FuzzFailTest.cpp | 2 +- tests/HelpersTest.cpp | 2 ++ tests/fuzzFail/fuzz_app_file_fail36 | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail36 diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index d6eba5af8..889a79ac1 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -234,7 +234,7 @@ CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(stati CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { if(code < 0x80) { // ascii code equivalent str.push_back(static_cast(code)); - } else if(code < 0x800) { //\u0080 to \u07FF + } else if(code < 0x800) { // \u0080 to \u07FF // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 str.push_back(make_char(0xC0 | code >> 6)); str.push_back(make_char(0x80 | (code & 0x3F))); diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index ebbef9b2f..9de07af22 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(35, 36)); + int index = GENERATE(range(1, 37)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index 7ae4803cd..c5ca8a0da 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -306,6 +306,8 @@ std::string from_u8string(const std::string &s) { return s; } std::string from_u8string(std::string &&s) { return std::move(s); } #if defined(__cpp_lib_char8_t) std::string from_u8string(const std::u8string &s) { return std::string(s.begin(), s.end()); } +#elif defined(__cpp_char8_t) +std::string from_u8string(const char8_t *s) { return std::string(reinterpret_cast(s)); } #endif TEST_CASE("StringTools: escapeConversion", "[helpers]") { diff --git a/tests/fuzzFail/fuzz_app_file_fail36 b/tests/fuzzFail/fuzz_app_file_fail36 new file mode 100644 index 000000000..dea8e482d --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail36 @@ -0,0 +1 @@ +"\ " \ No newline at end of file From f65da7a5105f94529e03817e535f08cde118ccff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:09:29 +0000 Subject: [PATCH 11/25] style: pre-commit.ci fixes --- tests/fuzzFail/fuzz_app_file_fail36 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzFail/fuzz_app_file_fail36 b/tests/fuzzFail/fuzz_app_file_fail36 index dea8e482d..ddd11facc 100644 --- a/tests/fuzzFail/fuzz_app_file_fail36 +++ b/tests/fuzzFail/fuzz_app_file_fail36 @@ -1 +1 @@ -"\ " \ No newline at end of file +"\ " From 05f2932b57f653466eec7498a82fa1f8a6fc044b Mon Sep 17 00:00:00 2001 From: Philip Top Date: Tue, 26 Dec 2023 08:27:18 -0800 Subject: [PATCH 12/25] fix a few bugs and other warnings --- include/CLI/impl/Config_inl.hpp | 47 ++++++++++++++-------------- include/CLI/impl/StringTools_inl.hpp | 2 +- tests/FuzzFailTest.cpp | 2 +- tests/HelpersTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail37 | 1 + 5 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail37 diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 343ad1e33..7a4b7dd25 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -279,7 +279,7 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons } std::size_t search_start = 0; if(line.front() == stringQuote || line.front() == literalQuote || line.front() == '`') { - search_start = detail::close_sequence(line, 1, line.front()); + search_start = detail::close_sequence(line, 0, line.front()); } // Find = in string, split and recombine auto delimiter_pos = line.find_first_of(valueDelimiter, search_start + 1); @@ -457,26 +457,6 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, if(single_name.empty()) { continue; } - if(single_name.find_first_of(commentTest) != std::string::npos || - single_name.compare(0, 3, multiline_string_quote) == 0 || - single_name.compare(0, 3, multiline_literal_quote) == 0 || - (single_name.front() == '[' && single_name.back() == ']') || - (single_name.find_first_of(stringQuote) != std::string::npos) || - (single_name.find_first_of(literalQuote) != std::string::npos) || - (single_name.find_first_of('`') != std::string::npos)) { - if(single_name.find_first_of(literalQuote) == std::string::npos) { - single_name.insert(0, 1, literalQuote); - single_name.push_back(literalQuote); - } else { - if(detail::has_escapable_character(single_name)) { - single_name = detail::add_escaped_characters(single_name); - } - single_name.insert(0, 1, stringQuote); - single_name.push_back(stringQuote); - } - } - - std::string name = prefix + single_name; std::string value = detail::ini_join( opt->reduced_results(), arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote); @@ -492,15 +472,16 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, } if(!value.empty()) { + if(!opt->get_fnames().empty()) { try { - value = opt->get_flag_value(name, value); + value = opt->get_flag_value(single_name, value); } catch(const CLI::ArgumentMismatch &) { bool valid{false}; for(const auto &test_name : opt->get_fnames()) { try { value = opt->get_flag_value(test_name, value); - name = test_name; + single_name = test_name; valid = true; } catch(const CLI::ArgumentMismatch &) { continue; @@ -516,6 +497,26 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, out << '\n'; out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n'; } + if(single_name.find_first_of(commentTest) != std::string::npos || + single_name.compare(0, 3, multiline_string_quote) == 0 || + single_name.compare(0, 3, multiline_literal_quote) == 0 || + (single_name.front() == '[' && single_name.back() == ']') || + (single_name.find_first_of(stringQuote) != std::string::npos) || + (single_name.find_first_of(literalQuote) != std::string::npos) || + (single_name.find_first_of('`') != std::string::npos)) { + if(single_name.find_first_of(literalQuote) == std::string::npos) { + single_name.insert(0, 1, literalQuote); + single_name.push_back(literalQuote); + } else { + if(detail::has_escapable_character(single_name)) { + single_name = detail::add_escaped_characters(single_name); + } + single_name.insert(0, 1, stringQuote); + single_name.push_back(stringQuote); + } + } + + std::string name = prefix + single_name; out << name << valueDelimiter << value << '\n'; } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 889a79ac1..b3bfe3846 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -390,7 +390,7 @@ CLI11_INLINE std::vector split_up(std::string str, char delimiter) auto bracketLoc = bracketChars.find_first_of(str[0]); auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); if(end >= std::string::npos - 1) { - output.push_back(str.substr(0)); + output.push_back(std::move(str)); str.clear(); } else { output.push_back(str.substr(0, end + 1)); diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index 9de07af22..323e30332 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 37)); + int index = GENERATE(range(37, 38)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index c5ca8a0da..e7b0a9ce1 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -1014,7 +1014,7 @@ TEST_CASE("SplitUp: SimpleMissingQuotes", "[helpers]") { } TEST_CASE("SplitUp: SimpleMissingQuotesEscaped", "[helpers]") { - std::vector oput = {"one", "\"two three\\\"\""}; + std::vector oput = {"one", R"("two three\"")"}; std::string orig{R"(one "two three\"")"}; std::vector result = CLI::detail::split_up(orig); CHECK(result == oput); diff --git a/tests/fuzzFail/fuzz_app_file_fail37 b/tests/fuzzFail/fuzz_app_file_fail37 new file mode 100644 index 000000000..5b0ecd379 --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail37 @@ -0,0 +1 @@ +"Ü-t2ÿÿÿÿp'--vopt1'â''e#ÿÿ'â''e \ No newline at end of file From 910cbe0d630fb0aaf3e35b4eb4c2a5c4cab77a18 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:27:37 +0000 Subject: [PATCH 13/25] style: pre-commit.ci fixes --- include/CLI/impl/Config_inl.hpp | 14 +++++++------- tests/fuzzFail/fuzz_app_file_fail37 | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 7a4b7dd25..f5214b924 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -472,7 +472,7 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, } if(!value.empty()) { - + if(!opt->get_fnames().empty()) { try { value = opt->get_flag_value(single_name, value); @@ -498,12 +498,12 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description, out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n'; } if(single_name.find_first_of(commentTest) != std::string::npos || - single_name.compare(0, 3, multiline_string_quote) == 0 || - single_name.compare(0, 3, multiline_literal_quote) == 0 || - (single_name.front() == '[' && single_name.back() == ']') || - (single_name.find_first_of(stringQuote) != std::string::npos) || - (single_name.find_first_of(literalQuote) != std::string::npos) || - (single_name.find_first_of('`') != std::string::npos)) { + single_name.compare(0, 3, multiline_string_quote) == 0 || + single_name.compare(0, 3, multiline_literal_quote) == 0 || + (single_name.front() == '[' && single_name.back() == ']') || + (single_name.find_first_of(stringQuote) != std::string::npos) || + (single_name.find_first_of(literalQuote) != std::string::npos) || + (single_name.find_first_of('`') != std::string::npos)) { if(single_name.find_first_of(literalQuote) == std::string::npos) { single_name.insert(0, 1, literalQuote); single_name.push_back(literalQuote); diff --git a/tests/fuzzFail/fuzz_app_file_fail37 b/tests/fuzzFail/fuzz_app_file_fail37 index 5b0ecd379..25d8567d6 100644 --- a/tests/fuzzFail/fuzz_app_file_fail37 +++ b/tests/fuzzFail/fuzz_app_file_fail37 @@ -1 +1 @@ -"Ü-t2ÿÿÿÿp'--vopt1'â''e#ÿÿ'â''e \ No newline at end of file +"Ü-t2ÿÿÿÿp'--vopt1'â''e#ÿÿ'â''e From 63b0a35c563e964b9d1b9b1789f820180a5256c1 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Tue, 26 Dec 2023 09:55:12 -0800 Subject: [PATCH 14/25] add fuzz test that doesn't seem to be broken --- tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail38 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail38 diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index 323e30332..1daf76343 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(37, 38)); + int index = GENERATE(range(1, 39)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail38 b/tests/fuzzFail/fuzz_app_file_fail38 new file mode 100644 index 000000000..2e7a1bb1e --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail38 @@ -0,0 +1 @@ +"\ÿ" \ No newline at end of file From 9c1b09d52cebe3dd4825fa69c44b664472742b14 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 17:55:31 +0000 Subject: [PATCH 15/25] style: pre-commit.ci fixes --- tests/fuzzFail/fuzz_app_file_fail38 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzFail/fuzz_app_file_fail38 b/tests/fuzzFail/fuzz_app_file_fail38 index 2e7a1bb1e..607bce903 100644 --- a/tests/fuzzFail/fuzz_app_file_fail38 +++ b/tests/fuzzFail/fuzz_app_file_fail38 @@ -1 +1 @@ -"\ÿ" \ No newline at end of file +"\ÿ" From 9b144f54970e8b6788348a5ef5112e4d827546cf Mon Sep 17 00:00:00 2001 From: Philip Top Date: Tue, 26 Dec 2023 10:30:14 -0800 Subject: [PATCH 16/25] test the file fail --- include/CLI/StringTools.hpp | 4 ++++ include/CLI/impl/Config_inl.hpp | 15 +++++++++++---- include/CLI/impl/StringTools_inl.hpp | 5 ++++- tests/FuzzFailTest.cpp | 4 ++-- .../{fuzz_app_file_fail38 => fuzz_file_fail3} | 0 tests/fuzzFail/fuzz_file_fail4 | 1 + 6 files changed, 22 insertions(+), 7 deletions(-) rename tests/fuzzFail/{fuzz_app_file_fail38 => fuzz_file_fail3} (100%) create mode 100644 tests/fuzzFail/fuzz_file_fail4 diff --git a/include/CLI/StringTools.hpp b/include/CLI/StringTools.hpp index 4f4cfb58f..1922d9bf8 100644 --- a/include/CLI/StringTools.hpp +++ b/include/CLI/StringTools.hpp @@ -215,6 +215,10 @@ template inline std::string find_and_modify(std::string str, return str; } +/// close a sequence of characters indicated by a closure character. Brackets allows sub sequences +/// recognized bracket sequences include "'`[(<{ other closure characters are assumed to be literal strings +CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char); + /// Split a string '"one two" "three"' into 'one two', 'three' /// Quote characters can be ` ' or " or bracket characters [{(< with matching to the matching bracket CLI11_INLINE std::vector split_up(std::string str, char delimiter = '\0'); diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index f5214b924..7411c59af 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -368,11 +368,18 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, comment_pos)); items_buffer = {"true"}; } - literalName = detail::process_quoted_string(name, stringQuote, literalQuote); + try + { + literalName = detail::process_quoted_string(name, stringQuote, literalQuote); - // clean up quotes on the items and check for escaped strings - for(auto &it : items_buffer) { - detail::process_quoted_string(it, stringQuote, literalQuote); + // clean up quotes on the items and check for escaped strings + for(auto &it : items_buffer) { + detail::process_quoted_string(it, stringQuote, literalQuote); + } + } + catch (const std::invalid_argument& ia) + { + throw CLI::ParseError(ia.what(), CLI::ExitCodes::InvalidError); } std::vector parents; if(literalName) { diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index b3bfe3846..0de3e3d18 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -259,8 +259,11 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size()); - for(auto loc = str.begin(); loc < str.end(); ++loc) { + for(auto loc = str.begin(); loc != str.end(); ++loc) { if(*loc == '\\') { + if(str.end() - loc < 2) { + throw std::invalid_argument("invalid escape sequence " + str); + } auto ecloc = escapedCharsCode.find_first_of(*(loc + 1)); if(ecloc != std::string::npos) { out.push_back(escapedChars[ecloc]); diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index 1daf76343..c4ee75299 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -50,7 +50,7 @@ TEST_CASE("file_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 3)); + int index = GENERATE(range(1, 5)); auto parseData = loadFailureFile("fuzz_file_fail", index); std::stringstream out(parseData); try { @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 39)); + int index = GENERATE(range(1,38)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail38 b/tests/fuzzFail/fuzz_file_fail3 similarity index 100% rename from tests/fuzzFail/fuzz_app_file_fail38 rename to tests/fuzzFail/fuzz_file_fail3 diff --git a/tests/fuzzFail/fuzz_file_fail4 b/tests/fuzzFail/fuzz_file_fail4 new file mode 100644 index 000000000..bb0b720ad --- /dev/null +++ b/tests/fuzzFail/fuzz_file_fail4 @@ -0,0 +1 @@ +""\" \ No newline at end of file From a5226e91dfc068eae1e1029fab2b14cfdeb089d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 18:30:32 +0000 Subject: [PATCH 17/25] style: pre-commit.ci fixes --- include/CLI/impl/Config_inl.hpp | 7 ++----- tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_file_fail4 | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 7411c59af..7b6188ecd 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -368,17 +368,14 @@ inline std::vector ConfigBase::from_config(std::istream &input) cons name = detail::trim_copy(line.substr(0, comment_pos)); items_buffer = {"true"}; } - try - { + try { literalName = detail::process_quoted_string(name, stringQuote, literalQuote); // clean up quotes on the items and check for escaped strings for(auto &it : items_buffer) { detail::process_quoted_string(it, stringQuote, literalQuote); } - } - catch (const std::invalid_argument& ia) - { + } catch(const std::invalid_argument &ia) { throw CLI::ParseError(ia.what(), CLI::ExitCodes::InvalidError); } std::vector parents; diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index c4ee75299..0eb50e2cd 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1,38)); + int index = GENERATE(range(1, 38)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_file_fail4 b/tests/fuzzFail/fuzz_file_fail4 index bb0b720ad..e7aac1a29 100644 --- a/tests/fuzzFail/fuzz_file_fail4 +++ b/tests/fuzzFail/fuzz_file_fail4 @@ -1 +1 @@ -""\" \ No newline at end of file +""\" From b4574d49153d9b0f60a3358b1fe0fae5f5856f7d Mon Sep 17 00:00:00 2001 From: Philip Top Date: Wed, 27 Dec 2023 07:54:45 -0800 Subject: [PATCH 18/25] fix bug from fuzzer, and add some tests for coverage --- include/CLI/impl/Option_inl.hpp | 4 +- include/CLI/impl/StringTools_inl.hpp | 17 ++--- tests/FuzzFailTest.cpp | 2 +- tests/HelpersTest.cpp | 104 ++++++++++++++++++++++++++- tests/fuzzFail/fuzz_app_file_fail38 | 1 + 5 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail38 diff --git a/include/CLI/impl/Option_inl.hpp b/include/CLI/impl/Option_inl.hpp index 2986b6bb0..aadf82d08 100644 --- a/include/CLI/impl/Option_inl.hpp +++ b/include/CLI/impl/Option_inl.hpp @@ -617,11 +617,11 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi // this check is to allow an empty vector in certain circumstances but not if expected is not zero. // {} is the indicator for an empty container if(out.empty()) { - if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0) { + if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max()>1) { out.push_back("{}"); out.push_back("%%"); } - } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0) { + } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max()>1) { out.push_back("%%"); } } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 0de3e3d18..ed2d5d89f 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -259,7 +259,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size()); - for(auto loc = str.begin(); loc != str.end(); ++loc) { + for (auto loc = str.begin(); loc < str.end(); ++loc) { if(*loc == '\\') { if(str.end() - loc < 2) { throw std::invalid_argument("invalid escape sequence " + str); @@ -270,7 +270,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { ++loc; } else if(*(loc + 1) == 'u') { // must have 4 hex characters - if(str.end() - loc < 5) { + if(str.end() - loc < 6) { throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); } std::uint32_t code{0}; @@ -287,7 +287,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { loc += 5; } else if(*(loc + 1) == 'U') { // must have 8 hex characters - if(str.end() - loc < 9) { + if(str.end() - loc < 10) { throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); } std::uint32_t code{0}; @@ -372,9 +372,6 @@ CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t star break; } } - if(loc == std::string::npos) { - return loc; - } ++loc; } return loc; @@ -540,12 +537,8 @@ CLI11_INLINE bool process_quoted_string(std::string &str, char string_char, char } return true; } - if(str.front() == literal_char && str.back() == literal_char) { - detail::remove_outer(str, literal_char); - return true; - } - if(str.front() == '`' && str.back() == '`') { - detail::remove_outer(str, literal_char); + if((str.front() == literal_char || str.front() == '`') && str.back() == str.front()) { + detail::remove_outer(str, str.front()); return true; } return false; diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index 0eb50e2cd..b4ccdfdd8 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 38)); + int index = GENERATE(range(1, 39)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index e7b0a9ce1..c452f09ef 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -251,6 +251,11 @@ TEST_CASE("StringTools: binaryEscapseConversion", "[helpers]") { std::string rstring = CLI::detail::extract_binary_string(estring); CHECK(rstring == testString2); + CLI::detail::remove_quotes(estring); + CHECK(CLI::detail::is_binary_escaped_string(estring)); + std::string rstringrq = CLI::detail::extract_binary_string(estring); + CHECK(rstringrq == testString2); + testString2.push_back(0); testString2.push_back(static_cast(197)); testString2.push_back(78); @@ -273,11 +278,13 @@ TEST_CASE("StringTools: binaryStrings", "[helpers]") { CHECK(CLI::detail::extract_binary_string(rstring).empty()); rstring = "B\"(\\x35\\xa7)\""; + CHECK(CLI::detail::is_binary_escaped_string(rstring)); auto result = CLI::detail::extract_binary_string(rstring); CHECK(result[0] == static_cast(0x35)); CHECK(result[1] == static_cast(0xa7)); - rstring = "B\"(\\x3e\\xf7)\""; + rstring = "'B\"(\\x3e\\xf7)\"'"; + CHECK(CLI::detail::is_binary_escaped_string(rstring)); result = CLI::detail::extract_binary_string(rstring); CHECK(result[0] == static_cast(0x3e)); CHECK(result[1] == static_cast(0xf7)); @@ -318,20 +325,113 @@ TEST_CASE("StringTools: escapeConversion", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\n\\r\\t\\f") == "test\n\r\t\f"); CHECK(CLI::detail::remove_escaped_characters("test\\r") == "test\r"); CHECK(CLI::detail::remove_escaped_characters("test\\f") == "test\f"); - CHECK(CLI::detail::remove_escaped_characters("test\\ttest\\n") == "test\ttest\n"); + std::string zstring="test"; + zstring.push_back('\0'); + zstring.append("test\n"); + CHECK(CLI::detail::remove_escaped_characters("test\\0test\\n") == zstring); CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\m_bad"), std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\"), std::invalid_argument); +} + +TEST_CASE("StringTools: quotedString", "[helpers]") { + + std::string rstring = "'B\"(\\x35\\xa7)\"'"; + auto s2=rstring; + CLI::detail::process_quoted_string(s2); + CHECK(s2[0] == static_cast(0x35)); + CHECK(s2[1] == static_cast(0xa7)); + s2=rstring; + CLI::detail::remove_quotes(s2); + CLI::detail::process_quoted_string(s2); + CHECK(s2[0] == static_cast(0x35)); + CHECK(s2[1] == static_cast(0xa7)); + + std::string qbase=R"("this\nis\na\nfour\tline test")"; + std::string qresult="this\nis\na\nfour\tline test"; + + std::string q1=qbase; + + //test remove quotes and escape processing + CLI::detail::process_quoted_string(q1); + CHECK(q1==qresult); + + + std::string q2=qbase; + q2.front()='\''; + q2.pop_back(); + q2.push_back('\''); + std::string qliteral=qbase.substr(1); + qliteral.pop_back(); + + //test remove quotes for literal string + CHECK(CLI::detail::process_quoted_string(q2)); + CHECK(q2==qliteral); + + std::string q3=qbase; + q3.front()='`'; + q3.pop_back(); + q3.push_back('`'); + + //test remove quotes for literal string + CHECK(CLI::detail::process_quoted_string(q3)); + CHECK(q3==qliteral); + + + std::string q4=qbase; + q4.front()='|'; + q4.pop_back(); + q4.push_back('|'); + + //check that it doesn't process + CHECK_FALSE(CLI::detail::process_quoted_string(q4)); + //test custom string quote character + CHECK(CLI::detail::process_quoted_string(q4,'|')); + CHECK(q4==qresult); + + std::string q5=qbase; + q5.front()='?'; + q5.pop_back(); + q5.push_back('?'); + + //test custom literal quote character + CHECK(CLI::detail::process_quoted_string(q5,'|','?')); + CHECK(q5==qliteral); + + q3=qbase; + q3.front()='`'; + q3.pop_back(); + q3.push_back('`'); + + //test that '`' still works regardless of the other specified characters + CHECK(CLI::detail::process_quoted_string(q3)); + CHECK(q3==qliteral); + } TEST_CASE("StringTools: unicode_literals", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\u03C0\\u00e9") == from_u8string(u8"test\u03C0\u00E9")); + CHECK(CLI::detail::remove_escaped_characters("test\\u73C0\\u0057") == from_u8string(u8"test\u73C0\u0057")); CHECK(CLI::detail::remove_escaped_characters("test\\U0001F600\\u00E9") == from_u8string(u8"test\U0001F600\u00E9")); CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001M600\\u00E9"), std::invalid_argument); CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\u00M9"), std::invalid_argument); CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\uD8E9"), std::invalid_argument); + + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E600\\uD8"), std::invalid_argument); + CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E60"), std::invalid_argument); +} + + +TEST_CASE("StringTools: close_sequence", "[helpers]") { + CHECK(CLI::detail::close_sequence("[test]",0,']')==5U); + CHECK(CLI::detail::close_sequence("[\"test]\"]",0,']')==8U); + CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]",0,']')==8U); + CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]",10,']')==13U); + CHECK(CLI::detail::close_sequence("{\"test]\"],[t2]", 0, '}') == 14U); + CHECK(CLI::detail::close_sequence("[(),(),{},\"]]52{}\",[],[54],[[],[],()]]", 0, ']') == 37U); } TEST_CASE("Trim: Various", "[helpers]") { diff --git a/tests/fuzzFail/fuzz_app_file_fail38 b/tests/fuzzFail/fuzz_app_file_fail38 new file mode 100644 index 000000000..117ee42ec --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail38 @@ -0,0 +1 @@ +ParseErrorEF'' --vo-d{} \ No newline at end of file From aa4c1aa3460ae724213bb156c7fc92357c3615b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:55:53 +0000 Subject: [PATCH 19/25] style: pre-commit.ci fixes --- include/CLI/impl/Option_inl.hpp | 5 +- include/CLI/impl/StringTools_inl.hpp | 2 +- tests/HelpersTest.cpp | 78 +++++++++++++--------------- tests/fuzzFail/fuzz_app_file_fail38 | 2 +- 4 files changed, 42 insertions(+), 45 deletions(-) diff --git a/include/CLI/impl/Option_inl.hpp b/include/CLI/impl/Option_inl.hpp index aadf82d08..8c9ece988 100644 --- a/include/CLI/impl/Option_inl.hpp +++ b/include/CLI/impl/Option_inl.hpp @@ -617,11 +617,12 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi // this check is to allow an empty vector in certain circumstances but not if expected is not zero. // {} is the indicator for an empty container if(out.empty()) { - if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max()>1) { + if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0 && + get_items_expected_max() > 1) { out.push_back("{}"); out.push_back("%%"); } - } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max()>1) { + } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max() > 1) { out.push_back("%%"); } } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index ed2d5d89f..3b7e9e6cd 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -259,7 +259,7 @@ CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { std::string out; out.reserve(str.size()); - for (auto loc = str.begin(); loc < str.end(); ++loc) { + for(auto loc = str.begin(); loc < str.end(); ++loc) { if(*loc == '\\') { if(str.end() - loc < 2) { throw std::invalid_argument("invalid escape sequence " + str); diff --git a/tests/HelpersTest.cpp b/tests/HelpersTest.cpp index c452f09ef..4397b1d4e 100644 --- a/tests/HelpersTest.cpp +++ b/tests/HelpersTest.cpp @@ -325,7 +325,7 @@ TEST_CASE("StringTools: escapeConversion", "[helpers]") { CHECK(CLI::detail::remove_escaped_characters("test\\n\\r\\t\\f") == "test\n\r\t\f"); CHECK(CLI::detail::remove_escaped_characters("test\\r") == "test\r"); CHECK(CLI::detail::remove_escaped_characters("test\\f") == "test\f"); - std::string zstring="test"; + std::string zstring = "test"; zstring.push_back('\0'); zstring.append("test\n"); CHECK(CLI::detail::remove_escaped_characters("test\\0test\\n") == zstring); @@ -337,76 +337,73 @@ TEST_CASE("StringTools: escapeConversion", "[helpers]") { TEST_CASE("StringTools: quotedString", "[helpers]") { std::string rstring = "'B\"(\\x35\\xa7)\"'"; - auto s2=rstring; + auto s2 = rstring; CLI::detail::process_quoted_string(s2); CHECK(s2[0] == static_cast(0x35)); CHECK(s2[1] == static_cast(0xa7)); - s2=rstring; + s2 = rstring; CLI::detail::remove_quotes(s2); CLI::detail::process_quoted_string(s2); CHECK(s2[0] == static_cast(0x35)); CHECK(s2[1] == static_cast(0xa7)); - std::string qbase=R"("this\nis\na\nfour\tline test")"; - std::string qresult="this\nis\na\nfour\tline test"; + std::string qbase = R"("this\nis\na\nfour\tline test")"; + std::string qresult = "this\nis\na\nfour\tline test"; - std::string q1=qbase; + std::string q1 = qbase; - //test remove quotes and escape processing + // test remove quotes and escape processing CLI::detail::process_quoted_string(q1); - CHECK(q1==qresult); + CHECK(q1 == qresult); - - std::string q2=qbase; - q2.front()='\''; + std::string q2 = qbase; + q2.front() = '\''; q2.pop_back(); q2.push_back('\''); - std::string qliteral=qbase.substr(1); + std::string qliteral = qbase.substr(1); qliteral.pop_back(); - //test remove quotes for literal string + // test remove quotes for literal string CHECK(CLI::detail::process_quoted_string(q2)); - CHECK(q2==qliteral); + CHECK(q2 == qliteral); - std::string q3=qbase; - q3.front()='`'; + std::string q3 = qbase; + q3.front() = '`'; q3.pop_back(); q3.push_back('`'); - - //test remove quotes for literal string - CHECK(CLI::detail::process_quoted_string(q3)); - CHECK(q3==qliteral); + // test remove quotes for literal string + CHECK(CLI::detail::process_quoted_string(q3)); + CHECK(q3 == qliteral); - std::string q4=qbase; - q4.front()='|'; + std::string q4 = qbase; + q4.front() = '|'; q4.pop_back(); q4.push_back('|'); - //check that it doesn't process + // check that it doesn't process CHECK_FALSE(CLI::detail::process_quoted_string(q4)); - //test custom string quote character - CHECK(CLI::detail::process_quoted_string(q4,'|')); - CHECK(q4==qresult); + // test custom string quote character + CHECK(CLI::detail::process_quoted_string(q4, '|')); + CHECK(q4 == qresult); - std::string q5=qbase; - q5.front()='?'; + std::string q5 = qbase; + q5.front() = '?'; q5.pop_back(); q5.push_back('?'); - //test custom literal quote character - CHECK(CLI::detail::process_quoted_string(q5,'|','?')); - CHECK(q5==qliteral); + // test custom literal quote character + CHECK(CLI::detail::process_quoted_string(q5, '|', '?')); + CHECK(q5 == qliteral); - q3=qbase; - q3.front()='`'; + q3 = qbase; + q3.front() = '`'; q3.pop_back(); q3.push_back('`'); - //test that '`' still works regardless of the other specified characters + // test that '`' still works regardless of the other specified characters CHECK(CLI::detail::process_quoted_string(q3)); - CHECK(q3==qliteral); - + CHECK(q3 == qliteral); } TEST_CASE("StringTools: unicode_literals", "[helpers]") { @@ -424,12 +421,11 @@ TEST_CASE("StringTools: unicode_literals", "[helpers]") { CHECK_THROWS_AS(CLI::detail::remove_escaped_characters("test\\U0001E60"), std::invalid_argument); } - TEST_CASE("StringTools: close_sequence", "[helpers]") { - CHECK(CLI::detail::close_sequence("[test]",0,']')==5U); - CHECK(CLI::detail::close_sequence("[\"test]\"]",0,']')==8U); - CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]",0,']')==8U); - CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]",10,']')==13U); + CHECK(CLI::detail::close_sequence("[test]", 0, ']') == 5U); + CHECK(CLI::detail::close_sequence("[\"test]\"]", 0, ']') == 8U); + CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]", 0, ']') == 8U); + CHECK(CLI::detail::close_sequence("[\"test]\"],[t2]", 10, ']') == 13U); CHECK(CLI::detail::close_sequence("{\"test]\"],[t2]", 0, '}') == 14U); CHECK(CLI::detail::close_sequence("[(),(),{},\"]]52{}\",[],[54],[[],[],()]]", 0, ']') == 37U); } diff --git a/tests/fuzzFail/fuzz_app_file_fail38 b/tests/fuzzFail/fuzz_app_file_fail38 index 117ee42ec..981220297 100644 --- a/tests/fuzzFail/fuzz_app_file_fail38 +++ b/tests/fuzzFail/fuzz_app_file_fail38 @@ -1 +1 @@ -ParseErrorEF'' --vo-d{} \ No newline at end of file +ParseErrorEF'' --vo-d{} From 30f9114fd1cab2065e5cf525cc1b640fd2021ffc Mon Sep 17 00:00:00 2001 From: Philip Top Date: Wed, 27 Dec 2023 08:58:57 -0800 Subject: [PATCH 20/25] fix fuzz issue with empty vector indicator --- include/CLI/impl/Option_inl.hpp | 15 +++++++++++---- tests/FuzzFailTest.cpp | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/include/CLI/impl/Option_inl.hpp b/include/CLI/impl/Option_inl.hpp index 8c9ece988..814c8ca64 100644 --- a/include/CLI/impl/Option_inl.hpp +++ b/include/CLI/impl/Option_inl.hpp @@ -609,7 +609,15 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi throw ArgumentMismatch::AtLeast(get_name(), static_cast(num_min), original.size()); } if(original.size() > num_max) { - throw ArgumentMismatch::AtMost(get_name(), static_cast(num_max), original.size()); + if (original.size() == 2 && num_max == 1 && original[1] == "%%" && original[0] == "{}") + { + //this condition is a trap for the following empty indicator check on config files + out=original; + } + else + { + throw ArgumentMismatch::AtMost(get_name(), static_cast(num_max), original.size()); + } } break; } @@ -617,12 +625,11 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi // this check is to allow an empty vector in certain circumstances but not if expected is not zero. // {} is the indicator for an empty container if(out.empty()) { - if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0 && - get_items_expected_max() > 1) { + if(original.size() == 1 && original[0] == "{}" && get_items_expected_min() > 0) { out.push_back("{}"); out.push_back("%%"); } - } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 && get_items_expected_max() > 1) { + } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 ) { out.push_back("%%"); } } diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index b4ccdfdd8..c1e205693 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(1, 39)); + int index = GENERATE(range(38, 39)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { From fc00398c1e3c5cd5f2077807cd9b43479e18e791 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 17:00:01 +0000 Subject: [PATCH 21/25] style: pre-commit.ci fixes --- include/CLI/impl/Option_inl.hpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/include/CLI/impl/Option_inl.hpp b/include/CLI/impl/Option_inl.hpp index 814c8ca64..987f39987 100644 --- a/include/CLI/impl/Option_inl.hpp +++ b/include/CLI/impl/Option_inl.hpp @@ -609,13 +609,10 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi throw ArgumentMismatch::AtLeast(get_name(), static_cast(num_min), original.size()); } if(original.size() > num_max) { - if (original.size() == 2 && num_max == 1 && original[1] == "%%" && original[0] == "{}") - { - //this condition is a trap for the following empty indicator check on config files - out=original; - } - else - { + if(original.size() == 2 && num_max == 1 && original[1] == "%%" && original[0] == "{}") { + // this condition is a trap for the following empty indicator check on config files + out = original; + } else { throw ArgumentMismatch::AtMost(get_name(), static_cast(num_max), original.size()); } } @@ -629,7 +626,7 @@ CLI11_INLINE void Option::_reduce_results(results_t &out, const results_t &origi out.push_back("{}"); out.push_back("%%"); } - } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0 ) { + } else if(out.size() == 1 && out[0] == "{}" && get_items_expected_min() > 0) { out.push_back("%%"); } } From 4be293758cad1474240e359af417401d4a7bf65d Mon Sep 17 00:00:00 2001 From: Philip Top Date: Wed, 27 Dec 2023 09:27:32 -0800 Subject: [PATCH 22/25] fix possible recursion in string locations --- include/CLI/impl/StringTools_inl.hpp | 7 ++++++- tests/FuzzFailTest.cpp | 2 +- tests/fuzzFail/fuzz_app_file_fail39 | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 tests/fuzzFail/fuzz_app_file_fail39 diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 3b7e9e6cd..8011cdbab 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -330,7 +330,8 @@ CLI11_INLINE std::size_t close_string_quote(const std::string &str, std::size_t } CLI11_INLINE std::size_t close_literal_quote(const std::string &str, std::size_t start, char closure_char) { - return str.find_first_of(closure_char, start + 1); + auto loc=str.find_first_of(closure_char, start + 1); + return (loc!=std::string::npos?loc:str.size()); } CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char) { @@ -374,6 +375,10 @@ CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t star } ++loc; } + if (loc > str.size()) + { + loc = str.size(); + } return loc; } diff --git a/tests/FuzzFailTest.cpp b/tests/FuzzFailTest.cpp index c1e205693..39e37fc5f 100644 --- a/tests/FuzzFailTest.cpp +++ b/tests/FuzzFailTest.cpp @@ -63,7 +63,7 @@ TEST_CASE("app_file_gen_fail") { CLI::FuzzApp fuzzdata; auto app = fuzzdata.generateApp(); - int index = GENERATE(range(38, 39)); + int index = GENERATE(range(1, 40)); std::string optionString, flagString; auto parseData = loadFailureFile("fuzz_app_file_fail", index); if(parseData.size() > 25) { diff --git a/tests/fuzzFail/fuzz_app_file_fail39 b/tests/fuzzFail/fuzz_app_file_fail39 new file mode 100644 index 000000000..98cc3fa7a --- /dev/null +++ b/tests/fuzzFail/fuzz_app_file_fail39 @@ -0,0 +1 @@ +[--' \ No newline at end of file From c31bb269e5668c5a0ef44359fd2989eefd78a610 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 17:28:31 +0000 Subject: [PATCH 23/25] style: pre-commit.ci fixes --- include/CLI/impl/StringTools_inl.hpp | 7 +++---- tests/fuzzFail/fuzz_app_file_fail39 | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index 8011cdbab..ecc3ef439 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -330,8 +330,8 @@ CLI11_INLINE std::size_t close_string_quote(const std::string &str, std::size_t } CLI11_INLINE std::size_t close_literal_quote(const std::string &str, std::size_t start, char closure_char) { - auto loc=str.find_first_of(closure_char, start + 1); - return (loc!=std::string::npos?loc:str.size()); + auto loc = str.find_first_of(closure_char, start + 1); + return (loc != std::string::npos ? loc : str.size()); } CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char) { @@ -375,8 +375,7 @@ CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t star } ++loc; } - if (loc > str.size()) - { + if(loc > str.size()) { loc = str.size(); } return loc; diff --git a/tests/fuzzFail/fuzz_app_file_fail39 b/tests/fuzzFail/fuzz_app_file_fail39 index 98cc3fa7a..991c5c3bd 100644 --- a/tests/fuzzFail/fuzz_app_file_fail39 +++ b/tests/fuzzFail/fuzz_app_file_fail39 @@ -1 +1 @@ -[--' \ No newline at end of file +[--' From 754eb9ab473ec6d50e7e13c9f0b0db1971cbead9 Mon Sep 17 00:00:00 2001 From: Philip Top Date: Wed, 27 Dec 2023 10:18:11 -0800 Subject: [PATCH 24/25] add more coverage tests --- include/CLI/impl/Config_inl.hpp | 3 --- include/CLI/impl/StringTools_inl.hpp | 2 +- tests/ConfigFileTest.cpp | 9 +++++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/CLI/impl/Config_inl.hpp b/include/CLI/impl/Config_inl.hpp index 7b6188ecd..4723c5015 100644 --- a/include/CLI/impl/Config_inl.hpp +++ b/include/CLI/impl/Config_inl.hpp @@ -54,9 +54,6 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char literalQuote, if(isprint(static_cast(arg.front())) == 0) { return binary_escape_string(arg); } - if(arg == "\\") { - return std::string(1, literalQuote) + "\\" + literalQuote; - } if(arg == "'") { return std::string(1, stringQuote) + "'" + stringQuote; } diff --git a/include/CLI/impl/StringTools_inl.hpp b/include/CLI/impl/StringTools_inl.hpp index ecc3ef439..4f186cc95 100644 --- a/include/CLI/impl/StringTools_inl.hpp +++ b/include/CLI/impl/StringTools_inl.hpp @@ -393,7 +393,7 @@ CLI11_INLINE std::vector split_up(std::string str, char delimiter) if(bracketChars.find_first_of(str[0]) != std::string::npos) { auto bracketLoc = bracketChars.find_first_of(str[0]); auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); - if(end >= std::string::npos - 1) { + if(end >= str.size()) { output.push_back(std::move(str)); str.clear(); } else { diff --git a/tests/ConfigFileTest.cpp b/tests/ConfigFileTest.cpp index 20b8d7559..902aa0d60 100644 --- a/tests/ConfigFileTest.cpp +++ b/tests/ConfigFileTest.cpp @@ -27,6 +27,15 @@ TEST_CASE("StringBased: convert_arg_for_ini", "[config]") { CHECK("-22E14" == CLI::detail::convert_arg_for_ini("-22E14")); CHECK("'a'" == CLI::detail::convert_arg_for_ini("a")); + + CHECK("'\\'" == CLI::detail::convert_arg_for_ini("\\")); + + CHECK("\"'\"" == CLI::detail::convert_arg_for_ini("'")); + + std::string tstring1; + tstring1.push_back('\0'); + //binary string conversion single character + CHECK("'B\"(\\x00)\"'" == CLI::detail::convert_arg_for_ini(tstring1)); // hex CHECK("0x5461FAED" == CLI::detail::convert_arg_for_ini("0x5461FAED")); // hex fail From 82d7d740812cf5e956f69455fed43368894f48b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 18:18:47 +0000 Subject: [PATCH 25/25] style: pre-commit.ci fixes --- tests/ConfigFileTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ConfigFileTest.cpp b/tests/ConfigFileTest.cpp index 902aa0d60..55ceac2cd 100644 --- a/tests/ConfigFileTest.cpp +++ b/tests/ConfigFileTest.cpp @@ -34,7 +34,7 @@ TEST_CASE("StringBased: convert_arg_for_ini", "[config]") { std::string tstring1; tstring1.push_back('\0'); - //binary string conversion single character + // binary string conversion single character CHECK("'B\"(\\x00)\"'" == CLI::detail::convert_arg_for_ini(tstring1)); // hex CHECK("0x5461FAED" == CLI::detail::convert_arg_for_ini("0x5461FAED"));