Boost.ProgramOptions issues porting from 1.56 to 1.64
I have an SVN-like "multi-command" single executable, i.e. exe [global-options] command [command-options] It uses two levels of option parsing. First level: typedef std::vector<std::string> UnparsedArgs; inline boost::program_options::typed_value<UnparsedArgs>* anyArgs() { return boost::program_options::value<UnparsedArgs>(); } options_description global_all_desc("Global Options", columns); global_all_desc.add_options() ("command", txtArg(), "Command to execute") ("cmdargs", anyArgs(), "Arguments for command") ...; // from http://stackoverflow.com/questions/15541498 // see also https://gist.github.com/randomphrase/10801888 po::positional_options_description pos; pos .add("command", 1) .add("cmdargs", -1); variables_map global_args; parsed_options parsed = po::command_line_parser(argc, argv) .options(global_all_desc) .positional(pos) // for implicit command and cmdargs above .allow_unregistered() // for command-specific arguments. .run(); po::store(parsed, global_args); std::vector<std::string> cmd_args = po::collect_unrecognized( parsed.options, po::include_positional ); Second level parsed_options reparsed = po::command_line_parser(cmd_args) .options(cmd_all_desc) .run(); po::store(reparsed, args_); // JIRA FOO-426 for (const auto& option : reparsed.options) { if (option.position_key >= 0) { std::ostringstream oss; for (const auto& token : option.original_tokens) { oss << token << ' '; } auto s = oss.str(); s.resize(s.size() - 1); err() << "Error: Too many commands: " << command() << ' ' << s << endl; return 1; } } in this particular case, the option that fails is ( "new", txtArgOpt(), "Creates a new group" ), with txtArgOpt() as below: inline boost::program_options::typed_value<std::string>* txtArg() { return boost::program_options::value<std::string>(); } inline boost::program_options::typed_value<std::string>* txtArgOpt(const std::string& implicit_value = "") { return txtArg()->implicit_value(implicit_value); } in the case that fails, the first parsing works fine, and remains ["--new", "groupA"] as cmd_args. in 1.56, reparsed.options contains 1 element: - [0] {string_key="new" position_key=-1 value={ size=1 } ...} boost::program_options::basic_option<char> + string_key "new" std::basic_string<char,std::char_traits<char>,std::allocator<char> > position_key -1 int - value { size=1 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [size] 1 __int64 [capacity] 1 __int64 + [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [Raw View] 0x0000000009fae6c0 {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > * - original_tokens { size=2 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [size] 2 __int64 [capacity] 2 __int64 + [0] "--new" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [1] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [Raw View] 0x0000000009fae6e0 {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > * unregistered false bool case_insensitive false bool but in 1.64, it contains 2 elements: - [0] {string_key="new" position_key=-1 value={ size=0 } ...} boost::program_options::basic_option<char> + string_key "new" std::basic_string<char,std::char_traits<char>,std::allocator<char> > position_key -1 int - value { size=0 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [capacity] 0 __int64 + [allocator] allocator std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,1> + [Raw View] {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > - original_tokens { size=1 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [capacity] 1 __int64 + [allocator] allocator std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,1> + [0] "--new" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [Raw View] {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > unregistered false bool case_insensitive false bool - [1] {string_key="" position_key=0 value={ size=1 } ...} boost::program_options::basic_option<char> + string_key "" std::basic_string<char,std::char_traits<char>,std::allocator<char> > position_key 0 int - value { size=1 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [capacity] 1 __int64 + [allocator] allocator std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,1> + [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [Raw View] {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > - original_tokens { size=1 } std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > [capacity] 1 __int64 + [allocator] allocator std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > >,1> + [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char> > + [Raw View] {...} std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char> >,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char> > > > unregistered false bool case_insensitive false bool This is copy/pasted from VS, sorry, a little hard to read, but basically 1.56 has #0 string_key = "new" position_key = -1 value = ["groupA"] original_tokens = ["--new", "groupA"] unregistered = false case_insensitive = false while 1.64 has 2 items: #0 string_key = "new" position_key = -1 value = [] original_tokens = ["--new"] unregistered = false case_insensitive = false #1 string_key = "" position_key = 0 value = ["groupA"] original_tokens = ["groupA"] unregistered = false case_insensitive = false as-if the arg to --new was now a positional parameter, instead of an optional arg to --new. And this triggers the work-around code that errors out I had put in place to catch cases when the old 1.56 code wouldn't complain about extra arguments (e.g. "exe ... cmd1 cmd1_args cmd2" would be silently accepted, despite the extra cmd2, or "exe ... cmd --arg1 read-only" was accepted, when "exe ... cmd --arg1 --read-only" was expected), and this even though the second parsing doesn't have .allow_unregistered(). I'm trying to understand what's going on; obviously there's some kind of semantic change with optional argument to switches. Could anyone shed some light on this please? I'm not sure how to proceed now. TIA, -DD
I'm trying to understand what's going on; obviously there's some kind of semantic change with optional argument to switches.
Could anyone shed some light on this please? I'm not sure how to proceed now. TIA, -DD
OK, maybe with a small stand-alone program, I'll have more chances of getting help. Basically the main difference is that implicit_value() no longer works in 1.64 as it used to in 1.56. the new arg doesn't get the explicit value on the next token anymore, but the implicit value, and this despite the changes from June 10 [1] and Jan 13 [2]. I've even tried having a non-empty implicit value, but that makes no difference. My app uses implicit values extensively, and switching to default values instead is not an option (no pun intended), since then the options would be defined which affects the behavior of the app extensively. Is this a regression? Given [2] I'd say yes, but given that test_implicit_value() [3] does test something similar to what I'm doing, I'm a bit baffled by my little example not working properly. What am I missing? Thanks, --DD [1] https://github.com/boostorg/program_options/commit/ed72cc2f92d0e48d0573614bb... [2] https://github.com/boostorg/program_options/commit/7729850bb7eb25b42552b25ef... [3] https://github.com/boostorg/program_options/commit/ed72cc2f92d0e48d0573614bb... [ddevienne@arachne po]$ boost-1.56-gcc-4.8 --new groupA has --new option: yes --new option val: groupA parsed.options = { { string_key = "new", position_key = -1, value = ["groupA",], original_tokens = ["--new","groupA",] }, } [ddevienne@arachne po]$ boost-1.64-gcc-6.2 --new groupA has --new option: yes --new option val: parsed.options = { { string_key = "new", position_key = -1, value = [], original_tokens = ["--new",] }, { string_key = "", position_key = 0, value = ["groupA",], original_tokens = ["groupA",] }, } Here's the full program and how I built it on both cases and running it in different circumstances. [ddevienne@arachne po]$ cat implicit_value_regression.cpp #include <iostream> #include <string> #include <vector> #include <boost/program_options.hpp> using namespace std; using namespace boost::program_options; std::ostream& operator<<(std::ostream& os, const std::vector<std::string>& strings) { std::cout << '['; for (const auto& str : strings) { std::cout << '"' << str << '"' << ','; } return os << ']'; } int main(int argc, char* argv[]) { options_description opts_desc("Options"); opts_desc.add_options() ("new", value<std::string>()->implicit_value(""), "New group") ; parsed_options parsed = command_line_parser(argc, argv) .options(opts_desc) .run(); variables_map args; store(parsed, args); auto new_arg = args["new"]; if (new_arg.empty()) { cout << "has --new option: no" << endl; } else { cout << "has --new option: yes" << endl; cout << "--new option val: " << new_arg.as<std::string>() << endl; } cout << "parsed.options = {"; for (const auto& opt : parsed.options) { cout << "\n {" << endl; cout << " string_key = \"" << opt.string_key << "\"," << endl; cout << " position_key = " << opt.position_key << "," << endl; cout << " value = " << opt.value << "," << endl; cout << " original_tokens = " << opt.original_tokens << endl; cout << " }," << endl; } cout << "}" << endl; return 0; } [ddevienne@arachne po]$ ls $BOOSTDIR/lib/*pro* /users/gdadmin/trunk/project/toolworks/boost/1.56.0z/Linux_x64_2.12_gcc48//lib/libboost_program_options.so /users/gdadmin/trunk/project/toolworks/boost/1.56.0z/Linux_x64_2.12_gcc48//lib/libboost_program_options.so.1.56.0 [ddevienne@arachne po]$ g++ -std=c++11 -I$BOOSTDIR/include -L$BOOSTDIR/lib -lboost_program_options implicit_value_regression.cpp -o boost-1.56-gcc-4.8 [ddevienne@arachne po]$ boost-1.56-gcc-4.8 --new groupA has --new option: yes --new option val: groupA parsed.options = { { string_key = "new", position_key = -1, value = ["groupA",], original_tokens = ["--new","groupA",] }, } [ddevienne@arachne po]$ boost-1.56-gcc-4.8 --new=groupA has --new option: yes --new option val: groupA parsed.options = { { string_key = "new", position_key = -1, value = ["groupA",], original_tokens = ["--new=groupA",] }, } [ddevienne@arachne po]$ boost-1.56-gcc-4.8 --new has --new option: yes --new option val: parsed.options = { { string_key = "new", position_key = -1, value = [], original_tokens = ["--new",] }, } [ddevienne@arachne po]$ g++ --version g++ (GCC) 4.8.2 20140120 (Red Hat 4.8.2-15) Copyright (C) 2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. [ddevienne@arachne po]$ [ddevienne@arachne po]$ ls $BOOSTDIR/lib/*pro* /users/gdadmin/trunk/project/toolworks/boost/1.64.0/Linux_x64_2.12_gcc62/lib/libboost_program_options.so /users/gdadmin/trunk/project/toolworks/boost/1.64.0/Linux_x64_2.12_gcc62/lib/libboost_program_options.so.1.64.0 [ddevienne@arachne po]$ g++ -std=c++11 -I$BOOSTDIR/include -L$BOOSTDIR/lib -lboost_program_options implicit_value_regression.cpp -o boost-1.64-gcc-6.2 [ddevienne@arachne po]$ boost-1.64-gcc-6.2 --new groupA has --new option: yes --new option val: parsed.options = { { string_key = "new", position_key = -1, value = [], original_tokens = ["--new",] }, { string_key = "", position_key = 0, value = ["groupA",], original_tokens = ["groupA",] }, } [ddevienne@arachne po]$ boost-1.64-gcc-6.2 --new=groupA has --new option: yes --new option val: groupA parsed.options = { { string_key = "new", position_key = -1, value = ["groupA",], original_tokens = ["--new=groupA",] }, } [ddevienne@arachne po]$ boost-1.64-gcc-6.2 --new has --new option: yes --new option val: parsed.options = { { string_key = "new", position_key = -1, value = [], original_tokens = ["--new",] }, } [ddevienne@arachne po]$ g++ --version g++ (GCC) 6.2.1 20160916 (Red Hat 6.2.1-3) Copyright (C) 2016 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
On Tue, Jul 25, 2017 at 6:24 PM, Dominique Devienne <ddevienne@gmail.com> wrote:
I'm trying to understand what's going on;
obviously there's some kind of semantic change with optional argument to switches.
Could anyone shed some light on this please? I'm not sure how to proceed now. TIA, -DD
OK, maybe with a small stand-alone program, I'll have more chances of getting help. ... Is this a regression? Given [2] I'd say yes, but given that test_implicit_value() [3] does test something similar to what I'm doing, I'm a bit baffled by my little example not working properly. What am I missing? Thanks, --DD
[1] https://github.com/boostorg/program_options/commit/ ed72cc2f92d0e48d0573614bb8d180325bff9866 [2] https://github.com/boostorg/program_options/commit/ 7729850bb7eb25b42552b25eff41f141dfa3729f [3] https://github.com/boostorg/program_options/commit/ ed72cc2f92d0e48d0573614bb8d180325bff9866
Rah, the test code in 1.64 reads like test_case test_cases1[] = { {"--foo bar", s_success, "foo: positional:bar"}, {"--foo=bar foobar", s_success, "foo:bar positional:foobar"}, {0, 0, 0} }; while the one on the develop branch behaves as 1.56 and the way I need! (see below) So it means Vladimir hasn't merged these commits into the release, despite predating it :( Bummer bummer bummer... Specific reasons this wasn't done? Will they be in 1.65? --DD test_case test_cases1[] = { // 'bar' does not even look like option, so is consumed {"--foo bar", s_success, "foo:bar"}, // '--bar' looks like option, and such option exists, so we don't consume this token {"--foo --bar", s_success, "foo: bar:"}, // '--biz' looks like option, but does not match any existing one. // Presently this results in parse error, since // (1) in cmdline.cpp:finish_option, we only consume following tokens if they are // requires // (2) in cmdline.cpp:run, we let options consume following positional options // For --biz, an exception is thrown between 1 and 2. // We might want to fix that in future. {"--foo --biz", s_unknown_option, ""}, {0, 0, 0} };
On 25/07/2017 02:33, Dominique Devienne wrote:
I'm trying to understand what's going on; obviously there's some kind of semantic change with optional argument to switches.
Could anyone shed some light on this please? I'm not sure how to proceed now. TIA, -DD
Sounds like https://github.com/boostorg/program_options/issues/25. This is apparently fixed in 1.65.
On Thu, Jul 27, 2017 at 2:11 AM, Gavin Lambert via Boost-users < boost-users@lists.boost.org> wrote:
On 25/07/2017 02:33, Dominique Devienne wrote:
I'm trying to understand what's going on; obviously there's some kind of semantic change with optional argument to switches.
Could anyone shed some light on this please? I'm not sure how to proceed now. TIA, -DD
Sounds like https://github.com/boostorg/program_options/issues/25.
Thanks a bunch Gavin for pointing it out. The work-around allows me to move past this despite using 1.64. This is apparently fixed in 1.65.
Yes, now that you've pointed it out, I see this have been merged on master, so will be in 1.65 indeed (I guess, I don't quite know the Boost release process). --DD
participants (2)
-
Dominique Devienne
-
Gavin Lambert