memory corruption in async_read_until using asio::ssl

Afternoon All I am seeing an occasional memory corruption bug, using valgrind, in our web server application that is using boost::asio::ssl, and it seems to be related to calls to async_read_until, where several chunks of data are required before the terminating regex is found. What appears to be happening is that engine::perform() causes a write (via ssl3_read_bytes()) on a deleted memory region, the buffer into which decrypted data is written. This region was delete'd via invocation of the detail::io_op::handler, and in turn via read_until_delim_op::operator(), due to prepare() being called on the streambuf, and a std::vector<char> is delete'd in order to create a larger one. It seems like one of the chain of handlers is using a pointer which is becomes invalid when the streambuff is expanded after failing to find the regex terminator after one or more reads, and prepare() is called before the next async_read_some() on the underlying stream. I have included the valgrind output at the end - sorry for the large size. I am pretty certain that I have ruled out the obvious (multiple simultaneous attempts to read using the same streambuf), and I am only using one thread for the io_service. This seems a pretty common potential failure mode, so perhaps I am barking up the wrong tree - I'd appreciate anyone's opinion before I start a potentially tricky debug effort requiring modification to the boost libraries on our production servers. I'm using boost 1.49. There's a lot of optimization on the application which is why some calls are optimized out of the valgrind dump. The machines are running Centos 5.8, x86_64. Compiler is gcc 4.4.6. Thanks for looking Jim Page ==29870== Thread 22: ==29870== Invalid write of size 1 ==29870== at 0x4A088F8: memcpy (mc_replace_strmem.c:587) ==29870== by 0x3400821EDE: ssl3_read_bytes (in /lib64/libssl.so.0.9.8e) ==29870== by 0x340081E750: ??? (in /lib64/libssl.so.0.9.8e) ==29870== by 0x66A75B: boost::asio::ssl::detail::engine::perform(int (boost::asio::ssl::detail::engine::*)(void*, unsigned long), void*, unsigned long, boost::system::error_code&, unsigned long*) (engine.ipp:219) ==29870== by 0x7E3326: boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio ::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >, boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>, boost::asio::detail::rea d_until_expr_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<bo ost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp>
, std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char, boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&, char const*>, boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >, boost::arg<1> (*)(), boost::_bi::value<char const*> > > > ::operator()(boost::system::error_code, unsigned long, int) (engine.ipp:151) ==29870== by 0x7E45C2: boost::asio::detail::wait_handler<boost::asio::ssl::detail::io_op<boost::asi o::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >, boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>, boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio ::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> > >, std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char, boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&, char const*>, boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >, boost::arg<1> (*)(), boost::_bi::value<char const*> > > > > ::do_complete(boost::asio::detail::task_io_service*, boost::asio::detail::task_io_service_operation*, boost::system::error_code const&, unsigned long) (bind_handler.hpp:46) ==29870== by 0x619783: boost::asio::detail::task_io_service::run(boost::system::error_code&) (task_io_service_operation.hpp:37) ==29870== by 0x79F4FE: BayeuxServer::run() (io_service.ipp:59) ==29870== by 0x364800DB8E: ??? (in /usr/lib64/libboost_thread.so.1.49.0) ==29870== by 0x3AE300677C: start_thread (in /lib64/libpthread-2.5.so) ==29870== by 0x3AE20D325C: clone (in /lib64/libc-2.5.so) ==29870== Address 0x1b508610 is 0 bytes inside a block of size 1,024 free'd ==29870== at 0x4A05A33: operator delete(void*) (vg_replace_malloc.c:346) ==29870== by 0x658E54: std::vector<char, std::allocator<char> ::_M_fill_insert(__gnu_cxx::__normal_iterator<char*, std::vector<char, std::allocator<char> > >, unsigned long, char const&) (new_allocator.h:95) ==29870== by 0x68B805: boost::asio::basic_streambuf<std::allocator<char> ::reserve(unsigned long) (stl_vector.h:851) ==29870== by 0x7E2EF2: boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio ::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> > >, std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char, boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&, char const*>, boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >, boost::arg<1> (*)(), boost::_bi::value<char const*> > > ::operator()(boost::system::error_code const&, unsigned long, int) (basic_streambuf.hpp:207) ==29870== by 0x7E32DE: boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio ::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >, boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>, boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio ::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> > >, std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char, boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&, char const*>, boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >, boost::arg<1> (*)(), boost::_bi::value<char const*> > > > ::operator()(boost::system::error_code, unsigned long, int) (read_op.hpp:59) ==29870== by 0x7E493C: boost::asio::detail::reactive_socket_recv_op<boost::asio::mutable_buffers_1, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio ::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >, boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>, boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio ::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> > >, std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char, boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&, char const*>, boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >, boost::arg<1> (*)(), boost::_bi::value<char const*> > > > > ::do_complete(boost::asio::detail::task_io_service*, boost::asio::detail::task_io_service_operation*, boost::system::error_code const&, unsigned long) (bind_handler.hpp:118) ==29870== by 0x613778: boost::asio::detail::epoll_reactor::descriptor_state::do_complete(boost::asi o::detail::task_io_service*, boost::asio::detail::task_io_service_operation*, boost::system::error_code const&, unsigned long) (task_io_service_operation.hpp:37) ==29870== by 0x619783: boost::asio::detail::task_io_service::run(boost::system::error_code&) (task_io_service_operation.hpp:37) ==29870== by 0x79F4FE: BayeuxServer::run() (io_service.ipp:59) ==29870== by 0x364800DB8E: ??? (in /usr/lib64/libboost_thread.so.1.49.0) ==29870== by 0x3AE300677C: start_thread (in /lib64/libpthread-2.5.so) ==29870== by 0x3AE20D325C: clone (in /lib64/libc-2.5.so)
participants (1)
-
Jim Page