Hi,
I have build a job queue with a combination io_service, io_service::work,
packaged_task and a thread group. I am seeing randomly either segfaults or
strange asserts in pthread or even malloc, which suggest to me memory is
being corrupted somehow.
The problem is it does not show in debug mode running under valgrind, but
if I run valgrind on the non debug version I get valgrind errors in
pthreads called from asio. I try to downgrade my boost lib from 1.54 to
1.52, but still the same.
I am completely stuck on this, does anyone have some hints to possible
causes ?
I can not show the complete source since it's not possible to isolate, but
here's the types I have :
std::vector
io_error_;
std::vector
io_future_;
boost::asio::io_service
io_service_threads_;
boost::shared_ptrboost::asio::io_service::work
io_work_;
boost::thread_group
io_threads_;
I am calling it ala like this :
/* Create new tasks */
basistasks_[workerid] =
boost::make_shared
(boost::bind(&CallBasisSelection,&worker,boost::ref(io_error_[workerid])));
/* Make threads */
io_service_threads_.post(boost::bind(&boost::packaged_task<BasisSelectRet>::operator(),basistasks_[workerid]));
io_future_[workerid] = basistasks_[workerid]->get_future().share();
The called function is defined as follows (BasisSelectRet is just a enum) :
/* Helper function */
BasisSelectRet CallBasisSelection(TreeWorker
*worker,
boost::shared_ptr<ExceptionTransfer>
&error)
{
BasisSelectRet ret = BasisSelectRetOk;
try
{
worker->BasisSelection();
}
catch( ... )
{
error->SetException(boost::current_exception());
}
return ret;
}
I can wait for a job to finish like this :
boost::wait_for_any(io_future_.begin(),io_future_.end());
Here's the val grind errors :
==3723== by 0xB5D77BF: ???
==3723==
==3723== Thread 5:
==3723== Invalid read of size 4
==3723== at 0x6506C01: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:50)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEAA2A: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c334 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6506C36: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:61)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEAA2A: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c328 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6506C40: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:62)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEAA2A: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c330 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid write of size 4
==3723== at 0x6506C4C: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:125)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEAA2A: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c330 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 1
==3723== at 0x5F27B2B:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0xDD41297: ???
==3723== Address 0x670c360 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6500311: __pthread_mutex_unlock_usercnt
(pthread_mutex_unlock.c:37)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c334 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid write of size 4
==3723== at 0x650032F: __pthread_mutex_unlock_usercnt
(pthread_mutex_unlock.c:46)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c330 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6500360: __pthread_mutex_unlock_usercnt
(pthread_mutex_unlock.c:49)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c338 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6500340: __pthread_mutex_unlock_usercnt
(pthread_mutex_unlock.c:52)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c328 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid write of size 4
==3723== at 0x6503C55: __lll_unlock_wake (lowlevellock.S:374)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c328 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Syscall param futex(futex) points to unaddressable byte(s)
==3723== at 0x6503C7C: __lll_unlock_wake (lowlevellock.S:380)
==3723== by 0x5EEAA58: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0x670c328 is not stack'd, malloc'd or (recently) free'd
==3723==
<Log> 20 0 20 -3.05198e+02 - - - 1127
31.18
==3723== Thread 3:
==3723== Invalid read of size 8
==3723== at 0x4E0ED3F: __intel_ssse3_memcpy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x7439DF7: ???
==3723== Address 0x66f3b58 is 112 bytes inside a block of size 116 alloc'd
==3723== at 0x4024F20: malloc (vg_replace_malloc.c:236)
==3723== by 0x582A7B6: _ZN3slm6Matrix16ResizeVecsInternEiPKib. (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x7439DF7: ???
==3723==
==3723== Invalid read of size 8
==3723== at 0x4E0ED6F: __intel_ssse3_memcpy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x7439DF7: ???
==3723== Address 0x66f5d48 is 136 bytes inside a block of size 140 alloc'd
==3723== at 0x4024F20: malloc (vg_replace_malloc.c:236)
==3723== by 0x582A7B6: _ZN3slm6Matrix16ResizeVecsInternEiPKib. (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x7439DF7: ???
==3723==
<Log> 20 0 20 -3.05198e+02 -2.68521e+02* 12.0 1 1139
36.72
==3723== Thread 4:
==3723== Invalid read of size 4
==3723== at 0x6506C01: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:50)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEA9C5: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0xb93ffac is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6506C36: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:61)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEA9C5: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0xb93ffa0 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid read of size 4
==3723== at 0x6506C40: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:62)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEA9C5: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0xb93ffa8 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Invalid write of size 4
==3723== at 0x6506C4C: __pthread_mutex_cond_lock
(pthread_mutex_lock.c:125)
==3723== by 0x65010B2: pthread_cond_wait@@GLIBC_2.3.2
(pthread_cond_wait.S:203)
==3723== by 0x5F27B73:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x5EEA9C5: boost::asio::io_service::run() (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0x4C160F9:
boost::detail::thread_data,
boost::_bi::list1boost::asio::io_service* > > >::run()
(in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x4E25DAB: thread_proxy (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulumnet20.so)
==3723== by 0x64FC96D: start_thread (pthread_create.c:300)
==3723== by 0x65DD98D: clone (clone.S:130)
==3723== Address 0xb93ffa8 is not stack'd, malloc'd or (recently) free'd
==3723==
==3723== Thread 5:
==3723== Invalid read of size 1
==3723== at 0x5F27B2B:
boost::asio::detail::task_io_service::do_run_one(boost::asio::detail::scoped_lockboost::asio::detail::posix_mutex&,
boost::asio::detail::task_io_service::thread_info&,
boost::system::error_code const&) (in
/home/bj/sulum/root/src/bjam/bin/intel-linux/release/address-model-32/threading-multi/libsulum20.so)
==3723== by 0xCD40297: ???
==3723== Address 0xb93ffd8 is not stack'd, malloc'd or (recently) free'd
==3723==
