ThreadSanitizer: double lock of a mutex with std::condition_variable::wait_for

Question

GCC thread sanitizer reports "double lock of a mutex" warning with code below:

#include <mutex>
#include <condition_variable>
#include <chrono>
#include <stop_token>
#include <thread>

template<typename Rep, typename Period>
void sleep_for(const std::chrono::duration<Rep, Period>& d, const std::stop_token& token)
{
    std::mutex mutex;

    std::unique_lock<std::mutex> lock{ mutex };

    std::condition_variable_any().wait_for(lock, token, d, [&token]
    {
        return false;
    });
}

int main()
{
    std::jthread watch_dog_thread([](std::stop_token token)
    {
        sleep_for(std::chrono::seconds(std::chrono::seconds(3)), token);
    });

    std::this_thread::sleep_for(std::chrono::seconds(1));

    return 0;
}

GCC sanitizer output:

==================
    WARNING: ThreadSanitizer: double lock of a mutex (pid=6767)
    #0 pthread_mutex_lock ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:4250 (libtsan.so.0+0x53908)
    #1 __gthread_mutex_lock /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 (MyAppTest+0x83c54)
    #2 std::mutex::lock() /usr/include/c++/11/bits/std_mutex.h:100 (MyAppTest+0x83fd2)
    #3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/include/c++/11/bits/std_mutex.h:229 (MyAppTest+0x86974)
    #4 std::_V2::condition_variable_any::notify_all() /usr/include/c++/11/condition_variable:299 (MyAppTest+0x857f6)
    #5 operator() /usr/include/c++/11/condition_variable:404 (MyAppTest+0x2d9556)
    #6 _S_execute /usr/include/c++/11/stop_token:638 (MyAppTest+0x2d9d19)
    #7 std::stop_token::_Stop_cb::_M_run() /usr/include/c++/11/stop_token:148 (MyAppTest+0x849fd)
    #8 std::stop_token::_Stop_state_t::_M_request_stop() /usr/include/c++/11/stop_token:256 (MyAppTest+0x84d2a)
    #9 std::stop_source::request_stop() const /usr/include/c++/11/stop_token:536 (MyAppTest+0x8550c)
    #10 std::jthread::request_stop() /usr/include/c++/11/thread:201 (MyAppTest+0x856a6)
    #11 std::jthread::~jthread() /usr/include/c++/11/thread:129 (MyAppTest+0x8555b)
    #12 main /home/def/repos/MyApp/Tests/main.cpp:38 (MyAppTest+0x2d90c1)

Location is heap block of size 56 at 0x7b1000002000 allocated by thread T1:
    #0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.0+0x8f542)
    #1 __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/include/c++/11/ext/new_allocator.h:121 (MyAppTest+0x8b350)
    #2 std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long) /usr/include/c++/11/bits/allocator.h:173 (MyAppTest+0x8af4f)
    #3 std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/include/c++/11/bits/alloc_traits.h:460 (MyAppTest+0x8af4f)
    #4 std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<std::mutex, std::allocator<std::mutex>, (__gnu_cxx::_Lock_policy)2> >&) /usr/include/c++/11/bits/allocated_ptr.h:97 (MyAppTest+0x8a96e)
    #5 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<std::mutex, std::allocator<std::mutex>>(std::mutex*&, std::_Sp_alloc_shared_tag<std::allocator<std::mutex> >) /usr/include/c++/11/bits/shared_ptr_base.h:648 (MyAppTest+0x8a1d9)
    #6 std::__shared_ptr<std::mutex, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<std::mutex>>(std::_Sp_alloc_shared_tag<std::allocator<std::mutex> >) /usr/include/c++/11/bits/shared_ptr_base.h:1337 (MyAppTest+0x8996e)
    #7 std::shared_ptr<std::mutex>::shared_ptr<std::allocator<std::mutex>>(std::_Sp_alloc_shared_tag<std::allocator<std::mutex> >) /usr/include/c++/11/bits/shared_ptr.h:409 (MyAppTest+0x88b8d)
    #8 std::shared_ptr<std::mutex> std::allocate_shared<std::mutex, std::allocator<std::mutex>>(std::allocator<std::mutex> const&) /usr/include/c++/11/bits/shared_ptr.h:861 (MyAppTest+0x87c56)
    #9 std::shared_ptr<std::mutex> std::make_shared<std::mutex>() /usr/include/c++/11/bits/shared_ptr.h:877 (MyAppTest+0x8688f)
    #10 std::_V2::condition_variable_any::condition_variable_any() /usr/include/c++/11/condition_variable:283 (MyAppTest+0x85763)
    #11 sleep_for<long int, std::ratio<1> > /home/def/repos/MyApp/Tests/main.cpp:21 (MyAppTest+0x2d91c0)
    #12 operator() /home/def/repos/MyApp/Tests/main.cpp:32 (MyAppTest+0x2d9016)
    #13 __invoke_impl<void, main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:61 (MyAppTest+0x2da0db)
    #14 __invoke<main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:96 (MyAppTest+0x2d9fd2)
    #15 _M_invoke<0, 1> /usr/include/c++/11/bits/std_thread.h:253 (MyAppTest+0x2d9ede)
    #16 operator() /usr/include/c++/11/bits/std_thread.h:260 (MyAppTest+0x2d9e6a)
    #17 _M_run /usr/include/c++/11/bits/std_thread.h:211 (MyAppTest+0x2d9e20)
    #18 <null> <null> (libstdc++.so.6+0xda6b3)

Mutex M51 (0x7b1000002010) created at:
    #0 pthread_mutex_lock ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:4250 (libtsan.so.0+0x53908)
    #1 __gthread_mutex_lock /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 (MyAppTest+0x83c54)
    #2 std::mutex::lock() /usr/include/c++/11/bits/std_mutex.h:100 (MyAppTest+0x83fd2)
    #3 std::unique_lock<std::mutex>::lock() /usr/include/c++/11/bits/unique_lock.h:139 (MyAppTest+0x87e19)
    #4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/include/c++/11/bits/unique_lock.h:69 (MyAppTest+0x86c78)
    #5 wait_until<std::unique_lock<std::mutex>, std::chrono::_V2::steady_clock, std::chrono::duration<long int, std::ratio<1, 1000000000> >, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:410 (MyAppTest+0x2d9642)
    #6 wait_for<std::unique_lock<std::mutex>, long int, std::ratio<1>, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:435 (MyAppTest+0x2d93ef)
    #7 sleep_for<long int, std::ratio<1> > /home/def/repos/MyApp/Tests/main.cpp:21 (MyAppTest+0x2d9204)
    #8 operator() /home/def/repos/MyApp/Tests/main.cpp:32 (MyAppTest+0x2d9016)
    #9 __invoke_impl<void, main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:61 (MyAppTest+0x2da0db)
    #10 __invoke<main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:96 (MyAppTest+0x2d9fd2)
    #11 _M_invoke<0, 1> /usr/include/c++/11/bits/std_thread.h:253 (MyAppTest+0x2d9ede)
    #12 operator() /usr/include/c++/11/bits/std_thread.h:260 (MyAppTest+0x2d9e6a)
    #13 _M_run /usr/include/c++/11/bits/std_thread.h:211 (MyAppTest+0x2d9e20)
    #14 <null> <null> (libstdc++.so.6+0xda6b3)

Thread T1 (tid=6769, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:969 (libtsan.so.0+0x605f8)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xda989)
    #2 _S_create<main()::<lambda(std::stop_token)> > /usr/include/c++/11/thread:217 (MyAppTest+0x2d94c0)
    #3 jthread<main()::<lambda(std::stop_token)> > /usr/include/c++/11/thread:118 (MyAppTest+0x2d9301)
    #4 main /home/def/repos/MyApp/Tests/main.cpp:33 (MyAppTest+0x2d908a)

SUMMARY: ThreadSanitizer: double lock of a mutex /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 in __gthread_mutex_lock
==================
==================
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=6767)
Cycle in lock order graph: M48 (0x7fa6e2e9ece0) => M51 (0x7b1000002010) => M48

Mutex M51 acquired here while holding mutex M48 in thread T1:
    #0 pthread_mutex_lock ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:4250 (libtsan.so.0+0x53908)
    #1 __gthread_mutex_lock /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 (MyAppTest+0x83c54)
    #2 std::mutex::lock() /usr/include/c++/11/bits/std_mutex.h:100 (MyAppTest+0x83fd2)
    #3 std::unique_lock<std::mutex>::lock() /usr/include/c++/11/bits/unique_lock.h:139 (MyAppTest+0x87e19)
    #4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/include/c++/11/bits/unique_lock.h:69 (MyAppTest+0x86c78)
    #5 wait_until<std::unique_lock<std::mutex>, std::chrono::_V2::steady_clock, std::chrono::duration<long int, std::ratio<1, 1000000000> >, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:410 (MyAppTest+0x2d9642)
    #6 wait_for<std::unique_lock<std::mutex>, long int, std::ratio<1>, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:435 (MyAppTest+0x2d93ef)
    #7 sleep_for<long int, std::ratio<1> > /home/def/repos/MyApp/Tests/main.cpp:21 (MyAppTest+0x2d9204)
    #8 operator() /home/def/repos/MyApp/Tests/main.cpp:32 (MyAppTest+0x2d9016)
    #9 __invoke_impl<void, main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:61 (MyAppTest+0x2da0db)
    #10 __invoke<main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:96 (MyAppTest+0x2d9fd2)
    #11 _M_invoke<0, 1> /usr/include/c++/11/bits/std_thread.h:253 (MyAppTest+0x2d9ede)
    #12 operator() /usr/include/c++/11/bits/std_thread.h:260 (MyAppTest+0x2d9e6a)
    #13 _M_run /usr/include/c++/11/bits/std_thread.h:211 (MyAppTest+0x2d9e20)
    #14 <null> <null> (libstdc++.so.6+0xda6b3)

    Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative warning message

Mutex M48 acquired here while holding mutex M51 in thread T1:
    #0 pthread_mutex_lock ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:4250 (libtsan.so.0+0x53908)
    #1 __gthread_mutex_lock /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 (MyAppTest+0x83c54)
    #2 std::mutex::lock() /usr/include/c++/11/bits/std_mutex.h:100 (MyAppTest+0x83fd2)
    #3 std::unique_lock<std::mutex>::lock() /usr/include/c++/11/bits/unique_lock.h:139 (MyAppTest+0x87e19)
    #4 std::_V2::condition_variable_any::_Unlock<std::unique_lock<std::mutex> >::~_Unlock() /usr/include/c++/11/condition_variable:272 (MyAppTest+0x888a5)
    #5 wait_until<std::unique_lock<std::mutex>, std::chrono::_V2::steady_clock, std::chrono::duration<long int, std::ratio<1, 1000000000> >, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:419 (MyAppTest+0x2d9708)
    #6 wait_for<std::unique_lock<std::mutex>, long int, std::ratio<1>, (anonymous namespace)::sleep_for<long int, std::ratio<1> >(const std::chrono::duration<long int>&, const std::stop_token&)::<lambda()> > /usr/include/c++/11/condition_variable:435 (MyAppTest+0x2d93ef)
    #7 sleep_for<long int, std::ratio<1> > /home/def/repos/MyApp/Tests/main.cpp:21 (MyAppTest+0x2d9204)
    #8 operator() /home/def/repos/MyApp/Tests/main.cpp:32 (MyAppTest+0x2d9016)
    #9 __invoke_impl<void, main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:61 (MyAppTest+0x2da0db)
    #10 __invoke<main()::<lambda(std::stop_token)>, std::stop_token> /usr/include/c++/11/bits/invoke.h:96 (MyAppTest+0x2d9fd2)
    #11 _M_invoke<0, 1> /usr/include/c++/11/bits/std_thread.h:253 (MyAppTest+0x2d9ede)
    #12 operator() /usr/include/c++/11/bits/std_thread.h:260 (MyAppTest+0x2d9e6a)
    #13 _M_run /usr/include/c++/11/bits/std_thread.h:211 (MyAppTest+0x2d9e20)
    #14 <null> <null> (libstdc++.so.6+0xda6b3)

Thread T1 (tid=6769, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:969 (libtsan.so.0+0x605f8)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xda989)
    #2 _S_create<main()::<lambda(std::stop_token)> > /usr/include/c++/11/thread:217 (MyAppTest+0x2d94c0)
    #3 jthread<main()::<lambda(std::stop_token)> > /usr/include/c++/11/thread:118 (MyAppTest+0x2d9301)
    #4 main /home/def/repos/MyApp/Tests/main.cpp:33 (MyAppTest+0x2d908a)

SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) /usr/include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:749 in __gthread_mutex_lock
==================
ThreadSanitizer: reported 2 warnings

but if I replace sleep_for function with this:

template<typename Rep, typename Period>
void sleep_for(const std::chrono::duration<Rep, Period>& d, const std::stop_token& token)
{
    std::mutex mutex;

    std::unique_lock<std::mutex> lock{ mutex };

    std::condition_variable cv;

    std::stop_callback stop_wait
    {
        token,
        [&cv]()
        {
            cv.notify_one();
        }
    };

    cv.wait_for(lock, d, [&token]()
    {
        return token.stop_requested();
    });
}

GCC thread sanitizer stops reporting errors.

What can be the difference?

I am not sure if lambda in the first version should return false or token.stop_requested(), but the both alternatives have the same errors.

Answer 1

Looks like it's a bug in gcc (the one I mentioned in the comments). When wait_for() is used and another thread tries to lock the same mutex then it triggers "double lock" warning. A simplified example:

#include <chrono>
#include <condition_variable>
#include <mutex>
#include <thread>
using namespace std::chrono_literals;

std::mutex mtx;
std::condition_variable cv;

void run1() {
    std::unique_lock lck{mtx};
    cv.wait_for(lck, 1s);
}
void run2() {
    std::unique_lock lck{mtx};
    std::this_thread::sleep_for(500ms);
    cv.notify_all();
}
int main() {
    std::jthread th1{ run1 };
    std::jthread th2{ run2 };
}

The reason why it doesn't trigger the warning in your second version of sleep_for() is because in the stop_callback you don't try to lock the mutex before calling cv.notify_one();. And std::condition_variable_any does exactly that, see condition_variable:404 and condition_variable:298:

// line 404:
      std::stop_callback __cb(__stoken, [this] { notify_all(); });

// lines 295-300:
    void
    notify_all() noexcept
    {
      lock_guard<mutex> __lock(*_M_mutex);
      _M_cond.notify_all();
    }

I believe that corresponds to the stacktrace from your warning:

#4 std::_V2::condition_variable_any::notify_all()
/usr/include/c++/11/condition_variable:299 (MyAppTest+0x857f6)
#5 operator()
/usr/include/c++/11/condition_variable:404 (MyAppTest+0x2d9556)

EDIT:

Based on Why does C++20 std::condition_variable not support std::stop_token? your second version of sleep_for() probably has a race condition if that lock in the stop_callback is missing.

So let's see, std::condition_variable::wait_for (2) says that it is equivalent to wait_until() and std::condition_variable::wait_until (2) says it is equivalent to:

while (!pred()) {
    if (wait_until(lock, timeout_time) == std::cv_status::timeout) {
        return pred();
    }
}

Some possible scenario:

1. watchdog thread: inside wait_until() it gets a spurious wakeup, calls your predicate, which returns false. Before it gets a chance to call wait_until() again...
2. main thread: stops the watchdog thread, executes the stop_callback, which calls notify_all()...
3. watchdog thread: misses the notification because it hasn't called wait_until() yet. It calls it now and waits until timeout (or another spurious wakeup).

As a result, the thread doesn't stop when it should.