UploadRequestAWS gets destructed while FlushUploadQueue is waiting for it to finish

We are seeing 2 types of mutex error appear randomly, and abort the process: unlocking an unowned mutex, and mutex destroyed whilst busy.

The stack for unlocking unowned mutex is:

     ucrtbased.dll!issue_debug_notification(const wchar_t * const message) Line 28    C++
     ucrtbased.dll!__acrt_report_runtime_error(const wchar_t * message) Line 154    C++
     ucrtbased.dll!abort() Line 61    C++
     msvcp140d.dll!_Thrd_abort(const char * msg) Line 18    C++
     msvcp140d.dll!_Mtx_unlock(_Mtx_internal_imp_t * mtx) Line 167    C++
>    otter.dll!std::_Mutex_base::unlock() Line 67    C++
     otter.dll!std::unique_lock<std::mutex>::~unique_lock<std::mutex>() Line 188    C++
     otter.dll!OpenVDS::UploadRequestAWS::WaitForFinish() Line 302    C++
     otter.dll!OpenVDS::VolumeDataAccessManagerImpl::FlushUploadQueue() Line 598    C++
     otter.dll!OpenVDS::VolumeDataPageAccessorImpl::Commit() Line 504    C++
     otter.dll!otter_core::VdsImage::writeToBlock(int block_index, void * buffer, unsigned __int64 buffer_size) Line 410    C++

The stack for the mutex destroyed whilst busy is:

     ucrtbased.dll!issue_debug_notification(const wchar_t * const message) Line 28    C++
     ucrtbased.dll!__acrt_report_runtime_error(const wchar_t * message) Line 154    C++
     ucrtbased.dll!abort() Line 61    C++
     msvcp140d.dll!_Thrd_abort(const char * msg) Line 18    C++
     msvcp140d.dll!_Mtx_destroy_in_situ(_Mtx_internal_imp_t * mtx) Line 65    C++
>    otter.dll!std::_Mutex_base::~_Mutex_base() Line 44    C++
     otter.dll!std::mutex::~mutex()    C++
     otter.dll!OpenVDS::UploadRequestAWS::~UploadRequestAWS()    C++
     otter.dll!OpenVDS::UploadRequestAWS::`scalar deleting destructor'(unsigned int)    C++
     otter.dll!std::_Destroy_in_place<OpenVDS::UploadRequestAWS>(OpenVDS::UploadRequestAWS & _Obj) Line 242    C++
     otter.dll!std::_Ref_count_obj2<OpenVDS::UploadRequestAWS>::_Destroy() Line 1504    C++
     otter.dll!std::_Ref_count_base::_Decref() Line 651    C++
     otter.dll!std::_Ptr_base<OpenVDS::UploadRequestAWS>::_Decref() Line 882    C++
     otter.dll!std::shared_ptr<OpenVDS::UploadRequestAWS>::~shared_ptr<OpenVDS::UploadRequestAWS>() Line 1132    C++
     otter.dll!OpenVDS::upload_callback(const Aws::S3::S3Client * client, const Aws::S3::Model::PutObjectRequest & putRequest, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & outcome, std::weak_ptr<OpenVDS::UploadRequestAWS> weak_upload) Line 228    C++
     otter.dll!OpenVDS::UploadRequestAWS::run::__l2::<lambda>(const Aws::S3::S3Client * client, const Aws::S3::Model::PutObjectRequest & putRequest, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & outcome, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & __formal) Line 293    C++
     otter.dll!std::_Invoker_functor::_Call<void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) &,Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &>(OpenVDS::UploadRequestAWS::run::__l2::void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) & _Obj, const Aws::S3::S3Client * && <_Args_0>, const Aws::S3::Model::PutObjectRequest & <_Args_1>, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & <_Args_2>, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & <_Args_3>) Line 1579    C++
     otter.dll!std::invoke<void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) &,Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &>(OpenVDS::UploadRequestAWS::run::__l2::void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) & _Obj, const Aws::S3::S3Client * && <_Args_0>, const Aws::S3::Model::PutObjectRequest & <_Args_1>, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & <_Args_2>, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & <_Args_3>) Line 1579    C++
     otter.dll!std::_Invoker_ret<void,1>::_Call<void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) &,Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &>(OpenVDS::UploadRequestAWS::run::__l2::void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &) & <_Vals_0>, const Aws::S3::S3Client * && <_Vals_1>, const Aws::S3::Model::PutObjectRequest & <_Vals_2>, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & <_Vals_3>, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & <_Vals_4>) Line 1598    C++
     otter.dll!std::_Func_impl_no_alloc<void <lambda>(const Aws::S3::S3Client *, const Aws::S3::Model::PutObjectRequest &, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> &, const std::shared_ptr<Aws::Client::AsyncCallerContext const> &),void,Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &>::_Do_call(const Aws::S3::S3Client * && <_Args_0>, const Aws::S3::Model::PutObjectRequest & <_Args_1>, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & <_Args_2>, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & <_Args_3>) Line 927    C++
     aws-cpp-sdk-s3.dll!std::_Func_class<void,Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &>::operator()(const Aws::S3::S3Client * <_Args_0>, const Aws::S3::Model::PutObjectRequest & <_Args_1>, const Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> & <_Args_2>, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & <_Args_3>) Line 970    C++
     aws-cpp-sdk-s3.dll!Aws::S3::S3Client::PutObjectAsyncHelper(const Aws::S3::Model::PutObjectRequest & request, const std::function<void __cdecl(Aws::S3::S3Client const *,Aws::S3::Model::PutObjectRequest const &,Aws::Utils::Outcome<Aws::S3::Model::PutObjectResult,Aws::Client::AWSError<enum Aws::S3::S3Errors>> const &,std::shared_ptr<Aws::Client::AsyncCallerContext const> const &)> & handler, const std::shared_ptr<Aws::Client::AsyncCallerContext const> & context) Line 3316    C++
     aws-cpp-sdk-s3.dll!Aws::S3::S3Client::PutObjectAsync::__l2::<lambda>() Line 3311    C++
     aws-cpp-sdk-s3.dll!std::_Invoker_functor::_Call<void <lambda>(void) &>(Aws::S3::S3Client::PutObjectAsync::__l2::void <lambda>(void) & _Obj) Line 1579    C++
     aws-cpp-sdk-s3.dll!std::invoke<void <lambda>(void) &>(Aws::S3::S3Client::PutObjectAsync::__l2::void <lambda>(void) & _Obj) Line 1579    C++
    aws-cpp-sdk-s3.dll!std::_Invoker_ret<std::_Unforced,0>::_Call<void <lambda>(void) &>(Aws::S3::S3Client::PutObjectAsync::__l2::void <lambda>(void) & <_Vals_0>) Line 1615    C++
     aws-cpp-sdk-s3.dll!std::_Call_binder<std::_Unforced,void <lambda>(void),std::tuple<>,std::tuple<>>(std::_Invoker_ret<std::_Unforced,0> __formal, std::integer_sequence<unsigned __int64> __formal, Aws::S3::S3Client::PutObjectAsync::__l2::void <lambda>(void) & _Obj, std::tuple<> & _Tpl, std::tuple<> && _Ut) Line 1402    C++
     aws-cpp-sdk-s3.dll!std::_Binder<std::_Unforced,void <lambda>(void)>::operator()<>() Line 1442    C++
     aws-cpp-sdk-s3.dll!std::_Invoker_functor::_Call<std::_Binder<std::_Unforced,void <lambda>(void)> &>(std::_Binder<std::_Unforced,void <lambda>(void)> & _Obj) Line 1579    C++
     aws-cpp-sdk-s3.dll!std::invoke<std::_Binder<std::_Unforced,void <lambda>(void)> &>(std::_Binder<std::_Unforced,void <lambda>(void)> & _Obj) Line 1579    C++
     aws-cpp-sdk-s3.dll!std::_Invoker_ret<void,1>::_Call<std::_Binder<std::_Unforced,void <lambda>(void)> &>(std::_Binder<std::_Unforced,void <lambda>(void)> & <_Vals_0>) Line 1598    C++
     aws-cpp-sdk-s3.dll!std::_Func_impl_no_alloc<std::_Binder<std::_Unforced,void <lambda>(void)>,void>::_Do_call() Line 927    C++
     aws-cpp-sdk-core.dll!std::_Func_class<void>::operator()() Line 970    C++
     aws-cpp-sdk-core.dll!Aws::Utils::Threading::DefaultExecutor::SubmitToThread::__l2::<lambda>() Line 29    C++
     aws-cpp-sdk-core.dll!std::_Invoker_functor::_Call<void <lambda>(void)>(Aws::Utils::Threading::DefaultExecutor::SubmitToThread::__l2::void <lambda>(void) && _Obj) Line 1579    C++
     aws-cpp-sdk-core.dll!std::invoke<void <lambda>(void)>(Aws::Utils::Threading::DefaultExecutor::SubmitToThread::__l2::void <lambda>(void) && _Obj) Line 1579    C++
     aws-cpp-sdk-core.dll!std::thread::_Invoke<std::tuple<void <lambda>(void)>,0>(void * _RawVals) Line 43    C++
     ucrtbased.dll!thread_start<unsigned int (__cdecl*)(void *),1>(void * const parameter) Line 97    C++