Skip to content

Commit 0a4a03b

Browse files
PastaPastaPastaPastaPastaPasta
committed
lot's of mutex changes
1 parent 180ba2a commit 0a4a03b

12 files changed

+224
-200
lines changed

‎src/llmq/blockprocessor.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class CQuorumBlockProcessor
3535
CEvoDB& evoDb;
3636

3737
// TODO cleanup
38-
mutable CCriticalSection minableCommitmentsCs;
38+
mutable Mutex minableCommitmentsCs;
3939
std::map<std::pair<Consensus::LLMQType, uint256>, uint256> minableCommitmentsByQuorum GUARDED_BY(minableCommitmentsCs);
4040
std::map<uint256, CFinalCommitment> minableCommitments GUARDED_BY(minableCommitmentsCs);
4141

‎src/llmq/debug.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ class CDKGDebugStatus
8989
class CDKGDebugManager
9090
{
9191
private:
92-
mutable CCriticalSection cs;
92+
mutable Mutex cs;
9393
CDKGDebugStatus localStatus GUARDED_BY(cs);
9494

9595
public:

‎src/llmq/dkgsession.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ namespace llmq
3333
// - commit-omit
3434
// - commit-lie
3535

36-
static CCriticalSection cs_simDkgError;
37-
static std::map<std::string, double> simDkgErrorMap;
36+
static Mutex cs_simDkgError;
37+
static std::map<std::string, double> simDkgErrorMap GUARDED_BY(cs_simDkgError);
3838

3939
void SetSimulatedDKGErrorRate(const std::string& type, double rate)
4040
{
@@ -65,7 +65,6 @@ CDKGMember::CDKGMember(const CDeterministicMNCPtr& _dmn, size_t _idx) :
6565
idx(_idx),
6666
id(_dmn->proTxHash)
6767
{
68-
6968
}
7069

7170
bool CDKGSession::Init(const CBlockIndex* _pQuorumBaseBlockIndex, const std::vector<CDeterministicMNCPtr>& mns, const uint256& _myProTxHash)

‎src/llmq/dkgsessionhandler.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class CDKGPendingMessages
4343
using BinaryMessage = std::pair<NodeId, std::shared_ptr<CDataStream>>;
4444

4545
private:
46-
mutable CCriticalSection cs;
46+
mutable Mutex cs;
4747
const int invType;
4848
size_t maxMessagesPerNode GUARDED_BY(cs);
4949
std::list<BinaryMessage> pendingMessages GUARDED_BY(cs);
@@ -54,10 +54,10 @@ class CDKGPendingMessages
5454
explicit CDKGPendingMessages(size_t _maxMessagesPerNode, int _invType) :
5555
invType(_invType), maxMessagesPerNode(_maxMessagesPerNode) {};
5656

57-
void PushPendingMessage(NodeId from, CDataStream& vRecv);
58-
std::list<BinaryMessage> PopPendingMessages(size_t maxCount);
59-
bool HasSeen(const uint256& hash) const;
60-
void Clear();
57+
void PushPendingMessage(NodeId from, CDataStream& vRecv) LOCKS_EXCLUDED(cs);
58+
std::list<BinaryMessage> PopPendingMessages(size_t maxCount) LOCKS_EXCLUDED(cs);
59+
bool HasSeen(const uint256& hash) const LOCKS_EXCLUDED(cs);
60+
void Clear() LOCKS_EXCLUDED(cs);
6161

6262
template<typename Message>
6363
void PushPendingMessage(NodeId from, Message& msg)

‎src/llmq/dkgsessionmgr.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,6 @@ void CDKGSessionManager::WriteEncryptedContributions(Consensus::LLMQType llmqTyp
306306

307307
bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const std::vector<bool>& validMembers, std::vector<uint16_t>& memberIndexesRet, std::vector<BLSVerificationVectorPtr>& vvecsRet, BLSSecretKeyVector& skContributionsRet) const
308308
{
309-
LOCK(contributionsCacheCs);
310309
auto members = CLLMQUtils::GetAllQuorumMembers(GetLLMQParams(llmqType), pQuorumBaseBlockIndex);
311310

312311
memberIndexesRet.clear();
@@ -315,6 +314,8 @@ bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType,
315314
memberIndexesRet.reserve(members.size());
316315
vvecsRet.reserve(members.size());
317316
skContributionsRet.reserve(members.size());
317+
318+
LOCK(contributionsCacheCs);
318319
for (size_t i = 0; i < members.size(); i++) {
319320
if (validMembers[i]) {
320321
const uint256& proTxHash = members[i]->proTxHash;
@@ -371,8 +372,8 @@ bool CDKGSessionManager::GetEncryptedContributions(Consensus::LLMQType llmqType,
371372

372373
void CDKGSessionManager::CleanupCache() const
373374
{
374-
LOCK(contributionsCacheCs);
375375
auto curTime = GetTimeMillis();
376+
LOCK(contributionsCacheCs);
376377
for (auto it = contributionsCache.begin(); it != contributionsCache.end(); ) {
377378
if (curTime - it->second.entryTime > MAX_CONTRIBUTION_CACHE_TIME) {
378379
it = contributionsCache.erase(it);

‎src/llmq/dkgsessionmgr.h

+6-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ class CDKGSessionManager
2626

2727
std::map<Consensus::LLMQType, CDKGSessionHandler> dkgSessionHandlers;
2828

29-
mutable CCriticalSection contributionsCacheCs;
3029
struct ContributionsCacheKey {
3130
Consensus::LLMQType llmqType;
3231
uint256 quorumHash;
@@ -43,6 +42,7 @@ class CDKGSessionManager
4342
BLSVerificationVectorPtr vvec;
4443
CBLSSecretKey skContribution;
4544
};
45+
mutable Mutex contributionsCacheCs;
4646
mutable std::map<ContributionsCacheKey, ContributionsCacheEntry> contributionsCache GUARDED_BY(contributionsCacheCs);
4747

4848
public:
@@ -64,15 +64,18 @@ class CDKGSessionManager
6464
// Contributions are written while in the DKG
6565
void WriteVerifiedVvecContribution(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const uint256& proTxHash, const BLSVerificationVectorPtr& vvec);
6666
void WriteVerifiedSkContribution(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const uint256& proTxHash, const CBLSSecretKey& skContribution);
67-
bool GetVerifiedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const std::vector<bool>& validMembers, std::vector<uint16_t>& memberIndexesRet, std::vector<BLSVerificationVectorPtr>& vvecsRet, BLSSecretKeyVector& skContributionsRet) const;
67+
bool GetVerifiedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex,
68+
const std::vector<bool>& validMembers, std::vector<uint16_t>& memberIndexesRet,
69+
std::vector<BLSVerificationVectorPtr>& vvecsRet,
70+
BLSSecretKeyVector& skContributionsRet) const LOCKS_EXCLUDED(contributionsCacheCs);
6871
/// Write encrypted (unverified) DKG contributions for the member with the given proTxHash to the llmqDb
6972
void WriteEncryptedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const uint256& proTxHash, const CBLSIESMultiRecipientObjects<CBLSSecretKey>& contributions);
7073
/// Read encrypted (unverified) DKG contributions for the member with the given proTxHash from the llmqDb
7174
bool GetEncryptedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex, const std::vector<bool>& validMembers, const uint256& proTxHash, std::vector<CBLSIESEncryptedObject<CBLSSecretKey>>& vecRet) const;
7275

7376
private:
7477
void MigrateDKG();
75-
void CleanupCache() const;
78+
void CleanupCache() const LOCKS_EXCLUDED(contributionsCacheCs);
7679
};
7780

7881
bool IsQuorumDKGEnabled();

‎src/llmq/quorums.cpp

+22-25
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ static const std::string DB_QUORUM_QUORUM_VVEC = "q_Qqvvec";
3131

3232
CQuorumManager* quorumManager;
3333

34-
CCriticalSection cs_data_requests;
34+
Mutex cs_data_requests;
3535
static std::unordered_map<std::pair<uint256, bool>, CQuorumDataRequest, StaticSaltedHasher> mapQuorumDataRequests GUARDED_BY(cs_data_requests);
3636

3737
static uint256 MakeQuorumKey(const CQuorum& q)
@@ -285,7 +285,6 @@ void CQuorumManager::EnsureQuorumConnections(const Consensus::LLMQParams& llmqPa
285285

286286
CQuorumPtr CQuorumManager::BuildQuorumFromCommitment(const Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex) const
287287
{
288-
AssertLockHeld(quorumsCacheCs);
289288
assert(pQuorumBaseBlockIndex);
290289

291290
const uint256& quorumHash{pQuorumBaseBlockIndex->GetBlockHash()};
@@ -321,7 +320,7 @@ CQuorumPtr CQuorumManager::BuildQuorumFromCommitment(const Consensus::LLMQType l
321320
StartCachePopulatorThread(quorum);
322321
}
323322

324-
mapQuorumsCache[llmqType].insert(quorumHash, quorum);
323+
WITH_LOCK(quorumsCacheCs, mapQuorumsCache[llmqType].insert(quorumHash, quorum));
325324

326325
return quorum;
327326
}
@@ -336,20 +335,22 @@ bool CQuorumManager::BuildQuorumContributions(const CFinalCommitmentPtr& fqc, co
336335
}
337336

338337
cxxtimer::Timer t2(true);
339-
LOCK(quorum->cs);
340-
quorum->quorumVvec = blsWorker.BuildQuorumVerificationVector(vvecs);
341-
if (!quorum->HasVerificationVector()) {
342-
LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- failed to build quorumVvec\n", __func__);
343-
// without the quorum vvec, there can't be a skShare, so we fail here. Failure is not fatal here, as it still
344-
// allows to use the quorum as a non-member (verification through the quorum pub key)
345-
return false;
346-
}
347-
quorum->skShare = blsWorker.AggregateSecretKeys(skContributions);
348-
if (!quorum->skShare.IsValid()) {
349-
quorum->skShare.Reset();
350-
LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- failed to build skShare\n", __func__);
351-
// We don't bail out here as this is not a fatal error and still allows us to recover public key shares (as we
352-
// have a valid quorum vvec at this point)
338+
{
339+
LOCK(quorum->cs);
340+
quorum->quorumVvec = blsWorker.BuildQuorumVerificationVector(vvecs);
341+
if (!quorum->HasVerificationVector()) {
342+
LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- failed to build quorumVvec\n", __func__);
343+
// without the quorum vvec, there can't be a skShare, so we fail here. Failure is not fatal here, as it still
344+
// allows to use the quorum as a non-member (verification through the quorum pub key)
345+
return false;
346+
}
347+
quorum->skShare = blsWorker.AggregateSecretKeys(skContributions);
348+
if (!quorum->skShare.IsValid()) {
349+
quorum->skShare.Reset();
350+
LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- failed to build skShare\n", __func__);
351+
// We don't bail out here as this is not a fatal error and still allows us to recover public key shares (as we
352+
// have a valid quorum vvec at this point)
353+
}
353354
}
354355
t2.stop();
355356

@@ -493,9 +494,8 @@ CQuorumCPtr CQuorumManager::GetQuorum(Consensus::LLMQType llmqType, const CBlock
493494
return nullptr;
494495
}
495496

496-
LOCK(quorumsCacheCs);
497497
CQuorumPtr pQuorum;
498-
if (mapQuorumsCache[llmqType].get(quorumHash, pQuorum)) {
498+
if (WITH_LOCK(quorumsCacheCs, return mapQuorumsCache[llmqType].get(quorumHash, pQuorum))) {
499499
return pQuorum;
500500
}
501501

@@ -649,12 +649,9 @@ void CQuorumManager::ProcessMessage(CNode* pFrom, const std::string& strCommand,
649649
}
650650

651651
CQuorumPtr pQuorum;
652-
{
653-
LOCK(quorumsCacheCs);
654-
if (!mapQuorumsCache[request.GetLLMQType()].get(request.GetQuorumHash(), pQuorum)) {
655-
errorHandler("Quorum not found", 0); // Don't bump score because we asked for it
656-
return;
657-
}
652+
if (LOCK(quorumsCacheCs); !mapQuorumsCache[request.GetLLMQType()].get(request.GetQuorumHash(), pQuorum)) {
653+
errorHandler("Quorum not found", 0); // Don't bump score because we asked for it
654+
return;
658655
}
659656

660657
// Check if request has QUORUM_VERIFICATION_VECTOR data

‎src/llmq/quorums.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ class CQuorumManager
191191
CBLSWorker& blsWorker;
192192
CDKGSessionManager& dkgManager;
193193

194-
mutable CCriticalSection quorumsCacheCs;
194+
mutable Mutex quorumsCacheCs;
195195
mutable std::map<Consensus::LLMQType, unordered_lru_cache<uint256, CQuorumPtr, StaticSaltedHasher>> mapQuorumsCache GUARDED_BY(quorumsCacheCs);
196196
mutable std::map<Consensus::LLMQType, unordered_lru_cache<uint256, std::vector<CQuorumCPtr>, StaticSaltedHasher>> scanQuorumsCache GUARDED_BY(quorumsCacheCs);
197197

@@ -226,7 +226,7 @@ class CQuorumManager
226226
// all private methods here are cs_main-free
227227
void EnsureQuorumConnections(const Consensus::LLMQParams& llmqParams, const CBlockIndex *pindexNew) const;
228228

229-
CQuorumPtr BuildQuorumFromCommitment(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex) const EXCLUSIVE_LOCKS_REQUIRED(quorumsCacheCs);
229+
CQuorumPtr BuildQuorumFromCommitment(Consensus::LLMQType llmqType, const CBlockIndex* pQuorumBaseBlockIndex) const;
230230
bool BuildQuorumContributions(const CFinalCommitmentPtr& fqc, const std::shared_ptr<CQuorum>& quorum) const;
231231

232232
CQuorumCPtr GetQuorum(Consensus::LLMQType llmqType, const CBlockIndex* pindex) const;

‎src/llmq/signing.cpp

+6-12
Original file line numberDiff line numberDiff line change
@@ -358,8 +358,6 @@ void CRecoveredSigsDb::WriteRecoveredSig(const llmq::CRecoveredSig& recSig)
358358

359359
void CRecoveredSigsDb::RemoveRecoveredSig(CDBBatch& batch, Consensus::LLMQType llmqType, const uint256& id, bool deleteHashKey, bool deleteTimeKey)
360360
{
361-
AssertLockHeld(cs);
362-
363361
CRecoveredSig recSig;
364362
if (!ReadRecoveredSig(llmqType, id, recSig)) {
365363
return;
@@ -389,6 +387,7 @@ void CRecoveredSigsDb::RemoveRecoveredSig(CDBBatch& batch, Consensus::LLMQType l
389387
}
390388
}
391389

390+
LOCK(cs);
392391
hasSigForIdCache.erase(std::make_pair(recSig.llmqType, recSig.id));
393392
hasSigForSessionCache.erase(signHash);
394393
if (deleteHashKey) {
@@ -399,7 +398,6 @@ void CRecoveredSigsDb::RemoveRecoveredSig(CDBBatch& batch, Consensus::LLMQType l
399398
// Completely remove any traces of the recovered sig
400399
void CRecoveredSigsDb::RemoveRecoveredSig(Consensus::LLMQType llmqType, const uint256& id)
401400
{
402-
LOCK(cs);
403401
CDBBatch batch(*db);
404402
RemoveRecoveredSig(batch, llmqType, id, true, true);
405403
db->WriteBatch(batch);
@@ -409,7 +407,6 @@ void CRecoveredSigsDb::RemoveRecoveredSig(Consensus::LLMQType llmqType, const ui
409407
// This will leave the byHash key in-place so that HasRecoveredSigForHash still returns true
410408
void CRecoveredSigsDb::TruncateRecoveredSig(Consensus::LLMQType llmqType, const uint256& id)
411409
{
412-
LOCK(cs);
413410
CDBBatch batch(*db);
414411
RemoveRecoveredSig(batch, llmqType, id, false, false);
415412
db->WriteBatch(batch);
@@ -448,15 +445,12 @@ void CRecoveredSigsDb::CleanupOldRecoveredSigs(int64_t maxAge)
448445
}
449446

450447
CDBBatch batch(*db);
451-
{
452-
LOCK(cs);
453-
for (const auto& e : toDelete) {
454-
RemoveRecoveredSig(batch, e.first, e.second, true, false);
448+
for (const auto& e : toDelete) {
449+
RemoveRecoveredSig(batch, e.first, e.second, true, false);
455450

456-
if (batch.SizeEstimate() >= (1 << 24)) {
457-
db->WriteBatch(batch);
458-
batch.Clear();
459-
}
451+
if (batch.SizeEstimate() >= (1 << 24)) {
452+
db->WriteBatch(batch);
453+
batch.Clear();
460454
}
461455
}
462456

‎src/llmq/signing.h

+4-4
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ class CRecoveredSigsDb
7878
private:
7979
std::unique_ptr<CDBWrapper> db{nullptr};
8080

81-
mutable CCriticalSection cs;
81+
mutable Mutex cs;
8282
mutable unordered_lru_cache<std::pair<Consensus::LLMQType, uint256>, bool, StaticSaltedHasher, 30000> hasSigForIdCache GUARDED_BY(cs);
8383
mutable unordered_lru_cache<uint256, bool, StaticSaltedHasher, 30000> hasSigForSessionCache GUARDED_BY(cs);
8484
mutable unordered_lru_cache<uint256, bool, StaticSaltedHasher, 30000> hasSigForHashCache GUARDED_BY(cs);
@@ -113,7 +113,7 @@ class CRecoveredSigsDb
113113
void MigrateRecoveredSigs();
114114

115115
bool ReadRecoveredSig(Consensus::LLMQType llmqType, const uint256& id, CRecoveredSig& ret) const;
116-
void RemoveRecoveredSig(CDBBatch& batch, Consensus::LLMQType llmqType, const uint256& id, bool deleteHashKey, bool deleteTimeKey) EXCLUSIVE_LOCKS_REQUIRED(cs);
116+
void RemoveRecoveredSig(CDBBatch& batch, Consensus::LLMQType llmqType, const uint256& id, bool deleteHashKey, bool deleteTimeKey);
117117
};
118118

119119
class CRecoveredSigsListener
@@ -134,10 +134,10 @@ class CSigningManager
134134
static constexpr int SIGN_HEIGHT_OFFSET{8};
135135

136136
private:
137-
mutable CCriticalSection cs;
138-
139137
CRecoveredSigsDb db;
140138

139+
mutable CCriticalSection cs;
140+
141141
// Incoming and not verified yet
142142
std::unordered_map<NodeId, std::list<std::shared_ptr<const CRecoveredSig>>> pendingRecoveredSigs GUARDED_BY(cs);
143143
std::unordered_map<uint256, std::shared_ptr<const CRecoveredSig>, StaticSaltedHasher> pendingReconstructedRecoveredSigs GUARDED_BY(cs);

0 commit comments

Comments
 (0)