From b5d7d71e1e78fd34e6976e210bf0021994ba4746 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 30 Jun 2022 09:46:38 -0700 Subject: [PATCH 1/2] use connection pool instead of new connection for member status writes redis plus plus has an annoying feature where it will open a new connection for each tx or pipeline by default, rather than just fetching an existing connection from the pool. Let's change that --- controller/PostgreSQL.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 8e265da33..28418ec32 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -710,11 +710,11 @@ void PostgreSQL::initializeNetworks() if (_redisMemberStatus) { fprintf(stderr, "adding networks to redis...\n"); if (_rc->clusterMode) { - auto tx = _cluster->transaction(_myAddressStr, true); + auto tx = _cluster->transaction(_myAddressStr, true, false); tx.sadd(setKey, networkSet.begin(), networkSet.end()); tx.exec(); } else { - auto tx = _redis->transaction(true); + auto tx = _redis->transaction(true, false); tx.sadd(setKey, networkSet.begin(), networkSet.end()); tx.exec(); } @@ -766,13 +766,13 @@ void PostgreSQL::initializeMembers() if (!deletes.empty()) { try { if (_rc->clusterMode) { - auto tx = _cluster->transaction(_myAddressStr, true); + auto tx = _cluster->transaction(_myAddressStr, true, false); for (std::string k : deletes) { tx.del(k); } tx.exec(); } else { - auto tx = _redis->transaction(true); + auto tx = _redis->transaction(true, false); for (std::string k : deletes) { tx.del(k); } @@ -926,13 +926,13 @@ void PostgreSQL::initializeMembers() if (_redisMemberStatus) { fprintf(stderr, "Load member data into redis...\n"); if (_rc->clusterMode) { - auto tx = _cluster->transaction(_myAddressStr, true); + auto tx = _cluster->transaction(_myAddressStr, true, false); for (auto it : networkMembers) { tx.sadd(it.first, it.second); } tx.exec(); } else { - auto tx = _redis->transaction(true); + auto tx = _redis->transaction(true, false); for (auto it : networkMembers) { tx.sadd(it.first, it.second); } @@ -1696,10 +1696,10 @@ void PostgreSQL::onlineNotification_Redis() try { if (!lastOnline.empty()) { if (_rc->clusterMode) { - auto tx = _cluster->transaction(controllerId, true); + auto tx = _cluster->transaction(controllerId, true, false); count = _doRedisUpdate(tx, controllerId, lastOnline); } else { - auto tx = _redis->transaction(true); + auto tx = _redis->transaction(true, false); count = _doRedisUpdate(tx, controllerId, lastOnline); } } From 9cf8dacfbbab6b9663ba279b8de0a6d3f99a6a4e Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 30 Jun 2022 11:40:04 -0700 Subject: [PATCH 2/2] don't crash out of the controller heartbeat loop here --- controller/PostgreSQL.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 28418ec32..ffe9b0f1f 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -1014,12 +1014,16 @@ void PostgreSQL::heartbeat() } _pool->unborrow(c); - if (_redisMemberStatus) { - if (_rc->clusterMode) { - _cluster->zadd("controllers", "controllerId", ts); - } else { - _redis->zadd("controllers", "controllerId", ts); + try { + if (_redisMemberStatus) { + if (_rc->clusterMode) { + _cluster->zadd("controllers", "controllerId", ts); + } else { + _redis->zadd("controllers", "controllerId", ts); + } } + } catch (sw::redis::Error &e) { + fprintf(stderr, "ERROR: Redis error in heartbeat thread: %s\n", e.what()); } std::this_thread::sleep_for(std::chrono::milliseconds(1000));