github/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/ZeroTier/ZeroTierOne synced 2025-08-24 07:05:50 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove exploratory metrics

Browse source 
place controller request benchmarks behind ifdef
This commit is contained in:
Grant Limberg 2023-05-22 15:54:48 -07:00
commit ffd74eed5a
No known key found for this signature in database
GPG key ID: 8F2F97D3BE8D7735
5 changed files with 82 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

71
controller/EmbeddedNetworkController.cpp
View file

@ -471,6 +471,7 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPa
, _ssoExpiryRunning(true) , _ssoExpiryRunning(true)
, _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this)) , _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this))
, _rc(rc) , _rc(rc)
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
, _member_status_lookup{"nc_member_status_lookup",""} , _member_status_lookup{"nc_member_status_lookup",""}
, _member_status_lookup_count{"nc_member_status_lookup_count",""} , _member_status_lookup_count{"nc_member_status_lookup_count",""}
, _node_is_online{"nc_node_is_online",""} , _node_is_online{"nc_node_is_online",""}
@ -493,6 +494,7 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPa
, _save_member_count{"nc_save_member_count",""} , _save_member_count{"nc_save_member_count",""}
, _send_netconf{"nc_send_netconf2",""} , _send_netconf{"nc_send_netconf2",""}
, _send_netconf_count{"nc_send_netconf2_count",""} , _send_netconf_count{"nc_send_netconf2_count",""}
#endif
{ {
} }
@ -571,6 +573,18 @@ void EmbeddedNetworkController::request(
if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender))
return; return;
_startThreads(); _startThreads();
const int64_t now = OSUtils::now();
if (requestPacketId) {
std::lock_guard<std::mutex> l(_memberStatus_l);
_MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())];
if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) {
return;
}
ms.lastRequestTime = now;
}
_RQEntry *qe = new _RQEntry; _RQEntry *qe = new _RQEntry;
qe->nwid = nwid; qe->nwid = nwid;
qe->requestPacketId = requestPacketId; qe->requestPacketId = requestPacketId;
@ -1202,10 +1216,12 @@ void EmbeddedNetworkController::_request(
auto tid = std::this_thread::get_id(); auto tid = std::this_thread::get_id();
std::stringstream ss; ss << tid; std::stringstream ss; ss << tid;
std::string threadID = ss.str(); std::string threadID = ss.str();
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b1 = _member_status_lookup.Add({{"thread", threadID}}); auto b1 = _member_status_lookup.Add({{"thread", threadID}});
auto c1 = _member_status_lookup_count.Add({{"thread", threadID}}); auto c1 = _member_status_lookup_count.Add({{"thread", threadID}});
c1++; c1++;
b1.start(); b1.start();
#endif
char nwids[24]; char nwids[24];
DB::NetworkSummaryInfo ns; DB::NetworkSummaryInfo ns;
@ -1217,44 +1233,45 @@ void EmbeddedNetworkController::_request(
const int64_t now = OSUtils::now(); const int64_t now = OSUtils::now();
if (requestPacketId) { #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
std::lock_guard<std::mutex> l(_memberStatus_l);
_MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())];
if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) {
return;
}
ms.lastRequestTime = now;
}
b1.stop(); b1.stop();
auto b2 = _node_is_online.Add({{"thread",threadID}}); auto b2 = _node_is_online.Add({{"thread",threadID}});
auto c2 = _node_is_online_count.Add({{"thread",threadID}}); auto c2 = _node_is_online_count.Add({{"thread",threadID}});
c2++; c2++;
b2.start(); b2.start();
#endif
_db.nodeIsOnline(nwid,identity.address().toInt(),fromAddr); _db.nodeIsOnline(nwid,identity.address().toInt(),fromAddr);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b2.stop(); b2.stop();
auto b3 = _get_and_init_member.Add({{"thread", threadID}}); auto b3 = _get_and_init_member.Add({{"thread", threadID}});
auto c3 = _get_and_init_member_count.Add({{"thread",threadID}}); auto c3 = _get_and_init_member_count.Add({{"thread",threadID}});
c3++; c3++;
b3.start(); b3.start();
#endif
Utils::hex(nwid,nwids); Utils::hex(nwid,nwids);
_db.get(nwid,network,identity.address().toInt(),member,ns); _db.get(nwid,network,identity.address().toInt(),member,ns);
if ((!network.is_object())||(network.empty())) { if ((!network.is_object())||(network.empty())) {
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b3.stop(); b3.stop();
#endif
return; return;
} }
const bool newMember = ((!member.is_object())||(member.empty())); const bool newMember = ((!member.is_object())||(member.empty()));
DB::initMember(member); DB::initMember(member);
_MemberStatusKey msk(nwid,identity.address().toInt()); _MemberStatusKey msk(nwid,identity.address().toInt());
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b3.stop(); b3.stop();
#endif
{ {
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b4 = _have_identity.Add({{"thread",threadID}}); auto b4 = _have_identity.Add({{"thread",threadID}});
auto c4 = _have_identity_count.Add({{"thread",threadID}}); auto c4 = _have_identity_count.Add({{"thread",threadID}});
c4++; c4++;
b4.start(); b4.start();
#endif
const std::string haveIdStr(OSUtils::jsonString(member["identity"],"")); const std::string haveIdStr(OSUtils::jsonString(member["identity"],""));
if (haveIdStr.length() > 0) { if (haveIdStr.length() > 0) {
// If we already know this member's identity perform a full compare. This prevents // If we already know this member's identity perform a full compare. This prevents
@ -1263,12 +1280,16 @@ void EmbeddedNetworkController::_request(
try { try {
if (Identity(haveIdStr.c_str()) != identity) { if (Identity(haveIdStr.c_str()) != identity) {
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b4.stop(); b4.stop();
#endif
return; return;
} }
} catch ( ... ) { } catch ( ... ) {
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b4.stop(); b4.stop();
#endif
return; return;
} }
} else { } else {
@ -1276,7 +1297,9 @@ void EmbeddedNetworkController::_request(
char idtmp[1024]; char idtmp[1024];
member["identity"] = identity.toString(false,idtmp); member["identity"] = identity.toString(false,idtmp);
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b4.stop(); b4.stop();
#endif
} }
// These are always the same, but make sure they are set // These are always the same, but make sure they are set
@ -1289,10 +1312,12 @@ void EmbeddedNetworkController::_request(
} }
// Determine whether and how member is authorized // Determine whether and how member is authorized
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b5 = _determine_auth.Add({{"thread",threadID}}); auto b5 = _determine_auth.Add({{"thread",threadID}});
auto c5 = _determine_auth_count.Add({{"thread",threadID}}); auto c5 = _determine_auth_count.Add({{"thread",threadID}});
c5++; c5++;
b5.start(); b5.start();
#endif
bool authorized = false; bool authorized = false;
bool autoAuthorized = false; bool autoAuthorized = false;
json autoAuthCredentialType,autoAuthCredential; json autoAuthCredentialType,autoAuthCredential;
@ -1329,15 +1354,19 @@ void EmbeddedNetworkController::_request(
member["lastAuthorizedCredentialType"] = autoAuthCredentialType; member["lastAuthorizedCredentialType"] = autoAuthCredentialType;
member["lastAuthorizedCredential"] = autoAuthCredential; member["lastAuthorizedCredential"] = autoAuthCredential;
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b5.stop(); b5.stop();
#endif
// Should we check SSO Stuff? // Should we check SSO Stuff?
// If network is configured with SSO, and the member is not marked exempt: yes // If network is configured with SSO, and the member is not marked exempt: yes
// Otherwise no, we use standard auth logic. // Otherwise no, we use standard auth logic.
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b6 = _sso_check.Add({{"thread",threadID}}); auto b6 = _sso_check.Add({{"thread",threadID}});
auto c6 = _sso_check_count.Add({{"thread",threadID}}); auto c6 = _sso_check_count.Add({{"thread",threadID}});
c6++; c6++;
b6.start(); b6.start();
#endif
AuthInfo info; AuthInfo info;
int64_t authenticationExpiryTime = -1; int64_t authenticationExpiryTime = -1;
bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false); bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
@ -1364,18 +1393,21 @@ void EmbeddedNetworkController::_request(
} }
DB::cleanMember(member); DB::cleanMember(member);
_db.save(member,true); _db.save(member,true);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b6.stop(); b6.stop();
#endif
return; return;
} }
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b6.stop(); b6.stop();
auto b7 = _auth_check.Add({{"thread",threadID}}); auto b7 = _auth_check.Add({{"thread",threadID}});
auto c7 = _auth_check_count.Add({{"thread",threadID}}); auto c7 = _auth_check_count.Add({{"thread",threadID}});
c7++; c7++;
b7.start(); b7.start();
#endif
if (authorized) { if (authorized) {
Metrics::nc_authed_request++;
// Update version info and meta-data if authorized and if this is a genuine request // Update version info and meta-data if authorized and if this is a genuine request
if (requestPacketId) { if (requestPacketId) {
const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0);
@ -1406,15 +1438,18 @@ void EmbeddedNetworkController::_request(
} }
} }
} else { } else {
Metrics::nc_unauthed_request++;
// If they are not authorized, STOP! // If they are not authorized, STOP!
DB::cleanMember(member); DB::cleanMember(member);
_db.save(member,true); _db.save(member,true);
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b7.stop(); b7.stop();
#endif
return; return;
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b7.stop(); b7.stop();
#endif
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// If we made it this far, they are authorized (and authenticated). // If we made it this far, they are authorized (and authenticated).
@ -1422,10 +1457,12 @@ void EmbeddedNetworkController::_request(
// Default timeout: 15 minutes. Maximum: two hours. Can be specified by an optional field in the network config // Default timeout: 15 minutes. Maximum: two hours. Can be specified by an optional field in the network config
// if something longer than 15 minutes is desired. Minimum is 5 minutes since shorter than that would be flaky. // if something longer than 15 minutes is desired. Minimum is 5 minutes since shorter than that would be flaky.
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b8 = _json_schlep.Add({{"thread",threadID}}); auto b8 = _json_schlep.Add({{"thread",threadID}});
auto c8 = _json_schlep_count.Add({{"thread", threadID}}); auto c8 = _json_schlep_count.Add({{"thread", threadID}});
c8++; c8++;
b8.start(); b8.start();
#endif
int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_DFL_MAX_DELTA; int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_DFL_MAX_DELTA;
if (network.contains("certificateTimeoutWindowSize")) { if (network.contains("certificateTimeoutWindowSize")) {
credentialtmd = (int64_t)network["certificateTimeoutWindowSize"]; credentialtmd = (int64_t)network["certificateTimeoutWindowSize"];
@ -1815,13 +1852,17 @@ void EmbeddedNetworkController::_request(
} else { } else {
dns = json::object(); dns = json::object();
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b8.stop(); b8.stop();
#endif
// Issue a certificate of ownership for all static IPs // Issue a certificate of ownership for all static IPs
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
auto b9 = _issue_certificate.Add({{"thread",threadID}}); auto b9 = _issue_certificate.Add({{"thread",threadID}});
auto c9 = _issue_certificate_count.Add({{"thread",threadID}}); auto c9 = _issue_certificate_count.Add({{"thread",threadID}});
c9++; c9++;
b9.start(); b9.start();
#endif
if (nc->staticIpCount) { if (nc->staticIpCount) {
nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1);
for(unsigned int i=0;i<nc->staticIpCount;++i) { for(unsigned int i=0;i<nc->staticIpCount;++i) {
@ -1836,25 +1877,33 @@ void EmbeddedNetworkController::_request(
nc->com = com; nc->com = com;
} else { } else {
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b9.stop(); b9.stop();
#endif
return; return;
} }
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b9.stop(); b9.stop();
auto b10 = _save_member.Add({{"thread",threadID}}); auto b10 = _save_member.Add({{"thread",threadID}});
auto c10 = _save_member_count.Add({{"thread",threadID}}); auto c10 = _save_member_count.Add({{"thread",threadID}});
c10++; c10++;
b10.start(); b10.start();
#endif
DB::cleanMember(member); DB::cleanMember(member);
_db.save(member,true); _db.save(member,true);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b10.stop(); b10.stop();
auto b11 = _send_netconf.Add({{"thread",threadID}}); auto b11 = _send_netconf.Add({{"thread",threadID}});
auto c11 = _send_netconf_count.Add({{"thread",threadID}}); auto c11 = _send_netconf_count.Add({{"thread",threadID}});
c11++; c11++;
b11.start(); b11.start();
#endif
_sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6);
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
b11.stop(); b11.stop();
#endif
} }
void EmbeddedNetworkController::_startThreads() void EmbeddedNetworkController::_startThreads()

2
controller/EmbeddedNetworkController.hpp
View file

@ -151,6 +151,7 @@ private:
RedisConfig *_rc; RedisConfig *_rc;
std::string _ssoRedirectURL; std::string _ssoRedirectURL;
#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
prometheus::simpleapi::benchmark_family_t _member_status_lookup; prometheus::simpleapi::benchmark_family_t _member_status_lookup;
prometheus::simpleapi::counter_family_t _member_status_lookup_count; prometheus::simpleapi::counter_family_t _member_status_lookup_count;
prometheus::simpleapi::benchmark_family_t _node_is_online; prometheus::simpleapi::benchmark_family_t _node_is_online;
@ -173,6 +174,7 @@ private:
prometheus::simpleapi::counter_family_t _save_member_count; prometheus::simpleapi::counter_family_t _save_member_count;
prometheus::simpleapi::benchmark_family_t _send_netconf; prometheus::simpleapi::benchmark_family_t _send_netconf;
prometheus::simpleapi::counter_family_t _send_netconf_count; prometheus::simpleapi::counter_family_t _send_netconf_count;
#endif
}; };
} // namespace ZeroTier } // namespace ZeroTier

13
controller/PostgreSQL.cpp
View file

@ -389,9 +389,16 @@ AuthInfo PostgreSQL::getSSOAuthInfo(const nlohmann::json &member, const std::str
char authenticationURL[4096] = {0}; char authenticationURL[4096] = {0};
AuthInfo info; AuthInfo info;
info.enabled = true; info.enabled = true;
//if (memberId == "a10dccea52" && networkId == "8056c2e21c24673d") {
// fprintf(stderr, "invalid authinfo for grant's machine\n");
// info.version=1;
// return info;
//}
// fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str()); // fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str());
std::shared_ptr<PostgresConnection> c;
try { try {
auto c = _pool->borrow(); c = _pool->borrow();
pqxx::work w(*c->c); pqxx::work w(*c->c);
char nonceBytes[16] = {0}; char nonceBytes[16] = {0};
@ -520,6 +527,9 @@ AuthInfo PostgreSQL::getSSOAuthInfo(const nlohmann::json &member, const std::str
_pool->unborrow(c); _pool->unborrow(c);
} catch (std::exception &e) { } catch (std::exception &e) {
if (c) {
_pool->unborrow(c);
}
fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what()); fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what());
} }
@ -1271,6 +1281,7 @@ void PostgreSQL::commitThread()
continue; continue;
} }
Metrics::pgsql_commit_ticks++;
try { try {
nlohmann::json &config = (qitem.first); nlohmann::json &config = (qitem.first);
const std::string objtype = config["objtype"]; const std::string objtype = config["objtype"];

11
node/Metrics.cpp
View file

@ -233,13 +233,7 @@ namespace ZeroTier {
{ "controller_db_member_change", "counter" }; { "controller_db_member_change", "counter" };
prometheus::simpleapi::counter_metric_t db_network_change prometheus::simpleapi::counter_metric_t db_network_change
{ "controller_db_network_change", "counter" }; { "controller_db_network_change", "counter" };
prometheus::simpleapi::counter_metric_t db_get_sso_info
{ "controller_db_get_sso_info", "counter" };
prometheus::simpleapi::counter_metric_t nc_unauthed_request
{ "controller_authorized_requests", "counter" };
prometheus::simpleapi::counter_metric_t nc_authed_request
{ "controller_unauthorized_requests", "counter"};
#ifdef ZT_CONTROLLER_USE_LIBPQ #ifdef ZT_CONTROLLER_USE_LIBPQ
// Central Controller Metrics // Central Controller Metrics
prometheus::simpleapi::counter_metric_t pgsql_mem_notification prometheus::simpleapi::counter_metric_t pgsql_mem_notification
@ -248,6 +242,11 @@ namespace ZeroTier {
{ "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" }; { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
prometheus::simpleapi::counter_metric_t pgsql_node_checkin prometheus::simpleapi::counter_metric_t pgsql_node_checkin
{ "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" }; { "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" };
prometheus::simpleapi::counter_metric_t pgsql_commit_ticks
{ "controller_pgsql_commit_ticks", "number of commit ticks run (pgsql)" };
prometheus::simpleapi::counter_metric_t db_get_sso_info
{ "controller_db_get_sso_info", "counter" };
prometheus::simpleapi::counter_metric_t redis_mem_notification prometheus::simpleapi::counter_metric_t redis_mem_notification
{ "controller_redis_member_notifications_received", "number of member change notifications received via redis" }; { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
prometheus::simpleapi::counter_metric_t redis_net_notification prometheus::simpleapi::counter_metric_t redis_net_notification

6
node/Metrics.hpp
View file

@ -136,16 +136,16 @@ namespace ZeroTier {
extern prometheus::simpleapi::counter_metric_t db_get_network_list; extern prometheus::simpleapi::counter_metric_t db_get_network_list;
extern prometheus::simpleapi::counter_metric_t db_member_change; extern prometheus::simpleapi::counter_metric_t db_member_change;
extern prometheus::simpleapi::counter_metric_t db_network_change; extern prometheus::simpleapi::counter_metric_t db_network_change;
extern prometheus::simpleapi::counter_metric_t db_get_sso_info;
extern prometheus::simpleapi::counter_metric_t nc_unauthed_request;
extern prometheus::simpleapi::counter_metric_t nc_authed_request;
#ifdef ZT_CONTROLLER_USE_LIBPQ #ifdef ZT_CONTROLLER_USE_LIBPQ
// Central Controller Metrics // Central Controller Metrics
extern prometheus::simpleapi::counter_metric_t pgsql_mem_notification; extern prometheus::simpleapi::counter_metric_t pgsql_mem_notification;
extern prometheus::simpleapi::counter_metric_t pgsql_net_notification; extern prometheus::simpleapi::counter_metric_t pgsql_net_notification;
extern prometheus::simpleapi::counter_metric_t pgsql_node_checkin; extern prometheus::simpleapi::counter_metric_t pgsql_node_checkin;
extern prometheus::simpleapi::counter_metric_t pgsql_commit_ticks;
extern prometheus::simpleapi::counter_metric_t db_get_sso_info;
extern prometheus::simpleapi::counter_metric_t redis_mem_notification; extern prometheus::simpleapi::counter_metric_t redis_mem_notification;
extern prometheus::simpleapi::counter_metric_t redis_net_notification; extern prometheus::simpleapi::counter_metric_t redis_net_notification;
extern prometheus::simpleapi::counter_metric_t redis_node_checkin; extern prometheus::simpleapi::counter_metric_t redis_node_checkin;