diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d1537de8b..20ee020b3 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -460,17 +460,39 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } // anonymous namespace -EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPath,const char *dbPath, int listenPort, RedisConfig *rc) : - _startTime(OSUtils::now()), - _listenPort(listenPort), - _node(node), - _ztPath(ztPath), - _path(dbPath), - _sender((NetworkController::Sender *)0), - _db(this), - _ssoExpiryRunning(true), - _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this)), - _rc(rc) +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPath,const char *dbPath, int listenPort, RedisConfig *rc) + : _startTime(OSUtils::now()) + , _listenPort(listenPort) + , _node(node) + , _ztPath(ztPath) + , _path(dbPath) + , _sender((NetworkController::Sender *)0) + , _db(this) + , _ssoExpiryRunning(true) + , _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this)) + , _rc(rc) + , _member_status_lookup{"nc_member_status_lookup",""} + , _member_status_lookup_count{"nc_member_status_lookup_count",""} + , _node_is_online{"nc_node_is_online",""} + , _node_is_online_count{"nc_node_is_online_count",""} + , _get_and_init_member{"nc_get_and_init_member",""} + , _get_and_init_member_count{"nc_get_and_init_member_count",""} + , _have_identity{"nc_have_identity",""} + , _have_identity_count{"nc_have_identity_count",""} + , _determine_auth{"nc_determine_auth",""} + , _determine_auth_count{"nc_determine_auth_count",""} + , _sso_check{"nc_sso_check",""} + , _sso_check_count{"nc_sso_check_count",""} + , _auth_check{"nc_auth_check",""} + , _auth_check_count{"nc_auth_check_count",""} + , _json_schlep{"nc_json_schlep",""} + , _json_schlep_count{"nc_json_schlep_count",""} + , _issue_certificate{"nc_issue_certificate", ""} + , _issue_certificate_count{"nc_issue_certificate_count",""} + , _save_member{"nc_save_member",""} + , _save_member_count{"nc_save_member_count",""} + , _send_netconf{"nc_send_netconf2",""} + , _send_netconf_count{"nc_send_netconf2_count",""} { } @@ -1177,37 +1199,62 @@ void EmbeddedNetworkController::_request( const Dictionary &metaData) { Metrics::network_config_request++; - + auto tid = std::this_thread::get_id(); + std::stringstream ss; ss << tid; + std::string threadID = ss.str(); + auto b1 = _member_status_lookup.Add({{"thread", threadID}}); + auto c1 = _member_status_lookup_count.Add({{"thread", threadID}}); + c1++; + b1.start(); + char nwids[24]; DB::NetworkSummaryInfo ns; json network,member; - if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) { return; + } const int64_t now = OSUtils::now(); if (requestPacketId) { std::lock_guard l(_memberStatus_l); _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; - if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) { return; + } ms.lastRequestTime = now; } + b1.stop(); + auto b2 = _node_is_online.Add({{"thread",threadID}}); + auto c2 = _node_is_online_count.Add({{"thread",threadID}}); + c2++; + b2.start(); _db.nodeIsOnline(nwid,identity.address().toInt(),fromAddr); + b2.stop(); + auto b3 = _get_and_init_member.Add({{"thread", threadID}}); + auto c3 = _get_and_init_member_count.Add({{"thread",threadID}}); + c3++; + b3.start(); Utils::hex(nwid,nwids); _db.get(nwid,network,identity.address().toInt(),member,ns); if ((!network.is_object())||(network.empty())) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0); + b3.stop(); return; } const bool newMember = ((!member.is_object())||(member.empty())); DB::initMember(member); _MemberStatusKey msk(nwid,identity.address().toInt()); + b3.stop(); { + auto b4 = _have_identity.Add({{"thread",threadID}}); + auto c4 = _have_identity_count.Add({{"thread",threadID}}); + c4++; + b4.start(); const std::string haveIdStr(OSUtils::jsonString(member["identity"],"")); if (haveIdStr.length() > 0) { // If we already know this member's identity perform a full compare. This prevents @@ -1216,10 +1263,12 @@ void EmbeddedNetworkController::_request( try { if (Identity(haveIdStr.c_str()) != identity) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + b4.stop(); return; } } catch ( ... ) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + b4.stop(); return; } } else { @@ -1227,6 +1276,7 @@ void EmbeddedNetworkController::_request( char idtmp[1024]; member["identity"] = identity.toString(false,idtmp); } + b4.stop(); } // These are always the same, but make sure they are set @@ -1239,6 +1289,10 @@ void EmbeddedNetworkController::_request( } // Determine whether and how member is authorized + auto b5 = _determine_auth.Add({{"thread",threadID}}); + auto c5 = _determine_auth_count.Add({{"thread",threadID}}); + c5++; + b5.start(); bool authorized = false; bool autoAuthorized = false; json autoAuthCredentialType,autoAuthCredential; @@ -1275,10 +1329,15 @@ void EmbeddedNetworkController::_request( member["lastAuthorizedCredentialType"] = autoAuthCredentialType; member["lastAuthorizedCredential"] = autoAuthCredential; } + b5.stop(); // Should we check SSO Stuff? // If network is configured with SSO, and the member is not marked exempt: yes // Otherwise no, we use standard auth logic. + auto b6 = _sso_check.Add({{"thread",threadID}}); + auto c6 = _sso_check_count.Add({{"thread",threadID}}); + c6++; + b6.start(); AuthInfo info; int64_t authenticationExpiryTime = -1; bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false); @@ -1305,11 +1364,18 @@ void EmbeddedNetworkController::_request( } DB::cleanMember(member); _db.save(member,true); + b6.stop(); return; } } + b6.stop(); + auto b7 = _auth_check.Add({{"thread",threadID}}); + auto c7 = _auth_check_count.Add({{"thread",threadID}}); + c7++; + b7.start(); if (authorized) { + Metrics::nc_authed_request++; // Update version info and meta-data if authorized and if this is a genuine request if (requestPacketId) { const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); @@ -1340,12 +1406,15 @@ void EmbeddedNetworkController::_request( } } } else { + Metrics::nc_unauthed_request++; // If they are not authorized, STOP! DB::cleanMember(member); _db.save(member,true); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + b7.stop(); return; } + b7.stop(); // ------------------------------------------------------------------------- // If we made it this far, they are authorized (and authenticated). @@ -1353,6 +1422,10 @@ void EmbeddedNetworkController::_request( // Default timeout: 15 minutes. Maximum: two hours. Can be specified by an optional field in the network config // if something longer than 15 minutes is desired. Minimum is 5 minutes since shorter than that would be flaky. + auto b8 = _json_schlep.Add({{"thread",threadID}}); + auto c8 = _json_schlep_count.Add({{"thread", threadID}}); + c8++; + b8.start(); int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_DFL_MAX_DELTA; if (network.contains("certificateTimeoutWindowSize")) { credentialtmd = (int64_t)network["certificateTimeoutWindowSize"]; @@ -1420,8 +1493,9 @@ void EmbeddedNetworkController::_request( nc->remoteTraceLevel = (Trace::Level)OSUtils::jsonInt(network["remoteTraceLevel"],0ULL); } - for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) + for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) { nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + } json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -1741,12 +1815,18 @@ void EmbeddedNetworkController::_request( } else { dns = json::object(); } + b8.stop(); // Issue a certificate of ownership for all static IPs + auto b9 = _issue_certificate.Add({{"thread",threadID}}); + auto c9 = _issue_certificate_count.Add({{"thread",threadID}}); + c9++; + b9.start(); if (nc->staticIpCount) { nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); - for(unsigned int i=0;istaticIpCount;++i) + for(unsigned int i=0;istaticIpCount;++i) { nc->certificatesOfOwnership[0].addThing(nc->staticIps[i]); + } nc->certificatesOfOwnership[0].sign(_signingId); nc->certificateOfOwnershipCount = 1; } @@ -1756,19 +1836,33 @@ void EmbeddedNetworkController::_request( nc->com = com; } else { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0); + b9.stop(); return; } + b9.stop(); + auto b10 = _save_member.Add({{"thread",threadID}}); + auto c10 = _save_member_count.Add({{"thread",threadID}}); + c10++; + b10.start(); DB::cleanMember(member); _db.save(member,true); + b10.stop(); + + auto b11 = _send_netconf.Add({{"thread",threadID}}); + auto c11 = _send_netconf_count.Add({{"thread",threadID}}); + c11++; + b11.start(); _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); + b11.stop(); } void EmbeddedNetworkController::_startThreads() { std::lock_guard l(_threads_l); - if (!_threads.empty()) + if (!_threads.empty()) { return; + } const long hwc = std::max((long)std::thread::hardware_concurrency(),(long)1); for(long t=0;t(); - authorization_endpoint = r.at(0)[1].as(); - issuer = r.at(0)[2].as(); - provider = r.at(0)[3].as(); - sso_version = r.at(0)[4].as(); + client_id = r.at(0)[0].as>().value_or(""); + authorization_endpoint = r.at(0)[1].as>().value_or(""); + issuer = r.at(0)[2].as>().value_or(""); + provider = r.at(0)[3].as>().value_or(""); + sso_version = r.at(0)[4].as>().value_or(1); } else if (r.size() > 1) { fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str()); } else { @@ -1596,7 +1597,6 @@ void PostgreSQL::commitThread() } _pool->unborrow(c); c.reset(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); } fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str()); diff --git a/node/Metrics.cpp b/node/Metrics.cpp index 7440a9c31..1f8d7b97c 100644 --- a/node/Metrics.cpp +++ b/node/Metrics.cpp @@ -233,7 +233,13 @@ namespace ZeroTier { { "controller_db_member_change", "counter" }; prometheus::simpleapi::counter_metric_t db_network_change { "controller_db_network_change", "counter" }; + prometheus::simpleapi::counter_metric_t db_get_sso_info + { "controller_db_get_sso_info", "counter" }; + prometheus::simpleapi::counter_metric_t nc_unauthed_request + { "controller_authorized_requests", "counter" }; + prometheus::simpleapi::counter_metric_t nc_authed_request + { "controller_unauthorized_requests", "counter"}; #ifdef ZT_CONTROLLER_USE_LIBPQ // Central Controller Metrics prometheus::simpleapi::counter_metric_t pgsql_mem_notification diff --git a/node/Metrics.hpp b/node/Metrics.hpp index b4677f454..7aa9ff203 100644 --- a/node/Metrics.hpp +++ b/node/Metrics.hpp @@ -136,6 +136,10 @@ namespace ZeroTier { extern prometheus::simpleapi::counter_metric_t db_get_network_list; extern prometheus::simpleapi::counter_metric_t db_member_change; extern prometheus::simpleapi::counter_metric_t db_network_change; + extern prometheus::simpleapi::counter_metric_t db_get_sso_info; + + extern prometheus::simpleapi::counter_metric_t nc_unauthed_request; + extern prometheus::simpleapi::counter_metric_t nc_authed_request; #ifdef ZT_CONTROLLER_USE_LIBPQ // Central Controller Metrics