Think I found the magic spot for outgoing packet metrics

node/Metrics.cpp

@@ -134,6 +134,8 @@ namespace ZeroTier {
         { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "rx"}}) };
         prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in
         { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "rx"}}) };
+        prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in
+        { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "rx"}}) };
 
         // Outgoing Error Counts
         prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out
@@ -150,6 +152,8 @@ namespace ZeroTier {
         { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "tx"}}) };
         prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out
         { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "tx"}}) };
+        prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out
+        { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "tx"}}) };
 
         // Data Sent/Received Metrics
         prometheus::simpleapi::counter_metric_t udp_send

node/Metrics.hpp

@@ -82,6 +82,7 @@ namespace ZeroTier {
         extern prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in;
         extern prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in;
         extern prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in;
+        extern prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in;
 
         // outgoing errors
         extern prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out;
@@ -91,6 +92,7 @@ namespace ZeroTier {
         extern prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out;
         extern prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out;
         extern prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out;
+        extern prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out;
 
 
         // Data Sent/Received Metrics

node/Node.cpp

@@ -769,7 +769,6 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 				break;
 			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED: {
 				//fprintf(stderr, "\n\nGot auth required\n\n");
-
 				break;
 			} 
 
@@ -784,12 +783,15 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			//case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR:
 			default:
 				outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
+				Metrics::pkt_error_obj_not_found_out++;
 				break;
 			case NetworkController::NC_ERROR_ACCESS_DENIED:
 				outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
+				Metrics::pkt_error_network_access_denied_out++;
 				break;
 			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED:
 				outp.append((unsigned char)Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED);
+				Metrics::pkt_error_authentication_required_out++;
 				break;
 		}
 

node/Peer.cpp

@@ -623,72 +623,6 @@ void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t pack
 	if (_localMultipathSupported && _bond) {
 		_bond->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
 	}
-
-	switch (verb) {
-		case Packet::VERB_NOP:
-			Metrics::pkt_nop_out++;
-			break;
-		case Packet::VERB_HELLO:
-			Metrics::pkt_hello_out++;
-			break;
-		case Packet::VERB_ERROR:
-			Metrics::pkt_error_out++;
-			break;
-		case Packet::VERB_OK:
-			Metrics::pkt_ok_out++;
-			break;
-		case Packet::VERB_WHOIS:
-			Metrics::pkt_whois_out++;
-			break;
-		case Packet::VERB_RENDEZVOUS:
-			Metrics::pkt_rendezvous_out++;
-			break;
-		case Packet::VERB_FRAME:
-			Metrics::pkt_frame_out++;
-			break;
-		case Packet::VERB_EXT_FRAME:
-			Metrics::pkt_ext_frame_out++;
-			break;
-		case Packet::VERB_ECHO:
-			Metrics::pkt_echo_out++;
-			break;
-		case Packet::VERB_MULTICAST_LIKE:
-			Metrics::pkt_multicast_like_out++;
-			break;
-		case Packet::VERB_NETWORK_CREDENTIALS:
-			Metrics::pkt_network_credentials_out++;
-			break;
-		case Packet::VERB_NETWORK_CONFIG_REQUEST:
-			Metrics::pkt_network_config_request_out++;
-			break;
-		case Packet::VERB_NETWORK_CONFIG:
-			Metrics::pkt_network_config_out++;
-			break;
-		case Packet::VERB_MULTICAST_GATHER:
-			Metrics::pkt_multicast_gather_out++;
-			break;
-		case Packet::VERB_MULTICAST_FRAME:
-			Metrics::pkt_multicast_frame_out++;
-			break;
-		case Packet::VERB_PUSH_DIRECT_PATHS:
-			Metrics::pkt_push_direct_paths_out++;
-			break;
-		case Packet::VERB_ACK:
-			Metrics::pkt_ack_out++;
-			break;
-		case Packet::VERB_QOS_MEASUREMENT:
-			Metrics::pkt_qos_out++;
-			break;
-		case Packet::VERB_USER_MESSAGE:
-			Metrics::pkt_user_message_out++;
-			break;
-		case Packet::VERB_REMOTE_TRACE:
-			Metrics::pkt_remote_trace_out++;
-			break;
-		case Packet::VERB_PATH_NEGOTIATION_REQUEST:
-			Metrics::pkt_path_negotiation_request_out++;
-			break;
-	}
 }
 
 void Peer::recordIncomingInvalidPacket(const SharedPtr<Path>& path)

node/Switch.cpp

@@ -850,8 +850,10 @@ void Switch::removeNetworkQoSControlBlock(uint64_t nwid)
 void Switch::send(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 {
 	const Address dest(packet.destination());
-	if (dest == RR->identity.address())
+	if (dest == RR->identity.address()) {
 		return;
+	}
+	_recordOutgoingPacketMetrics(packet);
 	if (!_trySend(tPtr,packet,encrypt,flowId)) {
 		{
 			Mutex::Lock _l(_txQueue_m);
@@ -1080,4 +1082,72 @@ void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path
 	}
 }
 
+void Switch::_recordOutgoingPacketMetrics(const Packet &p) {
+	switch (p.verb()) {
+		case Packet::VERB_NOP:
+			Metrics::pkt_nop_out++;
+			break;
+		case Packet::VERB_HELLO:
+			Metrics::pkt_hello_out++;
+			break;
+		case Packet::VERB_ERROR:
+			Metrics::pkt_error_out++;
+			break;
+		case Packet::VERB_OK:
+			Metrics::pkt_ok_out++;
+			break;
+		case Packet::VERB_WHOIS:
+			Metrics::pkt_whois_out++;
+			break;
+		case Packet::VERB_RENDEZVOUS:
+			Metrics::pkt_rendezvous_out++;
+			break;
+		case Packet::VERB_FRAME:
+			Metrics::pkt_frame_out++;
+			break;
+		case Packet::VERB_EXT_FRAME:
+			Metrics::pkt_ext_frame_out++;
+			break;
+		case Packet::VERB_ECHO:
+			Metrics::pkt_echo_out++;
+			break;
+		case Packet::VERB_MULTICAST_LIKE:
+			Metrics::pkt_multicast_like_out++;
+			break;
+		case Packet::VERB_NETWORK_CREDENTIALS:
+			Metrics::pkt_network_credentials_out++;
+			break;
+		case Packet::VERB_NETWORK_CONFIG_REQUEST:
+			Metrics::pkt_network_config_request_out++;
+			break;
+		case Packet::VERB_NETWORK_CONFIG:
+			Metrics::pkt_network_config_out++;
+			break;
+		case Packet::VERB_MULTICAST_GATHER:
+			Metrics::pkt_multicast_gather_out++;
+			break;
+		case Packet::VERB_MULTICAST_FRAME:
+			Metrics::pkt_multicast_frame_out++;
+			break;
+		case Packet::VERB_PUSH_DIRECT_PATHS:
+			Metrics::pkt_push_direct_paths_out++;
+			break;
+		case Packet::VERB_ACK:
+			Metrics::pkt_ack_out++;
+			break;
+		case Packet::VERB_QOS_MEASUREMENT:
+			Metrics::pkt_qos_out++;
+			break;
+		case Packet::VERB_USER_MESSAGE:
+			Metrics::pkt_user_message_out++;
+			break;
+		case Packet::VERB_REMOTE_TRACE:
+			Metrics::pkt_remote_trace_out++;
+			break;
+		case Packet::VERB_PATH_NEGOTIATION_REQUEST:
+			Metrics::pkt_path_negotiation_request_out++;
+			break;
+	}
+}
+
 } // namespace ZeroTier

node/Switch.hpp

@@ -208,6 +208,7 @@ private:
 	bool _shouldUnite(const int64_t now,const Address &source,const Address &destination);
 	bool _trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId = ZT_QOS_NO_FLOW); // packet is modified if return is true
 	void _sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId);
+	void _recordOutgoingPacketMetrics(const Packet &p);
 
 	const RuntimeEnvironment *const RR;
 	int64_t _lastBeaconResponse;