Merge pull request #2427 from SwarmboticsAI/fix-active-backup-selection

Fix active backup link selection

.clangd

@@ -0,0 +1,6 @@
+CompileFlags:
+  Add:
+    - "-std=c++17"
+    - "-I../ext"
+    - "-I../ext/prometheus-cpp-lite-1.0/core/include"
+    - "-I../ext/prometheus-cpp-lite-1.0/simpleapi/include"

.github/workflows/build.yml

@@ -1,4 +1,7 @@
-on: [ push ]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 
 jobs:
   build_ubuntu:
@@ -9,13 +12,12 @@ jobs:
         git config --global core.autocrlf input
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
-        target: x86_64-unknown-linux-gnu
+        targets: x86_64-unknown-linux-gnu
-        override: true
         components: rustfmt, clippy
 
     - name: Set up cargo cache
@@ -33,6 +35,14 @@ jobs:
       run: |
         make selftest
         ./zerotier-selftest
+    - name: 'Tar files' # keeps permissions (execute)
+      run: tar -cvf zerotier-one.tar zerotier-one
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-ubuntu-x64
+        path: zerotier-one.tar
+        retention-days: 7
 
   build_macos:
     runs-on: macos-latest
@@ -42,13 +52,18 @@ jobs:
         git config --global core.autocrlf input
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - name: Install Rust
+    - name: Install Rust aarch64
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
         target: aarch64-apple-darwin
-        override: true
+        components: rustfmt, clippy
+    - name: Install Rust x86_64
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: stable
+        target: x86_64-apple-darwin
         components: rustfmt, clippy
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
@@ -58,13 +73,21 @@ jobs:
         shared-key: ${{ runner.os }}-cargo-
         workspaces: |
           rustybits/
-
     - name: make
       run: make
     - name: selftest
       run: |
         make selftest
         ./zerotier-selftest
+    - name: 'Tar files' # keeps permissions (execute)
+      run: tar -cvf zerotier-one.tar zerotier-one
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-mac
+        path: zerotier-one.tar
+        retention-days: 7
+
 
   build_windows:
     runs-on: windows-latest
@@ -74,13 +97,12 @@ jobs:
         git config --global core.autocrlf true
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
         target: aarch64-apple-darwin
-        override: true
         components: rustfmt, clippy
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
@@ -92,7 +114,13 @@ jobs:
           rustybits/
 
     - name: setup msbuild
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@v2
     - name: msbuild
       run: |
         msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release  /property:Platform=x64 /t:ZeroTierOne
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-windows
+        path: windows/Build
+        retention-days: 7

.github/workflows/validate.yml

@@ -1,4 +1,5 @@
 on:
+  pull_request:
   push:
   workflow_dispatch:
 
@@ -44,7 +45,7 @@ jobs:
         sudo ./.github/workflows/validate-linux.sh
 
     - name: Archive test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{github.sha}}-test-results
         path: "*test-results*"

.gitignore

@@ -124,6 +124,7 @@ attic/world/mkworld
 workspace/
 workspace2/
 zeroidc/target/
+tcp-proxy/target
 
 #snapcraft specifics
 /parts/

Dockerfile.release

@@ -1,6 +1,6 @@
 # vim: ft=dockerfile
 
-FROM debian:bullseye
+FROM debian:bookworm
 
 ARG VERSION
 
@@ -9,9 +9,9 @@ RUN mkdir -p /usr/share/zerotier && \
     curl -o /usr/share/zerotier/tmp.asc "https://download.zerotier.com/contact%40zerotier.com.gpg" && \
     gpg --no-default-keyring --keyring /usr/share/zerotier/zerotier.gpg --import /usr/share/zerotier/tmp.asc && \
     rm -f /usr/share/zerotier/tmp.asc && \
-    echo "deb [signed-by=/usr/share/zerotier/zerotier.gpg] http://download.zerotier.com/debian/bullseye bullseye main" > /etc/apt/sources.list.d/zerotier.list
+    echo "deb [signed-by=/usr/share/zerotier/zerotier.gpg] http://download.zerotier.com/debian/bookworm bookworm main" > /etc/apt/sources.list.d/zerotier.list
 
-RUN apt-get update -qq && apt-get install zerotier-one=${VERSION} curl iproute2 net-tools iputils-ping openssl libssl1.1 -y
+RUN apt-get update -qq && apt-get install zerotier-one=${VERSION} curl iproute2 net-tools iputils-ping openssl libssl3 -y
 RUN rm -rf /var/lib/zerotier-one
 
 COPY entrypoint.sh.release /entrypoint.sh

Makefile

@@ -31,3 +31,6 @@ drone:
 	@echo "rendering .drone.yaml from .drone.jsonnet"
 	drone jsonnet --format --stream
 	drone sign zerotier/ZeroTierOne --save
+
+clang-format:
+	find node osdep service tcp-proxy controller -iname '*.cpp' -o -iname '*.hpp' | xargs clang-format -i

OFFICIAL-RELEASE-STEPS.md

@@ -14,6 +14,7 @@ The version must be incremented in all of the following files:
     /debian/changelog
     /ext/installfiles/mac/ZeroTier One.pkgproj
     /ext/installfiles/windows/ZeroTier One.aip
+  ../DesktopUI/mac-app-template/ZeroTier.app/Contents/Info.plist
 
 The final .AIP file can only be edited on Windows with [Advanced Installer Enterprise](http://www.advancedinstaller.com/). In addition to incrementing the version be sure that a new product code is generated. (The "upgrade code" GUID on the other hand must never change.)
 

README.docker.md

@@ -64,6 +64,7 @@ You can control a few settings including the identity used and the authtoken use
 - `ZEROTIER_API_SECRET`: replaces the `authtoken.secret` before booting and allows you to manage the control socket's authentication key.
 - `ZEROTIER_IDENTITY_PUBLIC`: the `identity.public` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
 - `ZEROTIER_IDENTITY_SECRET`: the `identity.secret` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
+- `ZEROTIER_LOCAL_CONF`: Sets the the `local.conf` file content for zerotier-one
 
 ### Tips
 

README.md

@@ -58,7 +58,7 @@ To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU
    - Xcode command line tools for macOS 10.13 or newer are required.
    - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Linux**
-   - The minimum compiler versions required are GCC/G++ 4.9.3 or CLANG/CLANG++ 3.4.2. (Install `clang` on CentOS 7 as G++ is too old.)
+   - The minimum compiler versions required are GCC/G++ 8.x or CLANG/CLANG++ 5.x.
    - Linux makefiles automatically detect and prefer clang/clang++ if present as it produces smaller and slightly faster binaries in most cases. You can override by supplying CC and CXX variables on the make command line.
    - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Windows**

RELEASE-NOTES.md

@@ -1,6 +1,29 @@
 ZeroTier Release Notes
 ======
 
+# 2024-10-23 -- Version 1.14.2
+
+  * Fix for missing entitlement on macOS Sequoia.
+  * Fix for a problem correctly parsing local.conf to enable low bandwidth mode.
+  * Increment versions of some dependent libraries.
+  * Other fixes.
+
+# 2024-09-12 -- Version 1.14.1
+
+  * Multithreaded packet I/O support! Currently this is just for Linux and must
+    be enabled in local.conf. It will likely make the largest difference on small
+    multi-core devices where CPU is a bottleneck and high throughput is desired.
+    It may be enabled by default in the future but we want it to be thoroughly
+    tested. It's a little harder than it seems at first glance due to the need
+    to keep packets in sequence and balance load.
+  * Several multipath bug fixes.
+  * Updated the versions on a number of libraries related to OIDC support and HTTP.
+  * MacOS .app now shows the correct version in its Info.plist manifest.
+  * Sanitize MAC addresses in JSON format rules parser.
+  * Some basic information about the platform (OS, CPU architecture) is now reported
+    to network controllers when networks are joined so it can be displayed to
+    network admins and in the future used in policy checking and inventory operations.
+
 # 2024-05-02 -- Version 1.14.0
 
   * Linux I/O performance improvements under heavy load

controller/CV1.hpp

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_CV1_HPP
+#define ZT_CONTROLLER_CV1_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "../node/Metrics.hpp"
+#include "ConnectionPool.hpp"
+#include "PostgreSQL.hpp"
+
+#include <memory>
+#include <pqxx/pqxx>
+#include <redis++/redis++.h>
+
+namespace smeeclient {
+struct SmeeClient;
+}
+
+namespace ZeroTier {
+
+struct RedisConfig;
+
+/**
+ * A controller database driver that talks to  PostgreSQL
+ *
+ * This is for use with ZeroTier Central.  Others are free to build and use it
+ * but be aware that we might change it at any time.
+ */
+class CV1 : public DB {
+  public:
+	CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc);
+	virtual ~CV1();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json& record, bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL);
+
+	virtual bool ready()
+	{
+		return _ready == 2;
+	}
+
+  protected:
+	struct _PairHasher {
+		inline std::size_t operator()(const std::pair<uint64_t, uint64_t>& p) const
+		{
+			return (std::size_t)(p.first ^ p.second);
+		}
+	};
+	virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners)
+	{
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners)
+	{
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+  private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void _membersWatcher_Postgres();
+	void networksDbWatcher();
+	void _networksWatcher_Postgres();
+
+	void _membersWatcher_Redis();
+	void _networksWatcher_Redis();
+
+	void commitThread();
+	void onlineNotificationThread();
+	void onlineNotification_Postgres();
+	void onlineNotification_Redis();
+	uint64_t _doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher>& lastOnline);
+
+	void configureSmee();
+	void notifyNewMember(const std::string& networkID, const std::string& memberID);
+
+	enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 };
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+
+	RedisConfig* _rc;
+	std::shared_ptr<sw::redis::Redis> _redis;
+	std::shared_ptr<sw::redis::RedisCluster> _cluster;
+	bool _redisMemberStatus;
+
+	smeeclient::SmeeClient* _smee;
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_CONTROLLER_CV1_HPP
+
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/CV2.hpp

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c)2025 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_CV2_HPP
+#define ZT_CONTROLLER_CV2_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "../node/Metrics.hpp"
+#include "ConnectionPool.hpp"
+#include "PostgreSQL.hpp"
+
+#include <memory>
+#include <pqxx/pqxx>
+#include <redis++/redis++.h>
+
+namespace ZeroTier {
+
+class CV2 : public DB {
+  public:
+	CV2(const Identity& myId, const char* path, int listenPort);
+	virtual ~CV2();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json& record, bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL);
+
+	virtual bool ready()
+	{
+		return _ready == 2;
+	}
+
+  protected:
+	struct _PairHasher {
+		inline std::size_t operator()(const std::pair<uint64_t, uint64_t>& p) const
+		{
+			return (std::size_t)(p.first ^ p.second);
+		}
+	};
+	virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners)
+	{
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners)
+	{
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+  private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void networksDbWatcher();
+
+	void commitThread();
+	void onlineNotificationThread();
+
+	// void notifyNewMember(const std::string &networkID, const std::string &memberID);
+
+	enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 };
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_CONTROLLER_CV2_HPP
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/ConnectionPool.hpp

@@ -14,7 +14,6 @@
 #ifndef ZT_CONNECTION_POOL_H_
 #define ZT_CONNECTION_POOL_H_
 
-
 #ifndef _DEBUG
 #define _DEBUG(x)
 #endif
@@ -22,21 +21,21 @@
 #include "../node/Metrics.hpp"
 
 #include <deque>
-#include <set>
+#include <exception>
 #include <memory>
 #include <mutex>
-#include <exception>
+#include <set>
 #include <string>
 
 namespace ZeroTier {
 
 struct ConnectionUnavailable : std::exception {
-    char const* what() const throw() {
+	char const* what() const throw()
+	{
 		return "Unable to allocate connection";
 	};
 };
 
-
 class Connection {
   public:
 	virtual ~Connection() {};
@@ -53,13 +52,9 @@ struct ConnectionPoolStats {
 	size_t borrowed_size;
 };
 
-template<class T>
+template <class T> class ConnectionPool {
-class ConnectionPool {
   public:
-    ConnectionPool(size_t max_pool_size, size_t min_pool_size, std::shared_ptr<ConnectionFactory> factory)
+	ConnectionPool(size_t max_pool_size, size_t min_pool_size, std::shared_ptr<ConnectionFactory> factory) : m_maxPoolSize(max_pool_size), m_minPoolSize(min_pool_size), m_factory(factory)
-        : m_maxPoolSize(max_pool_size)
-        , m_minPoolSize(min_pool_size)
-        , m_factory(factory)
 	{
 		Metrics::max_pool_size += max_pool_size;
 		Metrics::min_pool_size += min_pool_size;
@@ -69,7 +64,8 @@ public:
 		}
 	};
 
-    ConnectionPoolStats get_stats() {
+	ConnectionPoolStats get_stats()
+	{
 		std::unique_lock<std::mutex> lock(m_poolMutex);
 
 		ConnectionPoolStats stats;
@@ -79,8 +75,7 @@ public:
 		return stats;
 	};
 
-    ~ConnectionPool() {
+	~ConnectionPool() {};
-    };
 
 	/**
 	 * Borrow
@@ -90,7 +85,8 @@ public:
 	 * When done, either (a) call unborrow() to return it, or (b) (if it's bad) just let it go out of scope.  This will cause it to automatically be replaced.
 	 * @retval a shared_ptr to the connection object
 	 */
-    std::shared_ptr<T> borrow() {
+	std::shared_ptr<T> borrow()
+	{
 		std::unique_lock<std::mutex> l(m_poolMutex);
 
 		while ((m_pool.size() + m_borrowed.size()) < m_minPoolSize) {
@@ -100,18 +96,19 @@ public:
 		}
 
 		if (m_pool.size() == 0) {
-            
 			if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
 				try {
 					std::shared_ptr<Connection> conn = m_factory->create();
 					m_borrowed.insert(conn);
 					Metrics::pool_in_use++;
 					return std::static_pointer_cast<T>(conn);
-                } catch (std::exception &e) {
+				}
+				catch (std::exception& e) {
 					Metrics::pool_errors++;
 					throw ConnectionUnavailable();
 				}
-            } else {
+			}
+			else {
 				for (auto it = m_borrowed.begin(); it != m_borrowed.end(); ++it) {
 					if ((*it).unique()) {
 						// This connection has been abandoned! Destroy it and create a new connection
@@ -122,7 +119,8 @@ public:
 							m_borrowed.erase(it);
 							m_borrowed.insert(conn);
 							return std::static_pointer_cast<T>(conn);
-                        } catch(std::exception& e) {
+						}
+						catch (std::exception& e) {
 							// Error creating a replacement connection
 							Metrics::pool_errors++;
 							throw ConnectionUnavailable();
@@ -151,7 +149,8 @@ public:
 	 * Only call this if you are returning a working connection.  If the connection was bad, just let it go out of scope (so the connection manager can replace it).
 	 * @param the connection
 	 */
-    void unborrow(std::shared_ptr<T> conn) {
+	void unborrow(std::shared_ptr<T> conn)
+	{
 		// Lock
 		std::unique_lock<std::mutex> lock(m_poolMutex);
 		m_borrowed.erase(conn);
@@ -161,6 +160,7 @@ public:
 			m_pool.push_back(conn);
 		}
 	};
+
   protected:
 	size_t m_maxPoolSize;
 	size_t m_minPoolSize;
@@ -170,6 +170,6 @@ protected:
 	std::mutex m_poolMutex;
 };
 
-}
+}	// namespace ZeroTier
 
 #endif

controller/CtlUtil.cpp

@@ -0,0 +1,64 @@
+#include "CtlUtil.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#include <iomanip>
+#include <sstream>
+
+namespace ZeroTier {
+
+const char* _timestr()
+{
+	time_t t = time(0);
+	char* ts = ctime(&t);
+	char* p = ts;
+	if (! p)
+		return "";
+	while (*p) {
+		if (*p == '\n') {
+			*p = (char)0;
+			break;
+		}
+		++p;
+	}
+	return ts;
+}
+
+std::vector<std::string> split(std::string str, char delim)
+{
+	std::istringstream iss(str);
+	std::vector<std::string> tokens;
+	std::string item;
+	while (std::getline(iss, item, delim)) {
+		tokens.push_back(item);
+	}
+	return tokens;
+}
+
+std::string url_encode(const std::string& value)
+{
+	std::ostringstream escaped;
+	escaped.fill('0');
+	escaped << std::hex;
+
+	for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
+		std::string::value_type c = (*i);
+
+		// Keep alphanumeric and other accepted characters intact
+		if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+			escaped << c;
+			continue;
+		}
+
+		// Any other characters are percent-encoded
+		escaped << std::uppercase;
+		escaped << '%' << std::setw(2) << int((unsigned char)c);
+		escaped << std::nouppercase;
+	}
+
+	return escaped.str();
+}
+
+}	// namespace ZeroTier
+
+#endif

controller/CtlUtil.hpp

@@ -0,0 +1,16 @@
+#ifndef ZT_CTLUTIL_HPP
+#define ZT_CTLUTIL_HPP
+
+#include <string>
+#include <vector>
+
+namespace ZeroTier {
+
+const char* _timestr();
+
+std::vector<std::string> split(std::string str, char delim);
+
+std::string url_encode(const std::string& value);
+}	// namespace ZeroTier
+
+#endif	 // namespace ZeroTier

controller/DB.cpp

@@ -12,11 +12,12 @@
 /****/
 
 #include "DB.hpp"
-#include "EmbeddedNetworkController.hpp"
-#include "../node/Metrics.hpp"
 
-#include <chrono>
+#include "../node/Metrics.hpp"
+#include "EmbeddedNetworkController.hpp"
+
 #include <algorithm>
+#include <chrono>
 #include <stdexcept>
 
 using json = nlohmann::json;
@@ -25,60 +26,96 @@ namespace ZeroTier {
 
 void DB::initNetwork(nlohmann::json& network)
 {
-	if (!network.count("private")) network["private"] = true;
+	if (! network.count("private"))
-	if (!network.count("creationTime")) network["creationTime"] = OSUtils::now();
+		network["private"] = true;
-	if (!network.count("name")) network["name"] = "";
+	if (! network.count("creationTime"))
-	if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32;
+		network["creationTime"] = OSUtils::now();
-	if (!network.count("enableBroadcast")) network["enableBroadcast"] = true;
+	if (! network.count("name"))
-	if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}};
+		network["name"] = "";
-	if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}};
+	if (! network.count("multicastLimit"))
-	if (!network.count("authTokens")) network["authTokens"] = {{}};
+		network["multicastLimit"] = (uint64_t)32;
-	if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array();
+	if (! network.count("enableBroadcast"))
-	if (!network.count("tags")) network["tags"] = nlohmann::json::array();
+		network["enableBroadcast"] = true;
-	if (!network.count("routes")) network["routes"] = nlohmann::json::array();
+	if (! network.count("v4AssignMode"))
-	if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array();
+		network["v4AssignMode"] = { { "zt", false } };
-	if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU;
+	if (! network.count("v6AssignMode"))
-	if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json();
+		network["v6AssignMode"] = { { "rfc4193", false }, { "zt", false }, { "6plane", false } };
-	if (!network.count("removeTraceLevel")) network["remoteTraceLevel"] = 0;
+	if (! network.count("authTokens"))
-	if (!network.count("rulesSource")) network["rulesSource"] = "";
+		network["authTokens"] = { {} };
+	if (! network.count("capabilities"))
+		network["capabilities"] = nlohmann::json::array();
+	if (! network.count("tags"))
+		network["tags"] = nlohmann::json::array();
+	if (! network.count("routes"))
+		network["routes"] = nlohmann::json::array();
+	if (! network.count("ipAssignmentPools"))
+		network["ipAssignmentPools"] = nlohmann::json::array();
+	if (! network.count("mtu"))
+		network["mtu"] = ZT_DEFAULT_MTU;
+	if (! network.count("remoteTraceTarget"))
+		network["remoteTraceTarget"] = nlohmann::json();
+	if (! network.count("removeTraceLevel"))
+		network["remoteTraceLevel"] = 0;
+	if (! network.count("rulesSource"))
+		network["rulesSource"] = "";
 	if (! network.count("rules")) {
 		// If unspecified, rules are set to allow anything and behave like a flat L2 segment
-		network["rules"] = {{
+		network["rules"] = { { { "not", false }, { "or", false }, { "type", "ACTION_ACCEPT" } } };
-			{ "not",false },
-			{ "or", false },
-			{ "type","ACTION_ACCEPT" }
-		}};
 	}
-	if (!network.count("dns")) network["dns"] = nlohmann::json::array();
+	if (! network.count("dns"))
-	if (!network.count("ssoEnabled")) network["ssoEnabled"] = false;
+		network["dns"] = nlohmann::json::array();
-	if (!network.count("clientId")) network["clientId"] = "";
+	if (! network.count("ssoEnabled"))
-	if (!network.count("authorizationEndpoint")) network["authorizationEndpoint"] = "";
+		network["ssoEnabled"] = false;
+	if (! network.count("clientId"))
+		network["clientId"] = "";
+	if (! network.count("authorizationEndpoint"))
+		network["authorizationEndpoint"] = "";
 
 	network["objtype"] = "network";
 }
 
 void DB::initMember(nlohmann::json& member)
 {
-	if (!member.count("authorized")) member["authorized"] = false;
+	if (! member.count("authorized"))
-	if (!member.count("ssoExempt")) member["ssoExempt"] = false;
+		member["authorized"] = false;
-	if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
+	if (! member.count("ssoExempt"))
-	if (!member.count("activeBridge")) member["activeBridge"] = false;
+		member["ssoExempt"] = false;
-	if (!member.count("tags")) member["tags"] = nlohmann::json::array();
+	if (! member.count("ipAssignments"))
-	if (!member.count("capabilities")) member["capabilities"] = nlohmann::json::array();
+		member["ipAssignments"] = nlohmann::json::array();
-	if (!member.count("creationTime")) member["creationTime"] = OSUtils::now();
+	if (! member.count("activeBridge"))
-	if (!member.count("noAutoAssignIps")) member["noAutoAssignIps"] = false;
+		member["activeBridge"] = false;
-	if (!member.count("revision")) member["revision"] = 0ULL;
+	if (! member.count("tags"))
-	if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL;
+		member["tags"] = nlohmann::json::array();
-	if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
+	if (! member.count("capabilities"))
-	if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json();
+		member["capabilities"] = nlohmann::json::array();
-	if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json();
+	if (! member.count("creationTime"))
-	if (!member.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = 0LL;
+		member["creationTime"] = OSUtils::now();
-	if (!member.count("vMajor")) member["vMajor"] = -1;
+	if (! member.count("noAutoAssignIps"))
-	if (!member.count("vMinor")) member["vMinor"] = -1;
+		member["noAutoAssignIps"] = false;
-	if (!member.count("vRev")) member["vRev"] = -1;
+	if (! member.count("revision"))
-	if (!member.count("vProto")) member["vProto"] = -1;
+		member["revision"] = 0ULL;
-	if (!member.count("remoteTraceTarget")) member["remoteTraceTarget"] = nlohmann::json();
+	if (! member.count("lastDeauthorizedTime"))
-	if (!member.count("removeTraceLevel")) member["remoteTraceLevel"] = 0;
+		member["lastDeauthorizedTime"] = 0ULL;
+	if (! member.count("lastAuthorizedTime"))
+		member["lastAuthorizedTime"] = 0ULL;
+	if (! member.count("lastAuthorizedCredentialType"))
+		member["lastAuthorizedCredentialType"] = nlohmann::json();
+	if (! member.count("lastAuthorizedCredential"))
+		member["lastAuthorizedCredential"] = nlohmann::json();
+	if (! member.count("authenticationExpiryTime"))
+		member["authenticationExpiryTime"] = 0LL;
+	if (! member.count("vMajor"))
+		member["vMajor"] = -1;
+	if (! member.count("vMinor"))
+		member["vMinor"] = -1;
+	if (! member.count("vRev"))
+		member["vRev"] = -1;
+	if (! member.count("vProto"))
+		member["vProto"] = -1;
+	if (! member.count("remoteTraceTarget"))
+		member["remoteTraceTarget"] = nlohmann::json();
+	if (! member.count("removeTraceLevel"))
+		member["remoteTraceLevel"] = 0;
 	member["objtype"] = "member";
 }
 
@@ -102,8 +139,12 @@ void DB::cleanMember(nlohmann::json &member)
 	member.erase("authenticationClientID");	  // computed
 }
 
-DB::DB() {}
+DB::DB()
-DB::~DB() {}
+{
+}
+DB::~DB()
+{
+}
 
 bool DB::get(const uint64_t networkId, nlohmann::json& network)
 {
@@ -299,7 +340,8 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool no
 				(*i)->onNetworkMemberUpdate(this, networkId, memberId, memberConfig);
 			}
 		}
-	} else if (memberId) {
+	}
+	else if (memberId) {
 		if (nw) {
 			std::unique_lock<std::shared_mutex> l(nw->lock);
 			nw->members.erase(memberId);
@@ -320,16 +362,19 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool no
 		if (networkId != 0 && memberId != 0 && old.is_object() && ! memberConfig.is_object()) {
 			// member delete
 			Metrics::member_count--;
-		} else if (networkId != 0 && memberId != 0 && !old.is_object() && memberConfig.is_object()) {
+		}
+		else if (networkId != 0 && memberId != 0 && ! old.is_object() && memberConfig.is_object()) {
 			// new member
 			Metrics::member_count++;
 		}
 
 		if (! wasAuth && isAuth) {
 			Metrics::member_auths++;
-		} else if (wasAuth && !isAuth) {
+		}
+		else if (wasAuth && ! isAuth) {
 			Metrics::member_deauths++;
-		} else {
+		}
+		else {
 			Metrics::member_changes++;
 		}
 	}
@@ -348,9 +393,11 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool
 	if (notifyListeners) {
 		if (old.is_object() && old.contains("id") && networkConfig.is_object() && networkConfig.contains("id")) {
 			Metrics::network_changes++;
-		} else if (!old.is_object() && networkConfig.is_object() && networkConfig.contains("id")) {
+		}
+		else if (! old.is_object() && networkConfig.is_object() && networkConfig.contains("id")) {
 			Metrics::network_count++;
-		} else if (old.is_object() && old.contains("id") && !networkConfig.is_object()) {
+		}
+		else if (old.is_object() && old.contains("id") && ! networkConfig.is_object()) {
 			Metrics::network_count--;
 		}
 	}
@@ -378,10 +425,30 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool
 				}
 			}
 		}
-	} else if (old.is_object()) {
+	}
+	else if (old.is_object()) {
 		const std::string ids = old["id"];
 		const uint64_t networkId = Utils::hexStrToU64(ids.c_str());
 		if (networkId) {
+			try {
+				// deauth all members on the network
+				nlohmann::json network;
+				std::vector<nlohmann::json> members;
+				this->get(networkId, network, members);
+				for (auto i = members.begin(); i != members.end(); ++i) {
+					const std::string nodeID = (*i)["id"];
+					const uint64_t memberId = Utils::hexStrToU64(nodeID.c_str());
+					std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+					for (auto j = _changeListeners.begin(); j != _changeListeners.end(); ++j) {
+						(*j)->onNetworkMemberDeauthorize(this, networkId, memberId);
+					}
+				}
+			}
+			catch (std::exception& e) {
+				std::cerr << "Error deauthorizing members on network delete: " << e.what() << std::endl;
+			}
+
+			// delete the network
 			std::unique_lock<std::shared_mutex> l(_networks_l);
 			_networks.erase(networkId);
 		}

controller/DB.hpp

@@ -19,44 +19,31 @@
 #include "../node/Constants.hpp"
 #include "../node/Identity.hpp"
 #include "../node/InetAddress.hpp"
-#include "../osdep/OSUtils.hpp"
 #include "../osdep/BlockingQueue.hpp"
+#include "../osdep/OSUtils.hpp"
 
+#include <atomic>
+#include <map>
 #include <memory>
+#include <nlohmann/json.hpp>
+#include <prometheus/simpleapi.h>
+#include <set>
+#include <shared_mutex>
 #include <string>
 #include <thread>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
-#include <atomic>
-#include <shared_mutex>
-#include <set>
-#include <map>
-
-#include <nlohmann/json.hpp>
-
-#include <prometheus/simpleapi.h>
 
 #define ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE 25000
 
-namespace ZeroTier
+namespace ZeroTier {
-{
 
-struct AuthInfo
+struct AuthInfo {
-{
   public:
-	AuthInfo() 
+	AuthInfo() : enabled(false), version(0), authenticationURL(), authenticationExpiryTime(0), issuerURL(), centralAuthURL(), ssoNonce(), ssoState(), ssoClientID(), ssoProvider("default")
-	: enabled(false)
+	{
-	, version(0)
+	}
-	, authenticationURL()
-	, authenticationExpiryTime(0)
-	, issuerURL()
-	, centralAuthURL()
-	, ssoNonce()
-	, ssoState()
-	, ssoClientID()
-	, ssoProvider("default")
-	{}
 
 	bool enabled;
 	uint64_t version;
@@ -73,22 +60,35 @@ public:
 /**
  * Base class with common infrastructure for all controller DB implementations
  */
-class DB
+class DB {
-{
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+	friend class MemberNotificationReceiver;
+	friend class NetworkNotificationReceiver;
+#endif
   public:
-	class ChangeListener
+	class ChangeListener {
-	{
 	  public:
-		ChangeListener() {}
+		ChangeListener()
-		virtual ~ChangeListener() {}
+		{
-		virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network) {}
+		}
-		virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member) {}
+		virtual ~ChangeListener()
-		virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId) {}
+		{
+		}
+		virtual void onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network)
+		{
+		}
+		virtual void onNetworkMemberUpdate(const void* db, uint64_t networkId, uint64_t memberId, const nlohmann::json& member)
+		{
+		}
+		virtual void onNetworkMemberDeauthorize(const void* db, uint64_t networkId, uint64_t memberId)
+		{
+		}
 	};
 
-	struct NetworkSummaryInfo
+	struct NetworkSummaryInfo {
+		NetworkSummaryInfo() : authorizedMemberCount(0), totalMemberCount(0), mostRecentDeauthTime(0)
 		{
-		NetworkSummaryInfo() : authorizedMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {}
+		}
 		std::vector<Address> activeBridges;
 		std::vector<InetAddress> allocatedIps;
 		unsigned long authorizedMemberCount;
@@ -120,8 +120,7 @@ public:
 
 	void networks(std::set<uint64_t>& networks);
 
-	template<typename F>
+	template <typename F> inline void each(F f)
-	inline void each(F f)
 	{
 		nlohmann::json nullJson;
 		std::unique_lock<std::shared_mutex> lck(_networks_l);
@@ -137,8 +136,12 @@ public:
 	virtual void eraseNetwork(const uint64_t networkId) = 0;
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId) = 0;
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress) = 0;
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch) = 0;
 
-	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL) { return AuthInfo(); }
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL)
+	{
+		return AuthInfo();
+	}
 
 	inline void addListener(DB::ChangeListener* const listener)
 	{
@@ -169,9 +172,10 @@ protected:
 		return false;
 	}
 
-	struct _Network
+	struct _Network {
+		_Network() : mostRecentDeauthTime(0)
 		{
-		_Network() : mostRecentDeauthTime(0) {}
+		}
 		nlohmann::json config;
 		std::unordered_map<uint64_t, nlohmann::json> members;
 		std::unordered_set<uint64_t> activeBridgeMembers;

controller/DBMirrorSet.cpp

@@ -15,12 +15,7 @@
 
 namespace ZeroTier {
 
-DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener)
+DBMirrorSet::DBMirrorSet(DB::ChangeListener* listener) : _listener(listener), _running(true), _syncCheckerThread(), _dbs(), _dbs_l()
-	: _listener(listener)
-	, _running(true)
-	, _syncCheckerThread()
-	, _dbs()
-	, _dbs_l()
 {
 	_syncCheckerThread = std::thread([this]() {
 		for (;;) {
@@ -52,7 +47,8 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener)
 										}
 									}
 								}
-							} else if (member.is_object()) {
+							}
+							else if (member.is_object()) {
 								for (auto db2 = dbs.begin(); db2 != dbs.end(); ++db2) {
 									if (db->get() != db2->get()) {
 										nlohmann::json nw2, m2;
@@ -64,7 +60,9 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener)
 								}
 							}
 						}
-					} catch ( ... ) {} // skip entries that generate JSON errors
+					}
+					catch (...) {
+					}	// skip entries that generate JSON errors
 				});
 			}
 		}
@@ -181,7 +179,8 @@ bool DBMirrorSet::save(nlohmann::json &record,bool notifyListeners)
 				return true;
 		}
 		return false;
-	} else {
+	}
+	else {
 		bool modified = false;
 		for (auto d = dbs.begin(); d != dbs.end(); ++d) {
 			modified |= (*d)->save(record, false);
@@ -206,14 +205,19 @@ void DBMirrorSet::eraseMember(const uint64_t networkId,const uint64_t memberId)
 	}
 }
 
-void DBMirrorSet::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	std::shared_lock<std::shared_mutex> l(_dbs_l);
 	for (auto d = _dbs.begin(); d != _dbs.end(); ++d) {
-		(*d)->nodeIsOnline(networkId,memberId,physicalAddress);
+		(*d)->nodeIsOnline(networkId, memberId, physicalAddress, osArch);
 	}
 }
 
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 void DBMirrorSet::onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network)
 {
 	nlohmann::json record(network);

controller/DBMirrorSet.hpp

@@ -16,16 +16,15 @@
 
 #include "DB.hpp"
 
-#include <vector>
 #include <memory>
-#include <shared_mutex>
 #include <set>
+#include <shared_mutex>
 #include <thread>
+#include <vector>
 
 namespace ZeroTier {
 
-class DBMirrorSet : public DB::ChangeListener
+class DBMirrorSet : public DB::ChangeListener {
-{
   public:
 	DBMirrorSet(DB::ChangeListener* listener);
 	virtual ~DBMirrorSet();
@@ -44,7 +43,8 @@ public:
 	bool save(nlohmann::json& record, bool notifyListeners);
 	void eraseNetwork(const uint64_t networkId);
 	void eraseMember(const uint64_t networkId, const uint64_t memberId);
-	void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
 	// These are called by various DB instances when changes occur.
 	virtual void onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network);

controller/EmbeddedNetworkController.cpp

@@ -21,33 +21,32 @@
 #ifndef _WIN32
 #include <sys/time.h>
 #endif
-#include <sys/types.h>
-
-#include <algorithm>
-#include <utility>
-#include <stdexcept>
-#include <map>
-#include <thread>
-#include <memory>
-#include <iomanip>
-#include <sstream>
-#include <cctype>
-
 #include "../include/ZeroTierOne.h"
 #include "../version.h"
-
 #include "EmbeddedNetworkController.hpp"
-#include "LFDB.hpp"
 #include "FileDB.hpp"
+#include "LFDB.hpp"
+
+#include <algorithm>
+#include <cctype>
+#include <iomanip>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <sys/types.h>
+#include <thread>
+#include <utility>
 #ifdef ZT_CONTROLLER_USE_LIBPQ
-#include "PostgreSQL.hpp"
+#include "CV1.hpp"
+#include "CV2.hpp"
 #endif
 
-#include "../node/Node.hpp"
 #include "../node/CertificateOfMembership.hpp"
-#include "../node/NetworkConfig.hpp"
 #include "../node/Dictionary.hpp"
 #include "../node/MAC.hpp"
+#include "../node/NetworkConfig.hpp"
+#include "../node/Node.hpp"
 
 using json = nlohmann::json;
 
@@ -125,12 +124,30 @@ static json _renderRule(ZT_VirtualNetworkRule &rule)
 				break;
 			case ZT_NETWORK_RULE_MATCH_MAC_SOURCE:
 				r["type"] = "MATCH_MAC_SOURCE";
-				OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]);
+				OSUtils::ztsnprintf(
+					tmp,
+					sizeof(tmp),
+					"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",
+					(unsigned int)rule.v.mac[0],
+					(unsigned int)rule.v.mac[1],
+					(unsigned int)rule.v.mac[2],
+					(unsigned int)rule.v.mac[3],
+					(unsigned int)rule.v.mac[4],
+					(unsigned int)rule.v.mac[5]);
 				r["mac"] = tmp;
 				break;
 			case ZT_NETWORK_RULE_MATCH_MAC_DEST:
 				r["type"] = "MATCH_MAC_DEST";
-				OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]);
+				OSUtils::ztsnprintf(
+					tmp,
+					sizeof(tmp),
+					"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",
+					(unsigned int)rule.v.mac[0],
+					(unsigned int)rule.v.mac[1],
+					(unsigned int)rule.v.mac[2],
+					(unsigned int)rule.v.mac[3],
+					(unsigned int)rule.v.mac[4],
+					(unsigned int)rule.v.mac[5]);
 				r["mac"] = tmp;
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE:
@@ -168,7 +185,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule)
 				r["icmpType"] = (unsigned int)rule.v.icmp.type;
 				if ((rule.v.icmp.flags & 0x01) != 0)
 					r["icmpCode"] = (unsigned int)rule.v.icmp.code;
-				else r["icmpCode"] = json();
+				else
+					r["icmpCode"] = json();
 				break;
 			case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE:
 				r["type"] = "MATCH_IP_SOURCE_PORT_RANGE";
@@ -262,7 +280,8 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
 
 	if (OSUtils::jsonBool(r["not"], false))
 		rule.t = 0x80;
-	else rule.t = 0x00;
+	else
+		rule.t = 0x00;
 	if (OSUtils::jsonBool(r["or"], false))
 		rule.t |= 0x40;
 
@@ -270,178 +289,221 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
 	if (t == "ACTION_DROP") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_DROP;
 		return true;
-	} else if (t == "ACTION_ACCEPT") {
+	}
+	else if (t == "ACTION_ACCEPT") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT;
 		return true;
-	} else if (t == "ACTION_TEE") {
+	}
+	else if (t == "ACTION_TEE") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_TEE;
 		rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"], "0").c_str()) & 0xffffffffffULL;
 		rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"], 0ULL) & 0xffffffffULL);
 		rule.v.fwd.length = (uint16_t)(OSUtils::jsonInt(r["length"], 0ULL) & 0xffffULL);
 		return true;
-	} else if (t == "ACTION_WATCH") {
+	}
+	else if (t == "ACTION_WATCH") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_WATCH;
 		rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"], "0").c_str()) & 0xffffffffffULL;
 		rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"], 0ULL) & 0xffffffffULL);
 		rule.v.fwd.length = (uint16_t)(OSUtils::jsonInt(r["length"], 0ULL) & 0xffffULL);
 		return true;
-	} else if (t == "ACTION_REDIRECT") {
+	}
+	else if (t == "ACTION_REDIRECT") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT;
 		rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"], "0").c_str()) & 0xffffffffffULL;
 		rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"], 0ULL) & 0xffffffffULL);
 		return true;
-	} else if (t == "ACTION_BREAK") {
+	}
+	else if (t == "ACTION_BREAK") {
 		rule.t |= ZT_NETWORK_RULE_ACTION_BREAK;
 		return true;
-	} else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") {
+	}
+	else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS;
 		rule.v.zt = Utils::hexStrToU64(OSUtils::jsonString(r["zt"], "0").c_str()) & 0xffffffffffULL;
 		return true;
-	} else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") {
+	}
+	else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS;
 		rule.v.zt = Utils::hexStrToU64(OSUtils::jsonString(r["zt"], "0").c_str()) & 0xffffffffffULL;
 		return true;
-	} else if (t == "MATCH_VLAN_ID") {
+	}
+	else if (t == "MATCH_VLAN_ID") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID;
 		rule.v.vlanId = (uint16_t)(OSUtils::jsonInt(r["vlanId"], 0ULL) & 0xffffULL);
 		return true;
-	} else if (t == "MATCH_VLAN_PCP") {
+	}
+	else if (t == "MATCH_VLAN_PCP") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP;
 		rule.v.vlanPcp = (uint8_t)(OSUtils::jsonInt(r["vlanPcp"], 0ULL) & 0xffULL);
 		return true;
-	} else if (t == "MATCH_VLAN_DEI") {
+	}
+	else if (t == "MATCH_VLAN_DEI") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI;
 		rule.v.vlanDei = (uint8_t)(OSUtils::jsonInt(r["vlanDei"], 0ULL) & 0xffULL);
 		return true;
-	} else if (t == "MATCH_MAC_SOURCE") {
+	}
+	else if (t == "MATCH_MAC_SOURCE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE;
-		const std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		std::string mac(OSUtils::jsonString(r["mac"], "0"));
+		Utils::cleanMac(mac);
 		Utils::unhex(mac.c_str(), (unsigned int)mac.length(), rule.v.mac, 6);
 		return true;
-	} else if (t == "MATCH_MAC_DEST") {
+	}
+	else if (t == "MATCH_MAC_DEST") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST;
-		const std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		std::string mac(OSUtils::jsonString(r["mac"], "0"));
+		Utils::cleanMac(mac);
 		Utils::unhex(mac.c_str(), (unsigned int)mac.length(), rule.v.mac, 6);
 		return true;
-	} else if (t == "MATCH_IPV4_SOURCE") {
+	}
+	else if (t == "MATCH_IPV4_SOURCE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE;
 		InetAddress ip(OSUtils::jsonString(r["ip"], "0.0.0.0").c_str());
 		rule.v.ipv4.ip = reinterpret_cast<struct sockaddr_in*>(&ip)->sin_addr.s_addr;
 		rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in*>(&ip)->sin_port) & 0xff;
-		if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32;
+		if (rule.v.ipv4.mask > 32)
+			rule.v.ipv4.mask = 32;
 		return true;
-	} else if (t == "MATCH_IPV4_DEST") {
+	}
+	else if (t == "MATCH_IPV4_DEST") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST;
 		InetAddress ip(OSUtils::jsonString(r["ip"], "0.0.0.0").c_str());
 		rule.v.ipv4.ip = reinterpret_cast<struct sockaddr_in*>(&ip)->sin_addr.s_addr;
 		rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in*>(&ip)->sin_port) & 0xff;
-		if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32;
+		if (rule.v.ipv4.mask > 32)
+			rule.v.ipv4.mask = 32;
 		return true;
-	} else if (t == "MATCH_IPV6_SOURCE") {
+	}
+	else if (t == "MATCH_IPV6_SOURCE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE;
 		InetAddress ip(OSUtils::jsonString(r["ip"], "::0").c_str());
 		memcpy(rule.v.ipv6.ip, reinterpret_cast<struct sockaddr_in6*>(&ip)->sin6_addr.s6_addr, 16);
 		rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in6*>(&ip)->sin6_port) & 0xff;
-		if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128;
+		if (rule.v.ipv6.mask > 128)
+			rule.v.ipv6.mask = 128;
 		return true;
-	} else if (t == "MATCH_IPV6_DEST") {
+	}
+	else if (t == "MATCH_IPV6_DEST") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST;
 		InetAddress ip(OSUtils::jsonString(r["ip"], "::0").c_str());
 		memcpy(rule.v.ipv6.ip, reinterpret_cast<struct sockaddr_in6*>(&ip)->sin6_addr.s6_addr, 16);
 		rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in6*>(&ip)->sin6_port) & 0xff;
-		if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128;
+		if (rule.v.ipv6.mask > 128)
+			rule.v.ipv6.mask = 128;
 		return true;
-	} else if (t == "MATCH_IP_TOS") {
+	}
+	else if (t == "MATCH_IP_TOS") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS;
 		rule.v.ipTos.mask = (uint8_t)(OSUtils::jsonInt(r["mask"], 0ULL) & 0xffULL);
 		rule.v.ipTos.value[0] = (uint8_t)(OSUtils::jsonInt(r["start"], 0ULL) & 0xffULL);
 		rule.v.ipTos.value[1] = (uint8_t)(OSUtils::jsonInt(r["end"], 0ULL) & 0xffULL);
 		return true;
-	} else if (t == "MATCH_IP_PROTOCOL") {
+	}
+	else if (t == "MATCH_IP_PROTOCOL") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL;
 		rule.v.ipProtocol = (uint8_t)(OSUtils::jsonInt(r["ipProtocol"], 0ULL) & 0xffULL);
 		return true;
-	} else if (t == "MATCH_ETHERTYPE") {
+	}
+	else if (t == "MATCH_ETHERTYPE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE;
 		rule.v.etherType = (uint16_t)(OSUtils::jsonInt(r["etherType"], 0ULL) & 0xffffULL);
 		return true;
-	} else if (t == "MATCH_ICMP") {
+	}
+	else if (t == "MATCH_ICMP") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_ICMP;
 		rule.v.icmp.type = (uint8_t)(OSUtils::jsonInt(r["icmpType"], 0ULL) & 0xffULL);
 		json& code = r["icmpCode"];
 		if (code.is_null()) {
 			rule.v.icmp.code = 0;
 			rule.v.icmp.flags = 0x00;
-		} else {
+		}
+		else {
 			rule.v.icmp.code = (uint8_t)(OSUtils::jsonInt(code, 0ULL) & 0xffULL);
 			rule.v.icmp.flags = 0x01;
 		}
 		return true;
-	} else if (t == "MATCH_IP_SOURCE_PORT_RANGE") {
+	}
+	else if (t == "MATCH_IP_SOURCE_PORT_RANGE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE;
 		rule.v.port[0] = (uint16_t)(OSUtils::jsonInt(r["start"], 0ULL) & 0xffffULL);
 		rule.v.port[1] = (uint16_t)(OSUtils::jsonInt(r["end"], (uint64_t)rule.v.port[0]) & 0xffffULL);
 		return true;
-	} else if (t == "MATCH_IP_DEST_PORT_RANGE") {
+	}
+	else if (t == "MATCH_IP_DEST_PORT_RANGE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE;
 		rule.v.port[0] = (uint16_t)(OSUtils::jsonInt(r["start"], 0ULL) & 0xffffULL);
 		rule.v.port[1] = (uint16_t)(OSUtils::jsonInt(r["end"], (uint64_t)rule.v.port[0]) & 0xffffULL);
 		return true;
-	} else if (t == "MATCH_CHARACTERISTICS") {
+	}
+	else if (t == "MATCH_CHARACTERISTICS") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS;
 		if (r.count("mask")) {
 			json& v = r["mask"];
 			if (v.is_number()) {
 				rule.v.characteristics = v;
-			} else {
+			}
+			else {
 				std::string tmp = v;
 				rule.v.characteristics = Utils::hexStrToU64(tmp.c_str());
 			}
 		}
 		return true;
-	} else if (t == "MATCH_FRAME_SIZE_RANGE") {
+	}
+	else if (t == "MATCH_FRAME_SIZE_RANGE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE;
 		rule.v.frameSize[0] = (uint16_t)(OSUtils::jsonInt(r["start"], 0ULL) & 0xffffULL);
 		rule.v.frameSize[1] = (uint16_t)(OSUtils::jsonInt(r["end"], (uint64_t)rule.v.frameSize[0]) & 0xffffULL);
 		return true;
-	} else if (t == "MATCH_RANDOM") {
+	}
+	else if (t == "MATCH_RANDOM") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM;
 		rule.v.randomProbability = (uint32_t)(OSUtils::jsonInt(r["probability"], 0ULL) & 0xffffffffULL);
 		return true;
-	} else if (t == "MATCH_TAGS_DIFFERENCE") {
+	}
+	else if (t == "MATCH_TAGS_DIFFERENCE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE;
 		tag = true;
-	} else if (t == "MATCH_TAGS_BITWISE_AND") {
+	}
+	else if (t == "MATCH_TAGS_BITWISE_AND") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND;
 		tag = true;
-	} else if (t == "MATCH_TAGS_BITWISE_OR") {
+	}
+	else if (t == "MATCH_TAGS_BITWISE_OR") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR;
 		tag = true;
-	} else if (t == "MATCH_TAGS_BITWISE_XOR") {
+	}
+	else if (t == "MATCH_TAGS_BITWISE_XOR") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR;
 		tag = true;
-	} else if (t == "MATCH_TAGS_EQUAL") {
+	}
+	else if (t == "MATCH_TAGS_EQUAL") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL;
 		tag = true;
-	} else if (t == "MATCH_TAG_SENDER") {
+	}
+	else if (t == "MATCH_TAG_SENDER") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAG_SENDER;
 		tag = true;
-	} else if (t == "MATCH_TAG_RECEIVER") {
+	}
+	else if (t == "MATCH_TAG_RECEIVER") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_TAG_RECEIVER;
 		tag = true;
-	} else if (t == "INTEGER_RANGE") {
+	}
+	else if (t == "INTEGER_RANGE") {
 		json& s = r["start"];
 		if (s.is_string()) {
 			std::string tmp = s;
 			rule.v.intRange.start = Utils::hexStrToU64(tmp.c_str());
-		} else {
+		}
+		else {
 			rule.v.intRange.start = OSUtils::jsonInt(s, 0ULL);
 		}
 		json& e = r["end"];
 		if (e.is_string()) {
 			std::string tmp = e;
 			rule.v.intRange.end = (uint32_t)(Utils::hexStrToU64(tmp.c_str()) - rule.v.intRange.start);
-		} else {
+		}
+		else {
 			rule.v.intRange.end = (uint32_t)(OSUtils::jsonInt(e, 0ULL) - rule.v.intRange.start);
 		}
 		rule.v.intRange.idx = (uint16_t)OSUtils::jsonInt(r["idx"], 0ULL);
@@ -519,7 +581,8 @@ EmbeddedNetworkController::~EmbeddedNetworkController()
 	_ssoExpiry.join();
 }
 
-void EmbeddedNetworkController::setSSORedirectURL(const std::string &url) {
+void EmbeddedNetworkController::setSSORedirectURL(const std::string& url)
+{
 	_ssoRedirectURL = url;
 }
 
@@ -532,8 +595,15 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 	if ((_path.length() > 9) && (_path.substr(0, 9) == "postgres:")) {
-		_db.addDB(std::shared_ptr<DB>(new PostgreSQL(_signingId,_path.substr(9).c_str(), _listenPort, _rc)));
+		fprintf(stderr, "CV1\n");
-	} else {
+		_db.addDB(std::shared_ptr<DB>(new CV1(_signingId, _path.substr(9).c_str(), _listenPort, _rc)));
+	}
+	else if ((_path.length() > 4) && (_path.substr(0, 4) == "cv2:")) {
+		fprintf(stderr, "CV2\n");
+		_db.addDB(std::shared_ptr<DB>(new CV2(_signingId, _path.substr(4).c_str(), _listenPort)));
+	}
+	else {
+		fprintf(stderr, "FileDB\n");
 #endif
 		_db.addDB(std::shared_ptr<DB>(new FileDB(_path.c_str())));
 #ifdef ZT_CONTROLLER_USE_LIBPQ
@@ -573,12 +643,7 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 	_db.waitForReady();
 }
 
-void EmbeddedNetworkController::request(
+void EmbeddedNetworkController::request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData)
-	uint64_t nwid,
-	const InetAddress &fromAddr,
-	uint64_t requestPacketId,
-	const Identity &identity,
-	const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData)
 {
 	if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) || (! _sender))
 		return;
@@ -615,37 +680,48 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 	json network;
 	_db.get(networkID, network);
 	DB::initNetwork(network);
-	if (b.count("name")) network["name"] = OSUtils::jsonString(b["name"],"");
+	if (b.count("name"))
-	if (b.count("private")) network["private"] = OSUtils::jsonBool(b["private"],true);
+		network["name"] = OSUtils::jsonString(b["name"], "");
-	if (b.count("enableBroadcast")) network["enableBroadcast"] = OSUtils::jsonBool(b["enableBroadcast"],false);
+	if (b.count("private"))
-	if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"],32ULL);
+		network["private"] = OSUtils::jsonBool(b["private"], true);
-	if (b.count("mtu")) network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU);
+	if (b.count("enableBroadcast"))
+		network["enableBroadcast"] = OSUtils::jsonBool(b["enableBroadcast"], false);
+	if (b.count("multicastLimit"))
+		network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"], 32ULL);
+	if (b.count("mtu"))
+		network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), (unsigned int)ZT_MIN_MTU);
 
 	if (b.count("remoteTraceTarget")) {
 		const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"], ""));
 		if (rtt.length() == 10) {
 			network["remoteTraceTarget"] = rtt;
-		} else {
+		}
+		else {
 			network["remoteTraceTarget"] = json();
 		}
 	}
-	if (b.count("remoteTraceLevel")) network["remoteTraceLevel"] = OSUtils::jsonInt(b["remoteTraceLevel"],0ULL);
+	if (b.count("remoteTraceLevel"))
+		network["remoteTraceLevel"] = OSUtils::jsonInt(b["remoteTraceLevel"], 0ULL);
 
 	if (b.count("v4AssignMode")) {
 		json nv4m;
 		json& v4m = b["v4AssignMode"];
 		if (v4m.is_string()) {	 // backward compatibility
 			nv4m["zt"] = (OSUtils::jsonString(v4m, "") == "zt");
-		} else if (v4m.is_object()) {
+		}
+		else if (v4m.is_object()) {
 			nv4m["zt"] = OSUtils::jsonBool(v4m["zt"], false);
-		} else nv4m["zt"] = false;
+		}
+		else
+			nv4m["zt"] = false;
 		network["v4AssignMode"] = nv4m;
 	}
 
 	if (b.count("v6AssignMode")) {
 		json nv6m;
 		json& v6m = b["v6AssignMode"];
-		if (!nv6m.is_object()) nv6m = json::object();
+		if (! nv6m.is_object())
+			nv6m = json::object();
 		if (v6m.is_string()) {	 // backward compatibility
 			std::vector<std::string> v6ms(OSUtils::split(OSUtils::jsonString(v6m, "").c_str(), ",", "", ""));
 			std::sort(v6ms.begin(), v6ms.end());
@@ -661,11 +737,16 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 				else if (*i == "6plane")
 					nv6m["6plane"] = true;
 			}
-		} else if (v6m.is_object()) {
+		}
-			if (v6m.count("rfc4193")) nv6m["rfc4193"] = OSUtils::jsonBool(v6m["rfc4193"],false);
+		else if (v6m.is_object()) {
-			if (v6m.count("zt")) nv6m["zt"] = OSUtils::jsonBool(v6m["zt"],false);
+			if (v6m.count("rfc4193"))
-			if (v6m.count("6plane")) nv6m["6plane"] = OSUtils::jsonBool(v6m["6plane"],false);
+				nv6m["rfc4193"] = OSUtils::jsonBool(v6m["rfc4193"], false);
-		} else {
+			if (v6m.count("zt"))
+				nv6m["zt"] = OSUtils::jsonBool(v6m["zt"], false);
+			if (v6m.count("6plane"))
+				nv6m["6plane"] = OSUtils::jsonBool(v6m["6plane"], false);
+		}
+		else {
 			nv6m["rfc4193"] = false;
 			nv6m["zt"] = false;
 			nv6m["6plane"] = false;
@@ -685,14 +766,16 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 					if (target.is_string()) {
 						InetAddress t(target.get<std::string>().c_str());
 						InetAddress v;
-						if (via.is_string()) v.fromString(via.get<std::string>().c_str());
+						if (via.is_string())
+							v.fromString(via.get<std::string>().c_str());
 						if (((t.ss_family == AF_INET) || (t.ss_family == AF_INET6)) && (t.netmaskBitsValid())) {
 							json tmp;
 							char tmp2[64];
 							tmp["target"] = t.toString(tmp2);
 							if (v.ss_family == t.ss_family)
 								tmp["via"] = v.toIpString(tmp2);
-							else tmp["via"] = json();
+							else
+								tmp["via"] = json();
 							nrts.push_back(tmp);
 							if (nrts.size() >= ZT_CONTROLLER_MAX_ARRAY_SIZE)
 								break;
@@ -756,7 +839,8 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 					nat[t.key()] = t.value();
 			}
 			network["authTokens"] = nat;
-		} else {
+		}
+		else {
 			network["authTokens"] = { {} };
 		}
 	}
@@ -817,7 +901,8 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 					json& dfl = tag["default"];
 					if (dfl.is_null())
 						ntag["default"] = dfl;
-					else ntag["default"] = OSUtils::jsonInt(dfl,0ULL);
+					else
+						ntag["default"] = OSUtils::jsonInt(dfl, 0ULL);
 					ntags[tagId] = ntag;
 				}
 			}
@@ -861,10 +946,7 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ
 	return network.dump();
 }
 
-void EmbeddedNetworkController::configureHTTPControlPlane(
+void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, httplib::Server& sv6, const std::function<void(const httplib::Request&, httplib::Response&, std::string)> setContent)
-	httplib::Server &s,
-	httplib::Server &sv6,
-	const std::function<void(const httplib::Request&, httplib::Response&, std::string)> setContent)
 {
 	// Control plane Endpoints
 	std::string controllerPath = "/controller";
@@ -876,7 +958,6 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	std::string memberListPath2 = "/unstable/controller/network/([0-9a-fA-F]{16})/member";
 	std::string memberPath = "/controller/network/([0-9a-fA-F]{16})/member/([0-9a-fA-F]{10})";
 
-
 	auto controllerGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
 		char tmp[4096];
 		const bool dbOk = _db.isReady();
@@ -936,7 +1017,9 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 
 					for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
 						bool a = OSUtils::jsonBool((*m)["authorized"], 0);
-						if (a) { authorizedCount++; }
+						if (a) {
+							authorizedCount++;
+						}
 					}
 
 					auto nwMeta = json::object();
@@ -981,7 +1064,8 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		for (unsigned long k = 0; k < 100000; ++k) {   // sanity limit on trials
 			Utils::getSecureRandom(&nwidPostfix, sizeof(nwidPostfix));
 			uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
-			if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
+			if ((tryNwid & 0xffffffULL) == 0ULL)
+				tryNwid |= 1ULL;
 			if (! _db.hasNetwork(tryNwid)) {
 				nwid = tryNwid;
 				break;
@@ -1013,7 +1097,8 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		for (unsigned long k = 0; k < 100000; ++k) {   // sanity limit on trials
 			Utils::getSecureRandom(&nwidPostfix, sizeof(nwidPostfix));
 			uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
-			if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
+			if ((tryNwid & 0xffffffULL) == 0ULL)
+				tryNwid |= 1ULL;
 			if (! _db.hasNetwork(tryNwid)) {
 				nwid = tryNwid;
 				break;
@@ -1101,7 +1186,9 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 			uint64_t totalCount = memTmp.size();
 			for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
 				bool a = OSUtils::jsonBool((*m)["authorized"], 0);
-				if (a) { authorizedCount++; }
+				if (a) {
+					authorizedCount++;
+				}
 			}
 
 			meta["totalCount"] = totalCount;
@@ -1111,7 +1198,8 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 			out["meta"] = meta;
 
 			setContent(req, res, out.dump());
-		} else {
+		}
+		else {
 			res.status = 404;
 			return;
 		}
@@ -1154,21 +1242,28 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 
 		json b = OSUtils::jsonParse(req.body);
 
-		if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"], false);
+		if (b.count("activeBridge"))
-		if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
+			member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"], false);
-		if (b.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
+		if (b.count("noAutoAssignIps"))
-		if (b.count("authenticationURL")) member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
+			member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
-		if (b.count("name")) member["name"] = OSUtils::jsonString(b["name"], "");
+		if (b.count("authenticationExpiryTime"))
+			member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
+		if (b.count("authenticationURL"))
+			member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
+		if (b.count("name"))
+			member["name"] = OSUtils::jsonString(b["name"], "");
 
 		if (b.count("remoteTraceTarget")) {
 			const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"], ""));
 			if (rtt.length() == 10) {
 				member["remoteTraceTarget"] = rtt;
-			} else {
+			}
+			else {
 				member["remoteTraceTarget"] = json();
 			}
 		}
-		if (b.count("remoteTraceLevel")) member["remoteTraceLevel"] = OSUtils::jsonInt(b["remoteTraceLevel"],0ULL);
+		if (b.count("remoteTraceLevel"))
+			member["remoteTraceLevel"] = OSUtils::jsonInt(b["remoteTraceLevel"], 0ULL);
 
 		if (b.count("authorized")) {
 			const bool newAuth = OSUtils::jsonBool(b["authorized"], false);
@@ -1298,15 +1393,26 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
 						++eq;
 						if (*eq) {
 							switch (*eq) {
-								case 'r': v.push_back('\r'); break;
+								case 'r':
-								case 'n': v.push_back('\n'); break;
+									v.push_back('\r');
-								case '0': v.push_back((char)0); break;
+									break;
-								case 'e': v.push_back('='); break;
+								case 'n':
-								default: v.push_back(*eq); break;
+									v.push_back('\n');
+									break;
+								case '0':
+									v.push_back((char)0);
+									break;
+								case 'e':
+									v.push_back('=');
+									break;
+								default:
+									v.push_back(*eq);
+									break;
 							}
 							++eq;
 						}
-					} else {
+					}
+					else {
 						v.push_back(*(eq++));
 					}
 				}
@@ -1322,7 +1428,8 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
 		d["ts"] = now;
 		d["nodeId"] = Utils::hex10(rt.origin, tmp);
 		_db.save(d, true);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		// drop invalid trace messages if an error occurs
 	}
 }
@@ -1346,7 +1453,9 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const void *db,uint64_t ne
 		_MemberStatus& ms = _memberStatus[_MemberStatusKey(networkId, memberId)];
 		if ((ms.online(OSUtils::now())) && (ms.lastRequestMetaData))
 			request(networkId, InetAddress(), 0, ms.identity, ms.lastRequestMetaData);
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
 void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void* db, uint64_t networkId, uint64_t memberId)
@@ -1363,16 +1472,12 @@ void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void *db,uint64
 	}
 }
 
-void EmbeddedNetworkController::_request(
+void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData)
-	uint64_t nwid,
-	const InetAddress &fromAddr,
-	uint64_t requestPacketId,
-	const Identity &identity,
-	const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData)
 {
 	Metrics::network_config_request++;
 	auto tid = std::this_thread::get_id();
-	std::stringstream ss; ss << tid;
+	std::stringstream ss;
+	ss << tid;
 	std::string threadID = ss.str();
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 	auto b1 = _member_status_lookup.Add({ { "thread", threadID } });
@@ -1398,7 +1503,11 @@ void EmbeddedNetworkController::_request(
 	c2++;
 	b2.start();
 #endif
-	_db.nodeIsOnline(nwid,identity.address().toInt(),fromAddr);
+	char osArch[256];
+	metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH, osArch, sizeof(osArch));
+	// fprintf(stderr, "Network Config Request: nwid=%.16llx, nodeid=%.10llx, osArch=%s\n",
+	// 	nwid, identity.address().toInt(), osArch);
+	_db.nodeIsOnline(nwid, identity.address().toInt(), fromAddr, osArch);
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 	b2.stop();
 
@@ -1443,14 +1552,16 @@ void EmbeddedNetworkController::_request(
 #endif
 					return;
 				}
-			} catch ( ... ) {
+			}
+			catch (...) {
 				_sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 				b4.stop();
 #endif
 				return;
 			}
-		} else {
+		}
+		else {
 			// If we do not yet know this member's identity, learn it.
 			char idtmp[1024];
 			member["identity"] = identity.toString(false, idtmp);
@@ -1481,11 +1592,13 @@ void EmbeddedNetworkController::_request(
 	json autoAuthCredentialType, autoAuthCredential;
 	if (OSUtils::jsonBool(member["authorized"], false)) {
 		authorized = true;
-	} else if (!OSUtils::jsonBool(network["private"],true)) {
+	}
+	else if (! OSUtils::jsonBool(network["private"], true)) {
 		authorized = true;
 		autoAuthorized = true;
 		autoAuthCredentialType = "public";
-	} else {
+	}
+	else {
 		char presentedAuth[512];
 		if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH, presentedAuth, sizeof(presentedAuth)) > 0) {
 			presentedAuth[511] = (char)0;	// sanity check
@@ -1539,7 +1652,8 @@ void EmbeddedNetworkController::_request(
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, (uint64_t)0ULL);
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_AUTHENTICATION_URL, info.authenticationURL.c_str());
 				_sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
-			} else if (info.version == 1) {
+			}
+			else if (info.version == 1) {
 				Dictionary<8192> authInfo;
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, info.version);
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_ISSUER_URL, info.issuerURL.c_str());
@@ -1547,6 +1661,7 @@ void EmbeddedNetworkController::_request(
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_NONCE, info.ssoNonce.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_STATE, info.ssoState.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_CLIENT_ID, info.ssoClientID.c_str());
+				authInfo.add(ZT_AUTHINFO_DICT_KEY_SSO_PROVIDER, info.ssoProvider.c_str());
 				_sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
 			}
 			DB::cleanMember(member);
@@ -1595,7 +1710,8 @@ void EmbeddedNetworkController::_request(
 				_expiringSoon.insert(std::pair<int64_t, _MemberStatusKey>(authenticationExpiryTime, msk));
 			}
 		}
-	} else {
+	}
+	else {
 		// If they are not authorized, STOP!
 		DB::cleanMember(member);
 		_db.save(member, true);
@@ -1635,7 +1751,8 @@ void EmbeddedNetworkController::_request(
 	nc->credentialTimeMaxDelta = credentialtmd;
 	nc->revision = OSUtils::jsonInt(network["revision"], 0ULL);
 	nc->issuedTo = identity.address();
-	if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST;
+	if (OSUtils::jsonBool(network["enableBroadcast"], true))
+		nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST;
 	Utils::scopy(nc->name, sizeof(nc->name), OSUtils::jsonString(network["name"], "").c_str());
 	nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), (unsigned int)ZT_MIN_MTU);
 	nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"], 32ULL);
@@ -1678,11 +1795,13 @@ void EmbeddedNetworkController::_request(
 	if (rtt.length() == 10) {
 		nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
 		nc->remoteTraceLevel = (Trace::Level)OSUtils::jsonInt(member["remoteTraceLevel"], 0ULL);
-	} else {
+	}
+	else {
 		rtt = OSUtils::jsonString(network["remoteTraceTarget"], "");
 		if (rtt.length() == 10) {
 			nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
-		} else {
+		}
+		else {
 			nc->remoteTraceTarget.zero();
 		}
 		nc->remoteTraceLevel = (Trace::Level)OSUtils::jsonInt(network["remoteTraceLevel"], 0ULL);
@@ -1711,7 +1830,8 @@ void EmbeddedNetworkController::_request(
 		// enforce rules on the inbound side.
 		nc->ruleCount = 1;
 		nc->rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT;
-	} else {
+	}
+	else {
 		if (rules.is_array()) {
 			for (unsigned long i = 0; i < rules.size(); ++i) {
 				if (nc->ruleCount >= ZT_MAX_NETWORK_RULES)
@@ -1813,7 +1933,8 @@ void EmbeddedNetworkController::_request(
 			if (target.is_string()) {
 				const InetAddress t(target.get<std::string>().c_str());
 				InetAddress v;
-				if (via.is_string()) v.fromString(via.get<std::string>().c_str());
+				if (via.is_string())
+					v.fromString(via.get<std::string>().c_str());
 				if ((t.ss_family == AF_INET) || (t.ss_family == AF_INET6)) {
 					ZT_VirtualNetworkRoute* r = &(nc->routes[nc->routeCount]);
 					*(reinterpret_cast<InetAddress*>(&(r->target))) = t;
@@ -1868,7 +1989,8 @@ void EmbeddedNetworkController::_request(
 				}
 			}
 		}
-	} else {
+	}
+	else {
 		ipAssignments = json::array();
 	}
 
@@ -1894,15 +2016,18 @@ void EmbeddedNetworkController::_request(
 							// First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that.
 							xx[0] = Utils::hton(x[0]);
 							xx[1] = Utils::hton(x[1] + identity.address().toInt());
-						} else {
+						}
+						else {
 							// Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway
 							Utils::getSecureRandom((void*)xx, 16);
 							if ((e[0] > s[0]))
 								xx[0] %= (e[0] - s[0]);
-							else xx[0] = 0;
+							else
+								xx[0] = 0;
 							if ((e[1] > s[1]))
 								xx[1] %= (e[1] - s[1]);
-							else xx[1] = 0;
+							else
+								xx[1] = 0;
 							xx[0] = Utils::hton(x[0] + xx[0]);
 							xx[1] = Utils::hton(x[1] + xx[1]);
 						}
@@ -2007,7 +2132,8 @@ void EmbeddedNetworkController::_request(
 				nc->dns.server_addr[j] = InetAddress(OSUtils::jsonString(addr, "").c_str());
 			}
 		}
-	} else {
+	}
+	else {
 		dns = json::object();
 	}
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
@@ -2033,7 +2159,8 @@ void EmbeddedNetworkController::_request(
 	CertificateOfMembership com(now, credentialtmd, nwid, identity);
 	if (com.sign(_signingId)) {
 		nc->com = com;
-	} else {
+	}
+	else {
 		_sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0);
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 		b9.stop();
@@ -2080,13 +2207,16 @@ void EmbeddedNetworkController::_startThreads()
 				auto timedWaitResult = _queue.get(qe, 1000);
 				if (timedWaitResult == BlockingQueue<_RQEntry*>::STOP) {
 					break;
-				} else if (timedWaitResult == BlockingQueue<_RQEntry *>::OK) {
+				}
+				else if (timedWaitResult == BlockingQueue<_RQEntry*>::OK) {
 					if (qe) {
 						try {
 							_request(qe->nwid, qe->fromAddr, qe->requestPacketId, qe->identity, qe->metaData);
-						} catch (std::exception &e) {
+						}
+						catch (std::exception& e) {
 							fprintf(stderr, "ERROR: exception in controller request handling thread: %s" ZT_EOL_S, e.what());
-						} catch ( ... ) {
+						}
+						catch (...) {
 							fprintf(stderr, "ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S);
 						}
 						delete qe;
@@ -2099,7 +2229,8 @@ void EmbeddedNetworkController::_startThreads()
 	}
 }
 
-void EmbeddedNetworkController::_ssoExpiryThread() {
+void EmbeddedNetworkController::_ssoExpiryThread()
+{
 	while (_ssoExpiryRunning) {
 		std::vector<_MemberStatusKey> expired;
 		nlohmann::json network, member;
@@ -2120,7 +2251,8 @@ void EmbeddedNetworkController::_ssoExpiryThread() {
 						}
 					}
 					s = _expiringSoon.erase(s);
-				} else {
+				}
+				else {
 					// Don't bother going further into the future than necessary.
 					break;
 				}

controller/EmbeddedNetworkController.hpp

@@ -14,41 +14,37 @@
 #ifndef ZT_SQLITENETWORKCONTROLLER_HPP
 #define ZT_SQLITENETWORKCONTROLLER_HPP
 
-#include <stdint.h>
+#include "../node/Address.hpp"
-
-#include <string>
-#include <map>
-#include <vector>
-#include <set>
-#include <list>
-#include <thread>
-#include <unordered_map>
-#include <atomic>
-
 #include "../node/Constants.hpp"
+#include "../node/InetAddress.hpp"
 #include "../node/NetworkController.hpp"
 #include "../node/Utils.hpp"
-#include "../node/Address.hpp"
+#include "../osdep/BlockingQueue.hpp"
-#include "../node/InetAddress.hpp"
-
 #include "../osdep/OSUtils.hpp"
 #include "../osdep/Thread.hpp"
-#include "../osdep/BlockingQueue.hpp"
-
-#include <nlohmann/json.hpp>
-
-#include <cpp-httplib/httplib.h>
-
 #include "DB.hpp"
 #include "DBMirrorSet.hpp"
 
+#include <atomic>
+#include <cpp-httplib/httplib.h>
+#include <list>
+#include <map>
+#include <nlohmann/json.hpp>
+#include <set>
+#include <stdint.h>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
 namespace ZeroTier {
 
 class Node;
 struct RedisConfig;
 
-class EmbeddedNetworkController : public NetworkController,public DB::ChangeListener
+class EmbeddedNetworkController
-{
+	: public NetworkController
+	, public DB::ChangeListener {
   public:
 	/**
 	 * @param node Parent node
@@ -61,17 +57,9 @@ public:
 
 	void setSSORedirectURL(const std::string& url);
 
-	virtual void request(
+	virtual void request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData);
-		uint64_t nwid,
-		const InetAddress &fromAddr,
-		uint64_t requestPacketId,
-		const Identity &identity,
-		const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData);
 
-	void configureHTTPControlPlane(
+	void configureHTTPControlPlane(httplib::Server& s, httplib::Server& sV6, const std::function<void(const httplib::Request&, httplib::Response&, std::string)>);
-		httplib::Server &s,
-		httplib::Server &sV6,
-		const std::function<void(const httplib::Request&, httplib::Response&, std::string)>);
 
 	void handleRemoteTrace(const ZT_RemoteTrace& rt);
 
@@ -86,39 +74,48 @@ private:
 
 	std::string networkUpdateFromPostData(uint64_t networkID, const std::string& body);
 
-	struct _RQEntry
+	struct _RQEntry {
-	{
 		uint64_t nwid;
 		uint64_t requestPacketId;
 		InetAddress fromAddr;
 		Identity identity;
 		Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> metaData;
-		enum {
+		enum { RQENTRY_TYPE_REQUEST = 0 } type;
-			RQENTRY_TYPE_REQUEST = 0
-		} type;
 	};
 
-	struct _MemberStatusKey
+	struct _MemberStatusKey {
+		_MemberStatusKey() : networkId(0), nodeId(0)
 		{
-		_MemberStatusKey() : networkId(0),nodeId(0) {}
+		}
-		_MemberStatusKey(const uint64_t nwid,const uint64_t nid) : networkId(nwid),nodeId(nid) {}
+		_MemberStatusKey(const uint64_t nwid, const uint64_t nid) : networkId(nwid), nodeId(nid)
+		{
+		}
 		uint64_t networkId;
 		uint64_t nodeId;
-		inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); }
+		inline bool operator==(const _MemberStatusKey& k) const
-		inline bool operator<(const _MemberStatusKey &k) const { return (k.networkId < networkId) || ((k.networkId == networkId)&&(k.nodeId < nodeId)); }
-	};
-	struct _MemberStatus
 		{
-		_MemberStatus() : lastRequestTime(0),authenticationExpiryTime(-1),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {}
+			return ((k.networkId == networkId) && (k.nodeId == nodeId));
+		}
+		inline bool operator<(const _MemberStatusKey& k) const
+		{
+			return (k.networkId < networkId) || ((k.networkId == networkId) && (k.nodeId < nodeId));
+		}
+	};
+	struct _MemberStatus {
+		_MemberStatus() : lastRequestTime(0), authenticationExpiryTime(-1), vMajor(-1), vMinor(-1), vRev(-1), vProto(-1)
+		{
+		}
 		int64_t lastRequestTime;
 		int64_t authenticationExpiryTime;
 		int vMajor, vMinor, vRev, vProto;
 		Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastRequestMetaData;
 		Identity identity;
-		inline bool online(const int64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); }
+		inline bool online(const int64_t now) const
-	};
-	struct _MemberStatusHash
 		{
+			return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2));
+		}
+	};
+	struct _MemberStatusHash {
 		inline std::size_t operator()(const _MemberStatusKey& networkIdNodeId) const
 		{
 			return (std::size_t)(networkIdNodeId.networkId + networkIdNodeId.nodeId);

controller/FileDB.cpp

@@ -15,15 +15,9 @@
 
 #include "../node/Metrics.hpp"
 
-namespace ZeroTier
+namespace ZeroTier {
-{
 
-FileDB::FileDB(const char *path) :
+FileDB::FileDB(const char* path) : DB(), _path(path), _networksPath(_path + ZT_PATH_SEPARATOR_S + "network"), _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"), _running(true)
-	DB(),
-	_path(path),
-	_networksPath(_path + ZT_PATH_SEPARATOR_S + "network"),
-	_tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"),
-	_running(true)
 {
 	OSUtils::mkdir(_path.c_str());
 	OSUtils::lockDownFile(_path.c_str(), true);
@@ -55,11 +49,15 @@ FileDB::FileDB(const char *path) :
 									_memberChanged(nullJson2, member, false);
 									Metrics::member_count++;
 								}
-							} catch ( ... ) {}
+							}
+							catch (...) {
 							}
 						}
 					}
-			} catch ( ... ) {}
+				}
+			}
+			catch (...) {
+			}
 		}
 	}
 }
@@ -71,11 +69,19 @@ FileDB::~FileDB()
 		_running = false;
 		_online_l.unlock();
 		_onlineUpdateThread.join();
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
-bool FileDB::waitForReady() { return true; }
+bool FileDB::waitForReady()
-bool FileDB::isReady() { return true; }
+{
+	return true;
+}
+bool FileDB::isReady()
+{
+	return true;
+}
 
 bool FileDB::save(nlohmann::json& record, bool notifyListeners)
 {
@@ -84,7 +90,6 @@ bool FileDB::save(nlohmann::json &record,bool notifyListeners)
 	try {
 		const std::string objtype = record["objtype"];
 		if (objtype == "network") {
-
 			const uint64_t nwid = OSUtils::jsonIntHex(record["id"], 0ULL);
 			if (nwid) {
 				nlohmann::json old;
@@ -99,9 +104,8 @@ bool FileDB::save(nlohmann::json &record,bool notifyListeners)
 					modified = true;
 				}
 			}
-
+		}
-		} else if (objtype == "member") {
+		else if (objtype == "member") {
-
 			const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL);
 			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL);
 			if ((id) && (nwid)) {
@@ -123,9 +127,10 @@ bool FileDB::save(nlohmann::json &record,bool notifyListeners)
 					modified = true;
 				}
 			}
-
 		}
-	} catch ( ... ) {} // drop invalid records missing fields
+	}
+	catch (...) {
+	}	// drop invalid records missing fields
 	return modified;
 }
 
@@ -155,7 +160,7 @@ void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
 	this->_online[networkId].erase(memberId);
 }
 
-void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	char mid[32], atmp[64];
 	OSUtils::ztsnprintf(mid, sizeof(mid), "%.10llx", (unsigned long long)memberId);
@@ -164,4 +169,9 @@ void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const
 	this->_online[networkId][memberId][OSUtils::now()] = physicalAddress;
 }
 
+void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 }	// namespace ZeroTier

controller/FileDB.hpp

@@ -16,11 +16,9 @@
 
 #include "DB.hpp"
 
-namespace ZeroTier
+namespace ZeroTier {
-{
 
-class FileDB : public DB
+class FileDB : public DB {
-{
   public:
 	FileDB(const char* path);
 	virtual ~FileDB();
@@ -31,6 +29,7 @@ public:
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
   protected:
 	std::string _path;

controller/LFDB.cpp

@@ -13,33 +13,34 @@
 
 #include "LFDB.hpp"
 
-#include <thread>
+#include "../ext/cpp-httplib/httplib.h"
+#include "../osdep/OSUtils.hpp"
+
 #include <chrono>
 #include <iostream>
 #include <sstream>
+#include <thread>
 
-#include "../osdep/OSUtils.hpp"
+namespace ZeroTier {
-#include "../ext/cpp-httplib/httplib.h"
 
-namespace ZeroTier
+LFDB::LFDB(const Identity& myId, const char* path, const char* lfOwnerPrivate, const char* lfOwnerPublic, const char* lfNodeHost, int lfNodePort, bool storeOnlineState)
-{
+	: DB()
-
+	, _myId(myId)
-LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState) :
+	, _lfOwnerPrivate((lfOwnerPrivate) ? lfOwnerPrivate : "")
-	DB(),
+	, _lfOwnerPublic((lfOwnerPublic) ? lfOwnerPublic : "")
-	_myId(myId),
+	, _lfNodeHost((lfNodeHost) ? lfNodeHost : "127.0.0.1")
-	_lfOwnerPrivate((lfOwnerPrivate) ? lfOwnerPrivate : ""),
+	, _lfNodePort(((lfNodePort > 0) && (lfNodePort < 65536)) ? lfNodePort : 9980)
-	_lfOwnerPublic((lfOwnerPublic) ? lfOwnerPublic : ""),
+	, _running(true)
-	_lfNodeHost((lfNodeHost) ? lfNodeHost : "127.0.0.1"),
+	, _ready(false)
-	_lfNodePort(((lfNodePort > 0)&&(lfNodePort < 65536)) ? lfNodePort : 9980),
+	, _storeOnlineState(storeOnlineState)
-	_running(true),
-	_ready(false),
-	_storeOnlineState(storeOnlineState)
 {
 	_syncThread = std::thread([this]() {
 		char controllerAddress[24];
 		const uint64_t controllerAddressInt = _myId.address().toInt();
 		_myId.address().toString(controllerAddress);
-		std::string networksSelectorName("com.zerotier.controller.lfdb:"); networksSelectorName.append(controllerAddress); networksSelectorName.append("/network");
+		std::string networksSelectorName("com.zerotier.controller.lfdb:");
+		networksSelectorName.append(controllerAddress);
+		networksSelectorName.append("/network");
 
 		// LF record masking key is the first 32 bytes of SHA512(controller private key) in hex,
 		// hiding record values from anything but the controller or someone who has its key.
@@ -71,15 +72,19 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 									if (resp->status == 200) {
 										ns->second.dirty = false;
 										// printf("SET network %.16llx %s\n",ns->first,resp->body.c_str());
-									} else {
+									}
+									else {
 										fprintf(stderr, "ERROR: LFDB: %d from node (create/update network): %s" ZT_EOL_S, resp->status, resp->body.c_str());
 									}
-								} else {
+								}
+								else {
 									fprintf(stderr, "ERROR: LFDB: node is offline" ZT_EOL_S);
 								}
-							} catch (std::exception &e) {
+							}
+							catch (std::exception& e) {
 								fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update network): %s" ZT_EOL_S, e.what());
-							} catch ( ... ) {
+							}
+							catch (...) {
 								fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update network): unknown exception" ZT_EOL_S);
 							}
 						}
@@ -123,15 +128,19 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 									if (resp->status == 200) {
 										ms->second.lastOnlineDirty = false;
 										// printf("SET member online %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
-									} else {
+									}
+									else {
 										fprintf(stderr, "ERROR: LFDB: %d from node (create/update member online status): %s" ZT_EOL_S, resp->status, resp->body.c_str());
 									}
-								} else {
+								}
+								else {
 									fprintf(stderr, "ERROR: LFDB: node is offline" ZT_EOL_S);
 								}
-							} catch (std::exception &e) {
+							}
+							catch (std::exception& e) {
 								fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update member online status): %s" ZT_EOL_S, e.what());
-							} catch ( ... ) {
+							}
+							catch (...) {
 								fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update member online status): unknown exception" ZT_EOL_S);
 							}
 						}
@@ -157,15 +166,19 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 										if (resp->status == 200) {
 											ms->second.dirty = false;
 											// printf("SET member %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
-										} else {
+										}
+										else {
 											fprintf(stderr, "ERROR: LFDB: %d from node (create/update member): %s" ZT_EOL_S, resp->status, resp->body.c_str());
 										}
-									} else {
+									}
+									else {
 										fprintf(stderr, "ERROR: LFDB: node is offline" ZT_EOL_S);
 									}
-								} catch (std::exception &e) {
+								}
+								catch (std::exception& e) {
 									fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update member): %s" ZT_EOL_S, e.what());
-								} catch ( ... ) {
+								}
+								catch (...) {
 									fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (create/update member): unknown exception" ZT_EOL_S);
 								}
 							}
@@ -176,15 +189,22 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 
 			try {
 				std::ostringstream query;
-				query <<
+				query << "{"
-					"{"
 						 "\"Ranges\":[{"
-							"\"Name\":\"" << networksSelectorName << "\","
+						 "\"Name\":\""
+					  << networksSelectorName
+					  << "\","
 						 "\"Range\":[0,18446744073709551615]"
 						 "}],"
-						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						 "\"TimeRange\":["
-						"\"MaskingKey\":\"" << maskingKey << "\","
+					  << timeRangeStart
-						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					  << ",9223372036854775807],"
+						 "\"MaskingKey\":\""
+					  << maskingKey
+					  << "\","
+						 "\"Owners\":[\""
+					  << _lfOwnerPublic
+					  << "\"]"
 						 "}";
 				auto resp = htcli.Post("/query", query.str(), "application/json");
 				if (resp) {
@@ -194,7 +214,6 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							for (std::size_t ri = 0; ri < results.size(); ++ri) {
 								nlohmann::json& rset = results[ri];
 								if ((rset.is_array()) && (! rset.empty())) {
-
 									nlohmann::json& result = rset[0];
 									if (result.is_object()) {
 										nlohmann::json& record = result["Record"];
@@ -214,45 +233,55 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 														if (prevRevision < revision) {
 															_networkChanged(oldNetwork, network, timeRangeStart > 0);
 														}
-													} else {
+													}
+													else {
 														nlohmann::json nullJson;
 														_networkChanged(nullJson, network, timeRangeStart > 0);
 													}
-
 												}
 											}
 										}
 									}
-
 								}
 							}
 						}
-					} else {
+					}
+					else {
 						fprintf(stderr, "ERROR: LFDB: %d from node (check for network updates): %s" ZT_EOL_S, resp->status, resp->body.c_str());
 					}
-				} else {
+				}
+				else {
 					fprintf(stderr, "ERROR: LFDB: node is offline" ZT_EOL_S);
 				}
-			} catch (std::exception &e) {
+			}
+			catch (std::exception& e) {
 				fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (check for network updates): %s" ZT_EOL_S, e.what());
-			} catch ( ... ) {
+			}
+			catch (...) {
 				fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (check for network updates): unknown exception" ZT_EOL_S);
 			}
 
 			try {
 				std::ostringstream query;
-				query <<
+				query << "{"
-					"{"
 						 "\"Ranges\":[{"
-							"\"Name\":\"" << networksSelectorName << "\","
+						 "\"Name\":\""
+					  << networksSelectorName
+					  << "\","
 						 "\"Range\":[0,18446744073709551615]"
 						 "},{"
 						 "\"Name\":\"member\","
 						 "\"Range\":[0,18446744073709551615]"
 						 "}],"
-						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						 "\"TimeRange\":["
-						"\"MaskingKey\":\"" << maskingKey << "\","
+					  << timeRangeStart
-						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					  << ",9223372036854775807],"
+						 "\"MaskingKey\":\""
+					  << maskingKey
+					  << "\","
+						 "\"Owners\":[\""
+					  << _lfOwnerPublic
+					  << "\"]"
 						 "}";
 				auto resp = htcli.Post("/query", query.str(), "application/json");
 				if (resp) {
@@ -262,7 +291,6 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							for (std::size_t ri = 0; ri < results.size(); ++ri) {
 								nlohmann::json& rset = results[ri];
 								if ((rset.is_array()) && (! rset.empty())) {
-
 									nlohmann::json& result = rset[0];
 									if (result.is_object()) {
 										nlohmann::json& record = result["Record"];
@@ -283,28 +311,31 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 														const uint64_t prevRevision = oldMember["revision"];
 														if (prevRevision < revision)
 															_memberChanged(oldMember, member, timeRangeStart > 0);
-													} else if (hasNetwork(nwid)) {
+													}
+													else if (hasNetwork(nwid)) {
 														nlohmann::json nullJson;
 														_memberChanged(nullJson, member, timeRangeStart > 0);
 													}
-
 												}
 											}
 										}
 									}
-
 								}
 							}
 						}
-					} else {
+					}
+					else {
 						fprintf(stderr, "ERROR: LFDB: %d from node (check for member updates): %s" ZT_EOL_S, resp->status, resp->body.c_str());
 					}
-				} else {
+				}
+				else {
 					fprintf(stderr, "ERROR: LFDB: node is offline" ZT_EOL_S);
 				}
-			} catch (std::exception &e) {
+			}
+			catch (std::exception& e) {
 				fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (check for member updates): %s" ZT_EOL_S, e.what());
-			} catch ( ... ) {
+			}
+			catch (...) {
 				fprintf(stderr, "ERROR: LFDB: unexpected exception querying node (check for member updates): unknown exception" ZT_EOL_S);
 			}
 
@@ -358,7 +389,8 @@ bool LFDB::save(nlohmann::json &record,bool notifyListeners)
 				modified = true;
 			}
 		}
-	} else if (objtype == "member") {
+	}
+	else if (objtype == "member") {
 		const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL);
 		const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL);
 		if ((id) && (nwid)) {
@@ -388,7 +420,7 @@ void LFDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
 	// TODO
 }
 
-void LFDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	std::lock_guard<std::mutex> l(_state_l);
 	auto nw = _state.find(networkId);
@@ -403,4 +435,9 @@ void LFDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const I
 	}
 }
 
+void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 }	// namespace ZeroTier

controller/LFDB.hpp

@@ -16,18 +16,17 @@
 
 #include "DB.hpp"
 
+#include <atomic>
 #include <mutex>
 #include <string>
 #include <unordered_map>
-#include <atomic>
 
 namespace ZeroTier {
 
 /**
  * DB implementation for controller that stores data in LF
  */
-class LFDB : public DB
+class LFDB : public DB {
-{
   public:
 	/**
 	 * @param myId This controller's identity
@@ -47,6 +46,7 @@ public:
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
   protected:
 	const Identity _myId;
@@ -55,23 +55,19 @@ protected:
 	std::string _lfNodeHost;
 	int _lfNodePort;
 
-	struct _MemberState
+	struct _MemberState {
+		_MemberState() : lastOnlineAddress(), lastOnlineTime(0), dirty(false), lastOnlineDirty(false)
 		{
-		_MemberState() :
+		}
-			lastOnlineAddress(),
-			lastOnlineTime(0),
-			dirty(false),
-			lastOnlineDirty(false) {}
 		InetAddress lastOnlineAddress;
 		int64_t lastOnlineTime;
 		bool dirty;
 		bool lastOnlineDirty;
 	};
-	struct _NetworkState
+	struct _NetworkState {
+		_NetworkState() : members(), dirty(false)
 		{
-		_NetworkState() :
+		}
-			members(),
-			dirty(false) {}
 		std::unordered_map<uint64_t, _MemberState> members;
 		bool dirty;
 	};

controller/PostgreSQL.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2019 ZeroTier, Inc.
+ * Copyright (c)2025 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -11,186 +11,87 @@
  */
 /****/
 
-#include "DB.hpp"
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
-#ifndef ZT_CONTROLLER_LIBPQ_HPP
+#ifndef ZT_CONTROLLER_POSTGRESQL_HPP
-#define ZT_CONTROLLER_LIBPQ_HPP
+#define ZT_CONTROLLER_POSTGRESQL_HPP
-
-#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
 
 #include "ConnectionPool.hpp"
-#include <pqxx/pqxx>
+#include "DB.hpp"
 
 #include <memory>
-#include <redis++/redis++.h>
+#include <pqxx/pqxx>
 
-#include "../node/Metrics.hpp"
+namespace ZeroTier {
 
 extern "C" {
 typedef struct pg_conn PGconn;
 }
 
-namespace smeeclient {
-	struct SmeeClient;
-}
-
-namespace ZeroTier {
-
-struct RedisConfig;
-
-
 class PostgresConnection : public Connection {
   public:
-	virtual ~PostgresConnection() {
+	virtual ~PostgresConnection()
+	{
 	}
 
 	std::shared_ptr<pqxx::connection> c;
 	int a;
 };
 
-
 class PostgresConnFactory : public ConnectionFactory {
   public:
-	PostgresConnFactory(std::string &connString) 
+	PostgresConnFactory(std::string& connString) : m_connString(connString)
-		: m_connString(connString)
 	{
 	}
 
-	virtual std::shared_ptr<Connection> create() {
+	virtual std::shared_ptr<Connection> create()
+	{
 		Metrics::conn_counter++;
 		auto c = std::shared_ptr<PostgresConnection>(new PostgresConnection());
 		c->c = std::make_shared<pqxx::connection>(m_connString);
 		return std::static_pointer_cast<Connection>(c);
 	}
+
   private:
 	std::string m_connString;
 };
 
-class PostgreSQL;
-
 class MemberNotificationReceiver : public pqxx::notification_receiver {
   public:
-	MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	MemberNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel);
-	virtual ~MemberNotificationReceiver() {
+	virtual ~MemberNotificationReceiver()
+	{
 		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
 	}
 
 	virtual void operator()(const std::string& payload, int backendPid);
+
   private:
-	PostgreSQL *_psql;
+	DB* _psql;
 };
 
 class NetworkNotificationReceiver : public pqxx::notification_receiver {
   public:
-	NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	NetworkNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel);
-	virtual ~NetworkNotificationReceiver() {
+	virtual ~NetworkNotificationReceiver()
+	{
 		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
 	};
 
 	virtual void operator()(const std::string& payload, int packend_pid);
-private:
-	PostgreSQL *_psql;
-};
-
-/**
- * A controller database driver that talks to PostgreSQL
- *
- * This is for use with ZeroTier Central.  Others are free to build and use it
- * but be aware that we might change it at any time.
- */
-class PostgreSQL : public DB
-{
-	friend class MemberNotificationReceiver;
-	friend class NetworkNotificationReceiver;
-public:
-	PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
-	virtual ~PostgreSQL();
-
-	virtual bool waitForReady();
-	virtual bool isReady();
-	virtual bool save(nlohmann::json &record,bool notifyListeners);
-	virtual void eraseNetwork(const uint64_t networkId);
-	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
-	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
-	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
-
-protected:
-	struct _PairHasher
-	{
-		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
-	};
-	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
-		DB::_memberChanged(old, memberConfig, notifyListeners);
-	}
-
-	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
-		DB::_networkChanged(old, networkConfig, notifyListeners);
-	}
 
   private:
-	void initializeNetworks();
+	DB* _psql;
-	void initializeMembers();
-	void heartbeat();
-	void membersDbWatcher();
-	void _membersWatcher_Postgres();
-	void networksDbWatcher();
-	void _networksWatcher_Postgres();
-
-	void _membersWatcher_Redis();
-	void _networksWatcher_Redis();
-
-	void commitThread();
-	void onlineNotificationThread();
-	void onlineNotification_Postgres();
-	void onlineNotification_Redis();
-	uint64_t _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
-		std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
-
-	void configureSmee();
-	void notifyNewMember(const std::string &networkID, const std::string &memberID);
-
-	enum OverrideMode {
-		ALLOW_PGBOUNCER_OVERRIDE = 0,
-		NO_OVERRIDE = 1
 };
 
-	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+struct NodeOnlineRecord {
-
+	uint64_t lastSeen;
-	const Identity _myId;
+	InetAddress physicalAddress;
-	const Address _myAddress;
+	std::string osArch;
-	std::string _myAddressStr;
-	std::string _connString;
-
-	BlockingQueue< std::pair<nlohmann::json,bool> > _commitQueue;
-
-	std::thread _heartbeatThread;
-	std::thread _membersDbWatcher;
-	std::thread _networksDbWatcher;
-	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
-	std::thread _onlineNotificationThread;
-
-	std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline;
-
-	mutable std::mutex _lastOnline_l;
-	mutable std::mutex _readyLock;
-	std::atomic<int> _ready, _connected, _run;
-	mutable volatile bool _waitNoticePrinted;
-
-	int _listenPort;
-	uint8_t _ssoPsk[48];
-
-	RedisConfig *_rc;
-	std::shared_ptr<sw::redis::Redis> _redis;
-	std::shared_ptr<sw::redis::RedisCluster> _cluster;
-    bool _redisMemberStatus;
-
-	smeeclient::SmeeClient *_smee;
 };
 
 }	// namespace ZeroTier
 
-#endif // ZT_CONTROLLER_LIBPQ_HPP
+#endif	 // ZT_CONTROLLER_POSTGRESQL_HPP
 
 #endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/Redis.hpp

@@ -10,6 +10,6 @@ struct RedisConfig {
 	std::string password;
 	bool clusterMode;
 };
-}
+}	// namespace ZeroTier
 
 #endif

debian/changelog

@@ -1,3 +1,15 @@
+zerotier-one (1.14.2) unstable; urgency=medium
+
+  * See RELEASE-NOTES.md for release notes.
+
+ -- Adam Ierymenko <adam.ierymenko@zerotier.com>  Wed, 23 Oct 2024 01:00:00 -0700
+
+zerotier-one (1.14.1) unstable; urgency=medium
+
+  * See RELEASE-NOTES.md for release notes.
+
+ -- Adam Ierymenko <adam.ierymenko@zerotier.com>  Wed, 11 Sep 2024 01:00:00 -0700
+
 zerotier-one (1.14.0) unstable; urgency=medium
 
   * See RELEASE-NOTES.md for release notes.

entrypoint.sh.release

@@ -9,15 +9,16 @@ mkztfile() {
   file=$1
   mode=$2
   content=$3
-
+  echo "creating $file"
   mkdir -p /var/lib/zerotier-one
-  echo "$content" > "/var/lib/zerotier-one/$file"
+  echo -n "$content" > "/var/lib/zerotier-one/$file"
   chmod "$mode" "/var/lib/zerotier-one/$file"
 }
 
 if [ "x$ZEROTIER_API_SECRET" != "x" ]
 then
   mkztfile authtoken.secret 0600 "$ZEROTIER_API_SECRET"
+  mkztfile metricstoken.secret 0600 "$ZEROTIER_API_SECRET"
 fi
 
 if [ "x$ZEROTIER_IDENTITY_PUBLIC" != "x" ]
@@ -30,6 +31,11 @@ then
   mkztfile identity.secret 0600 "$ZEROTIER_IDENTITY_SECRET"
 fi
 
+if [ "x$ZEROTIER_LOCAL_CONF" != "x" ]
+then
+  mkztfile local.conf 0644 "$ZEROTIER_LOCAL_CONF"
+fi
+
 mkztfile zerotier-one.port 0600 "9993"
 
 killzerotier() {

ext/central-controller-docker/Dockerfile

@@ -1,11 +1,16 @@
 # Dockerfile for ZeroTier Central Controllers
-FROM registry.zerotier.com/zerotier/ctlbuild:latest as builder
+FROM registry.zerotier.com/zerotier/ctlbuild:2025-05-13-01 AS builder
-MAINTAINER Adam Ierymekno <adam.ierymenko@zerotier.com>, Grant Limberg <grant.limberg@zerotier.com>
 ADD . /ZeroTierOne
 RUN export PATH=$PATH:~/.cargo/bin && cd ZeroTierOne && make clean && make central-controller -j8
 
-FROM registry.zerotier.com/zerotier/ctlrun:latest
+FROM golang:bookworm AS go_base
+RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
+
+FROM registry.zerotier.com/zerotier/ctlrun:2025-05-13-01
 COPY --from=builder /ZeroTierOne/zerotier-one /usr/local/bin/zerotier-one
+COPY --from=go_base /go/bin/migrate /usr/local/bin/migrate
+COPY ext/central-controller-docker/migrations /migrations
+
 RUN chmod a+x /usr/local/bin/zerotier-one
 RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/usr-local-lib64.conf && ldconfig
 

ext/central-controller-docker/Dockerfile.builder

@@ -1,8 +1,5 @@
 # Dockerfile for building ZeroTier Central Controllers
-FROM ubuntu:jammy as builder
+FROM debian:bookworm
-MAINTAINER Adam Ierymekno <adam.ierymenko@zerotier.com>, Grant Limberg <grant.limberg@zerotier.com>
-
-ARG git_branch=master
 
 RUN apt update && apt upgrade -y
 RUN apt -y install \

ext/central-controller-docker/Dockerfile.run_base

@@ -1,15 +1,17 @@
-FROM ubuntu:jammy
+FROM debian:bookworm
+
+
 
 RUN apt update && apt upgrade -y
-
 RUN apt -y install \
-    netcat \
+    netcat-traditional \
     postgresql-client \
     postgresql-client-common \
     libjemalloc2 \
     libpq5 \
     curl \
     binutils \
-    linux-tools-gke \
     perf-tools-unstable \
-    google-perftools 
+    google-perftools \
+    gnupg
+

ext/central-controller-docker/main.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-if [ -z "$ZT_IDENTITY_PATH" ]; then
-    echo '*** FAILED: ZT_IDENTITY_PATH environment variable is not defined'
-    exit 1
-fi
 if [ -z "$ZT_DB_HOST" ]; then
     echo '*** FAILED: ZT_DB_HOST environment variable not defined'
     exit 1
@@ -24,6 +20,9 @@ if [ -z "$ZT_DB_PASSWORD" ]; then
     echo '*** FAILED: ZT_DB_PASSWORD environment variable not defined'
     exit 1
 fi
+if [ -z "$ZT_DB_TYPE" ]; then
+    ZT_DB_TYPE="postgres"
+fi
 
 REDIS=""
 if [ "$ZT_USE_REDIS" == "true" ]; then
@@ -56,11 +55,15 @@ fi
 mkdir -p /var/lib/zerotier-one
 
 pushd /var/lib/zerotier-one
+if [ -d "$ZT_IDENTITY_PATH" ]; then
+    echo '*** Using existing ZT identity from path $ZT_IDENTITY_PATH'
+
     ln -s $ZT_IDENTITY_PATH/identity.public identity.public
     ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret
     if [ -f  "$ZT_IDENTITY_PATH/authtoken.secret" ]; then
         ln -s $ZT_IDENTITY_PATH/authtoken.secret authtoken.secret
     fi
+fi
 popd
 
 DEFAULT_PORT=9993
@@ -70,7 +73,7 @@ APP_NAME="controller-$(cat /var/lib/zerotier-one/identity.public | cut -d ':' -f
 
 echo "{
     \"settings\": {
-        \"controllerDbPath\": \"postgres:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\",
+        \"controllerDbPath\": \"${ZT_DB_TYPE}:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\",
         \"portMappingEnabled\": true,
         \"softwareUpdate\": \"disable\",
         \"interfacePrefixBlacklist\": [
@@ -100,6 +103,15 @@ else
     done
 fi
 
+if [ "$ZT_DB_TYPE" == "cv2" ]; then
+    echo "Migrating database (if needed)..."
+    if [ -n "$DB_SERVER_CA" ]; then
+        /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=verify-full&sslrootcert=$DB_SERVER_CA&sslcert=$DB_CLIENT_CERT&sslkey=$DB_CLIENT_KEY" up  
+    else 
+        /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=disable" up
+    fi
+fi
+
 if [ -n "$ZT_TEMPORAL_HOST" ] && [ -n "$ZT_TEMPORAL_PORT" ]; then
     echo "waiting for temporal..."
     while ! nc -z ${ZT_TEMPORAL_HOST} ${ZT_TEMPORAL_PORT}; do

ext/central-controller-docker/migrations/0001_init.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS network_memberships_ctl;
+DROP TABLE IF EXISTS networks_ctl;
+DROP TABLE IF EXISTS controllers_ctl;

ext/central-controller-docker/migrations/0001_init.up.sql

@@ -0,0 +1,47 @@
+-- inits controller db schema
+
+CREATE TABLE IF NOT EXISTS controllers_ctl (
+    id text NOT NULL PRIMARY KEY,
+    hostname text,
+    last_heartbeat timestamp with time zone,
+    public_identity text NOT NULL,
+    version text
+);
+
+CREATE TABLE IF NOT EXISTS networks_ctl (
+    id character varying(22) NOT NULL PRIMARY KEY,
+    name text NOT NULL,
+    configuration jsonb DEFAULT '{}'::jsonb NOT NULL,
+    controller_id text REFERENCES controllers_ctl(id),
+    revision integer DEFAULT 0 NOT NULL,
+    last_modified timestamp with time zone DEFAULT now(),
+    creation_time timestamp with time zone DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS network_memberships_ctl (
+    device_id character varying(22) NOT NULL,
+    network_id character varying(22) NOT NULL REFERENCES networks_ctl(id),
+    authorized boolean,
+    active_bridge boolean,
+    ip_assignments text[],
+    no_auto_assign_ips boolean,
+    sso_exempt boolean,
+    authentication_expiry_time timestamp with time zone,
+    capabilities jsonb,
+    creation_time timestamp with time zone DEFAULT now(),
+    last_modified timestamp with time zone DEFAULT now(),
+    identity text DEFAULT ''::text,
+    last_authorized_credential text,
+    last_authorized_time timestamp with time zone,
+    last_deauthorized_time timestamp with time zone,
+    last_seen jsonb DEFAULT '{}'::jsonb NOT NULL, -- in the context of the network
+    remote_trace_level integer DEFAULT 0 NOT NULL,
+    remote_trace_target text DEFAULT ''::text NOT NULL,
+    revision integer DEFAULT 0 NOT NULL,
+    tags jsonb,
+    version_major integer DEFAULT 0 NOT NULL,
+    version_minor integer DEFAULT 0 NOT NULL,
+    version_revision integer DEFAULT 0 NOT NULL,
+    version_protocol integer DEFAULT 0 NOT NULL,
+    PRIMARY KEY (device_id, network_id)
+);

ext/central-controller-docker/migrations/0002_os_arch.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE network_memberships_ctl
+    DROP COLUMN os,
+    DROP COLUMN arch;

ext/central-controller-docker/migrations/0002_os_arch.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE network_memberships_ctl
+    ADD COLUMN os TEXT NOT NULL DEFAULT 'unknown',
+    ADD COLUMN arch TEXT NOT NULL DEFAULT 'unknown';

ext/installfiles/mac/ZeroTier One.pkgproj

@@ -701,7 +701,7 @@
 				<key>USE_HFS+_COMPRESSION</key>
 				<false/>
 				<key>VERSION</key>
-				<string>1.14.0</string>
+				<string>1.14.2</string>
 			</dict>
 			<key>TYPE</key>
 			<integer>0</integer>

ext/installfiles/windows/ZeroTier One.aip

@@ -24,10 +24,10 @@
     <ROW Property="AiFeatIcoZeroTierOne" Value="ZeroTierIcon.exe" Type="8"/>
     <ROW Property="MSIFASTINSTALL" MultiBuildValue="DefaultBuild:2"/>
     <ROW Property="Manufacturer" Value="ZeroTier, Inc."/>
-    <ROW Property="ProductCode" Value="1033:{EC58088A-4E0F-4BD5-B0B2-FD81C803EEC4} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{0143A36C-46C6-458D-AB9B-C8843E089323} " Type="16"/>
     <ROW Property="ProductLanguage" Value="1033"/>
     <ROW Property="ProductName" Value="ZeroTier One"/>
-    <ROW Property="ProductVersion" Value="1.14.0" Options="32"/>
+    <ROW Property="ProductVersion" Value="1.14.2" Options="32"/>
     <ROW Property="REBOOT" MultiBuildValue="DefaultBuild:ReallySuppress"/>
     <ROW Property="SecureCustomProperties" Value="OLDPRODUCTS;AI_NEWERPRODUCTFOUND;AI_SETUPEXEPATH;SETUPEXEDIR"/>
     <ROW Property="UpgradeCode" Value="{B0E2A5F3-88B6-4E77-B922-CB4739B4C4C8}"/>
@@ -62,7 +62,7 @@
     <ROW Directory="regid.201001.com.zerotier_Dir" Directory_Parent="CommonAppDataFolder" DefaultDir="REGID2~1.ZER|regid.2010-01.com.zerotier" DirectoryOptions="12"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiCompsComponent">
-    <ROW Component="AI_CustomARPName" ComponentId="{8BC01817-02AC-4C44-A84C-0727BC5B6E22}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{DFE7A60C-C2B9-41F6-9171-8955BA30E556}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_DisableModify" ComponentId="{46FFA8C5-A0CB-4E05-9AD3-911D543DE8CA}" Directory_="APPDIR" Attributes="4" KeyPath="NoModify" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{8E02B36C-7A19-429B-A93E-77A9261AC918}" Directory_="APPDIR" Attributes="4" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{4DD7907D-D7FE-4CD6-B1A0-B5C1625F5133}" Directory_="APPDIR" Attributes="0"/>
@@ -498,7 +498,7 @@
     <ROW XmlAttribute="xsischemaLocation" XmlElement="swidsoftware_identification_tag" Name="xsi:schemaLocation" Flags="14" Order="3" Value="http://standards.iso.org/iso/19770/-2/2008/schema.xsd software_identification_tag.xsd"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.XmlElementComponent">
-    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="0" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="2" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="14" UpdateIndexInParent="0"/>

ext/prometheus-cpp-lite-1.0/core/include/prometheus/client_metric.h

@@ -8,6 +8,7 @@
 namespace prometheus {
 
   // структура, в которую копируются значения метрик перед их сериализацией
+  
   struct ClientMetric {
 
     // Label

include/ZeroTierOne.h

@@ -84,7 +84,7 @@ extern "C" {
 /**
  * Minimum UDP payload size allowed
  */
-#define ZT_MIN_PHYSMTU 1400
+#define ZT_MIN_PHYSMTU 510
 
 /**
  * Maximum physical interface name length. This number is gigantic because of Windows.

java/jni/com_zerotierone_sdk_Node.cpp

@@ -111,6 +111,44 @@ namespace {
         bool finishInitializing();
     };
 
+    //
+    // RAII construct for calling AttachCurrentThread and DetachCurrent automatically
+    //
+    struct ScopedJNIThreadAttacher {
+
+        JavaVM *jvm;
+        JNIEnv **env_p;
+        jint getEnvRet;
+
+        ScopedJNIThreadAttacher(JavaVM *jvmIn, JNIEnv **env_pIn, jint getEnvRetIn) :
+        jvm(jvmIn),
+        env_p(env_pIn),
+        getEnvRet(getEnvRetIn) {
+
+            if (getEnvRet != JNI_EDETACHED) {
+                return;
+            }
+
+            jint attachCurrentThreadRet;
+            if ((attachCurrentThreadRet = jvm->AttachCurrentThread(env_p, NULL)) != JNI_OK) {
+                LOGE("Error calling AttachCurrentThread: %d", attachCurrentThreadRet);
+                assert(false && "Error calling AttachCurrentThread");
+            }
+        }
+
+        ~ScopedJNIThreadAttacher() {
+
+            if (getEnvRet != JNI_EDETACHED) {
+                return;
+            }
+
+            jint detachCurrentThreadRet;
+            if ((detachCurrentThreadRet = jvm->DetachCurrentThread()) != JNI_OK) {
+                LOGE("Error calling DetachCurrentThread: %d", detachCurrentThreadRet);
+                assert(false && "Error calling DetachCurrentThread");
+            }
+        }
+    };
 
     /*
     * This must return 0 on success. It can return any OS-dependent error code
@@ -194,7 +232,25 @@ namespace {
         assert(ref);
         assert(ref->node == node);
         JNIEnv *env;
-        GETENV(env, ref->jvm);
+        
+        jint getEnvRet;
+        assert(ref->jvm);
+        getEnvRet = ref->jvm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6);
+
+        if (!(getEnvRet == JNI_OK || getEnvRet == JNI_EDETACHED)) {
+            LOGE("Error calling GetEnv: %d", getEnvRet);
+            assert(false && "Error calling GetEnv");
+        }
+
+        //
+        // Thread might actually be detached.
+        //
+        // e.g:
+        // https://github.com/zerotier/ZeroTierOne/blob/91e7ce87f09ac1cfdeaf6ff22c3cedcd93574c86/node/Switch.cpp#L519
+        //
+        // Make sure to attach if needed
+        //
+        ScopedJNIThreadAttacher attacher{ref->jvm, &env, getEnvRet};
 
         if (env->ExceptionCheck()) {
             LOGE("Unhandled pending exception");

make-linux.mk

@@ -71,7 +71,7 @@ else
 	override CFLAGS+=-Wall -Wno-deprecated -pthread $(INCLUDES) -DNDEBUG $(DEFS)
 	CXXFLAGS?=-O3 -fstack-protector
 	override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS)
-	LDFLAGS=-pie -Wl,-z,relro,-z,now
+	LDFLAGS?=-pie -Wl,-z,relro,-z,now
 	ZT_CARGO_FLAGS=--release
 endif
 
@@ -364,7 +364,7 @@ override CFLAGS+=-fPIC -fPIE
 override CXXFLAGS+=-fPIC -fPIE
 
 # Non-executable stack
-override ASFLAGS+=--noexecstack
+override LDFLAGS+=-Wl,-z,noexecstack
 
 .PHONY: all
 all:	one
@@ -431,6 +431,10 @@ central-controller-docker: _buildx FORCE
 	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
 	@echo Image: registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP}
 
+centralv2-controller-docker: _buildx FORCE
+	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:$(shell git rev-parse --short HEAD) -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
+	@echo Image: us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:$(shell git rev-parse --short HEAD)
+
 debug:	FORCE
 	make ZT_DEBUG=1 one
 	make ZT_DEBUG=1 selftest

make-mac.mk

@@ -1,8 +1,8 @@
 CC=clang
 CXX=clang++
-TOPDIR=$(shell PWD)
+TOPDIR=$(shell pwd)
 
-INCLUDES=-I$(shell PWD)/rustybits/target -isystem $(TOPDIR)/ext  -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/core/include -I$(TOPDIR)/ext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/simpleapi/include
+INCLUDES=-I$(shell pwd)/rustybits/target -isystem $(TOPDIR)/ext  -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/core/include -I$(TOPDIR)/ext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/simpleapi/include
 DEFS=
 LIBS=
 ARCH_FLAGS=-arch x86_64 -arch arm64 
@@ -57,9 +57,9 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec
 ifeq ($(ZT_CONTROLLER),1)
 	MACOS_VERSION_MIN=10.15
 	override CXXFLAGS=$(CFLAGS) -std=c++17 -stdlib=libc++
-	LIBS+=-L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a
+	LIBS+=-L/opt/homebrew/lib -L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a rustybits/target/libsmeeclient.a
 	DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER 
-	INCLUDES+=-I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/
+	INCLUDES+=-I/opt/homebrew/include -I/opt/homebrew/opt/libpq/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ -Irustybits/target/
 else
 	MACOS_VERSION_MIN=10.13
 endif
@@ -115,7 +115,11 @@ mac-agent: FORCE
 osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm
 	$(CXX) $(CXXFLAGS) -c osdep/MacDNSHelper.mm -o osdep/MacDNSHelper.o 
 
+ifeq ($(ZT_CONTROLLER),1)
+one:	zeroidc smeeclient $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+else
 one:	zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+endif
 	$(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/libzeroidc.a
 	# $(STRIP) zerotier-one
 	ln -sf zerotier-one zerotier-idtool
@@ -126,6 +130,15 @@ zerotier-one: one
 
 zeroidc: rustybits/target/libzeroidc.a
 
+ifeq ($(ZT_CONTROLLER),1)
+smeeclient: rustybits/target/libsmeeclient.a
+
+rustybits/target/libsmeeclient.a:	FORCE
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a -output target/libsmeeclient.a
+endif
+
 rustybits/target/libzeroidc.a:	FORCE
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
@@ -195,6 +208,13 @@ central-controller-docker: _buildx FORCE
 	docker buildx build --platform linux/arm64,linux/amd64 --no-cache -t registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=$(shell git name-rev --name-only HEAD) . --push
 	@echo Image: registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP}
 
+centralv2-controller-docker: _buildx FORCE
+	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t us-central1-docker.pkg.dev/zerotier-d648c7/central-v2/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
+	@echo Image: us-central1-docker.pkg.dev/zerotier-d648c7/central-v2/ztcentral-controller:${TIMESTAMP} 
+
+docker-release:	_buildx
+	docker buildx build --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/s390x -t zerotier/zerotier:${RELEASE_DOCKER_TAG} -t zerotier/zerotier:latest --build-arg VERSION=${RELEASE_VERSION} -f Dockerfile.release . --push
+	
 clean:
 	rm -rf MacEthernetTapAgent *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* rustybits/target/
 

node/AES.cpp

@@ -11,9 +11,10 @@
  */
 /****/
 
-#include "Constants.hpp"
 #include "AES.hpp"
 
+#include "Constants.hpp"
+
 #ifdef __GNUC__
 #pragma GCC diagnostic ignored "-Wstrict-aliasing"
 #endif
@@ -343,7 +344,8 @@ void AES::CTR::crypt(const void *const input, unsigned int len) noexcept
 				*reinterpret_cast<uint64_t*>(out + 8) = in1 ^ Utils::hton(((uint64_t)s2 << 32U) | (uint64_t)s3);
 				out += 16;
 			} while ((len -= 16) >= 16);
-		} else {
+		}
+		else {
 			do {
 				uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
 				s0 = ctr0rk0;
@@ -454,34 +456,62 @@ void AES::CTR::finish() noexcept
 
 // Software AES and AES key expansion ---------------------------------------------------------------------------------
 
-const uint32_t AES::Te0[256] = {0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b, 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153,
+const uint32_t AES::Te0[256] = { 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a,
-                                0x2a15153f, 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f,
+								 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b,
-                                0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497, 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba,
+								 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f,
-                                0x4ba8a8e3, 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d,
+								 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f,
-                                0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437,
+								 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497,
-                                0xf279798b, 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818,
+								 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a,
-                                0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9, 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878,
+								 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3,
-                                0xa5dfdf7a, 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a};
+								 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d,
-const uint32_t AES::Te4[256] = {0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b, 0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5, 0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b, 0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676, 0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d, 0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0, 0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf, 0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0, 0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626, 0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc, 0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1, 0x71717171, 0xd8d8d8d8, 0x31313131,
+								 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395,
-                                0x15151515, 0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3, 0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a, 0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2, 0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575,
+								 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76,
-                                0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a, 0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0, 0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3, 0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484, 0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed, 0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b, 0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939, 0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf, 0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb, 0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585, 0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f, 0x50505050, 0x3c3c3c3c, 0x9f9f9f9f,
+								 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b,
-                                0xa8a8a8a8, 0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f, 0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5, 0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121, 0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2,
+								 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818,
-                                0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec, 0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717, 0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d, 0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373, 0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc, 0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888, 0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414, 0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb, 0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a, 0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c, 0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262, 0x91919191, 0x95959595, 0xe4e4e4e4,
+								 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85,
-                                0x79797979, 0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d, 0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9, 0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea, 0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808,
+								 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9,
-                                0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e, 0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6, 0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f, 0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a, 0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666, 0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e, 0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9, 0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e, 0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111, 0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494, 0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9, 0xcececece, 0x55555555, 0x28282828,
+								 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a,
-                                0xdfdfdfdf, 0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d, 0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868, 0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f, 0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616};
+								 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a };
-const uint32_t AES::Td0[256] = {0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f, 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844, 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c,
+const uint32_t AES::Te4[256] = { 0x63636363, 0x7c7c7c7c, 0x77777777, 0x7b7b7b7b, 0xf2f2f2f2, 0x6b6b6b6b, 0x6f6f6f6f, 0xc5c5c5c5, 0x30303030, 0x01010101, 0x67676767, 0x2b2b2b2b, 0xfefefefe, 0xd7d7d7d7, 0xabababab, 0x76767676,
-                                0xf9082b94, 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c,
+								 0xcacacaca, 0x82828282, 0xc9c9c9c9, 0x7d7d7d7d, 0xfafafafa, 0x59595959, 0x47474747, 0xf0f0f0f0, 0xadadadad, 0xd4d4d4d4, 0xa2a2a2a2, 0xafafafaf, 0x9c9c9c9c, 0xa4a4a4a4, 0x72727272, 0xc0c0c0c0,
-                                0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051, 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb, 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1,
+								 0xb7b7b7b7, 0xfdfdfdfd, 0x93939393, 0x26262626, 0x36363636, 0x3f3f3f3f, 0xf7f7f7f7, 0xcccccccc, 0x34343434, 0xa5a5a5a5, 0xe5e5e5e5, 0xf1f1f1f1, 0x71717171, 0xd8d8d8d8, 0x31313131, 0x15151515,
-                                0x24362e3a, 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8,
+								 0x04040404, 0xc7c7c7c7, 0x23232323, 0xc3c3c3c3, 0x18181818, 0x96969696, 0x05050505, 0x9a9a9a9a, 0x07070707, 0x12121212, 0x80808080, 0xe2e2e2e2, 0xebebebeb, 0x27272727, 0xb2b2b2b2, 0x75757575,
-                                0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120, 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef, 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e,
+								 0x09090909, 0x83838383, 0x2c2c2c2c, 0x1a1a1a1a, 0x1b1b1b1b, 0x6e6e6e6e, 0x5a5a5a5a, 0xa0a0a0a0, 0x52525252, 0x3b3b3b3b, 0xd6d6d6d6, 0xb3b3b3b3, 0x29292929, 0xe3e3e3e3, 0x2f2f2f2f, 0x84848484,
-                                0x82c3aff5, 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6,
+								 0x53535353, 0xd1d1d1d1, 0x00000000, 0xedededed, 0x20202020, 0xfcfcfcfc, 0xb1b1b1b1, 0x5b5b5b5b, 0x6a6a6a6a, 0xcbcbcbcb, 0xbebebebe, 0x39393939, 0x4a4a4a4a, 0x4c4c4c4c, 0x58585858, 0xcfcfcfcf,
-                                0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f, 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713, 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14,
+								 0xd0d0d0d0, 0xefefefef, 0xaaaaaaaa, 0xfbfbfbfb, 0x43434343, 0x4d4d4d4d, 0x33333333, 0x85858585, 0x45454545, 0xf9f9f9f9, 0x02020202, 0x7f7f7f7f, 0x50505050, 0x3c3c3c3c, 0x9f9f9f9f, 0xa8a8a8a8,
-                                0x7844db86, 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742};
+								 0x51515151, 0xa3a3a3a3, 0x40404040, 0x8f8f8f8f, 0x92929292, 0x9d9d9d9d, 0x38383838, 0xf5f5f5f5, 0xbcbcbcbc, 0xb6b6b6b6, 0xdadadada, 0x21212121, 0x10101010, 0xffffffff, 0xf3f3f3f3, 0xd2d2d2d2,
-const uint8_t AES::Td4[256] = {0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d,
+								 0xcdcdcdcd, 0x0c0c0c0c, 0x13131313, 0xecececec, 0x5f5f5f5f, 0x97979797, 0x44444444, 0x17171717, 0xc4c4c4c4, 0xa7a7a7a7, 0x7e7e7e7e, 0x3d3d3d3d, 0x64646464, 0x5d5d5d5d, 0x19191919, 0x73737373,
-                               0x84, 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+								 0x60606060, 0x81818181, 0x4f4f4f4f, 0xdcdcdcdc, 0x22222222, 0x2a2a2a2a, 0x90909090, 0x88888888, 0x46464646, 0xeeeeeeee, 0xb8b8b8b8, 0x14141414, 0xdededede, 0x5e5e5e5e, 0x0b0b0b0b, 0xdbdbdbdb,
-                               0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c,
+								 0xe0e0e0e0, 0x32323232, 0x3a3a3a3a, 0x0a0a0a0a, 0x49494949, 0x06060606, 0x24242424, 0x5c5c5c5c, 0xc2c2c2c2, 0xd3d3d3d3, 0xacacacac, 0x62626262, 0x91919191, 0x95959595, 0xe4e4e4e4, 0x79797979,
-                               0xef, 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d};
+								 0xe7e7e7e7, 0xc8c8c8c8, 0x37373737, 0x6d6d6d6d, 0x8d8d8d8d, 0xd5d5d5d5, 0x4e4e4e4e, 0xa9a9a9a9, 0x6c6c6c6c, 0x56565656, 0xf4f4f4f4, 0xeaeaeaea, 0x65656565, 0x7a7a7a7a, 0xaeaeaeae, 0x08080808,
+								 0xbabababa, 0x78787878, 0x25252525, 0x2e2e2e2e, 0x1c1c1c1c, 0xa6a6a6a6, 0xb4b4b4b4, 0xc6c6c6c6, 0xe8e8e8e8, 0xdddddddd, 0x74747474, 0x1f1f1f1f, 0x4b4b4b4b, 0xbdbdbdbd, 0x8b8b8b8b, 0x8a8a8a8a,
+								 0x70707070, 0x3e3e3e3e, 0xb5b5b5b5, 0x66666666, 0x48484848, 0x03030303, 0xf6f6f6f6, 0x0e0e0e0e, 0x61616161, 0x35353535, 0x57575757, 0xb9b9b9b9, 0x86868686, 0xc1c1c1c1, 0x1d1d1d1d, 0x9e9e9e9e,
+								 0xe1e1e1e1, 0xf8f8f8f8, 0x98989898, 0x11111111, 0x69696969, 0xd9d9d9d9, 0x8e8e8e8e, 0x94949494, 0x9b9b9b9b, 0x1e1e1e1e, 0x87878787, 0xe9e9e9e9, 0xcececece, 0x55555555, 0x28282828, 0xdfdfdfdf,
+								 0x8c8c8c8c, 0xa1a1a1a1, 0x89898989, 0x0d0d0d0d, 0xbfbfbfbf, 0xe6e6e6e6, 0x42424242, 0x68686868, 0x41414141, 0x99999999, 0x2d2d2d2d, 0x0f0f0f0f, 0xb0b0b0b0, 0x54545454, 0xbbbbbbbb, 0x16161616 };
+const uint32_t AES::Td0[256] = { 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96, 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393, 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25, 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f,
+								 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1, 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6, 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da, 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844,
+								 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd, 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4, 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45, 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94,
+								 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7, 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a, 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5, 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c,
+								 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1, 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a, 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75, 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051,
+								 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46, 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff, 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77, 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb,
+								 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000, 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e, 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927, 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a,
+								 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e, 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16, 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d, 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8,
+								 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd, 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34, 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163, 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120,
+								 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d, 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0, 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422, 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef,
+								 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36, 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4, 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662, 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5,
+								 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3, 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b, 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8, 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6,
+								 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6, 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0, 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815, 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f,
+								 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df, 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f, 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e, 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713,
+								 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89, 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c, 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf, 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86,
+								 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f, 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541, 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190, 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742 };
+const uint8_t AES::Td4[256] = { 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
+								0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
+								0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+								0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+								0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
+								0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+								0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
+								0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d };
 const uint32_t AES::rcon[15] = { 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000, 0x40000000, 0x80000000, 0x1B000000, 0x36000000, 0x6c000000, 0xd8000000, 0xab000000, 0x4d000000, 0x9a000000 };
 
 void AES::p_initSW(const uint8_t* key) noexcept

node/AES.hpp

@@ -15,8 +15,8 @@
 #define ZT_AES_HPP
 
 #include "Constants.hpp"
-#include "Utils.hpp"
 #include "SHA512.hpp"
+#include "Utils.hpp"
 
 // Uncomment to disable all hardware acceleration (usually for testing)
 // #define ZT_AES_NO_ACCEL
@@ -40,8 +40,7 @@ namespace ZeroTier {
  * This includes hardware acceleration for certain processors. The software
  * mode is fallback and is significantly slower.
  */
-class AES
+class AES {
-{
   public:
 	/**
 	 * @return True if this system has hardware AES acceleration
@@ -63,7 +62,8 @@ public:
 	 * Create an un-initialized AES instance (must call init() before use)
 	 */
 	ZT_INLINE AES() noexcept
-	{}
+	{
+	}
 
 	/**
 	 * Create an AES instance with the given key
@@ -71,10 +71,14 @@ public:
 	 * @param key 256-bit key
 	 */
 	explicit ZT_INLINE AES(const void* const key) noexcept
-	{ this->init(key); }
+	{
+		this->init(key);
+	}
 
 	ZT_INLINE ~AES()
-	{ Utils::burn(&p_k, sizeof(p_k)); }
+	{
+		Utils::burn(&p_k, sizeof(p_k));
+	}
 
 	/**
 	 * Set (or re-set) this AES256 cipher's key
@@ -150,8 +154,7 @@ public:
 	/**
 	 * Streaming GMAC calculator
 	 */
-	class GMAC
+	class GMAC {
-	{
 		friend class GMACSIVEncryptor;
 		friend class GMACSIVDecryptor;
 
@@ -178,7 +181,8 @@ public:
 		 * @param aes Keyed AES instance to use
 		 */
 		ZT_INLINE GMAC(const AES& aes) : _aes(aes)
-		{}
+		{
+		}
 
 		/**
 		 * Reset and initialize for a new GMAC calculation
@@ -249,14 +253,14 @@ public:
 	 * NOTE: this doesn't support overflow of the counter in the least significant 32 bits.
 	 * AES-GMAC-CTR doesn't need this, so we don't support it as an optimization.
 	 */
-	class CTR
+	class CTR {
-	{
 		friend class GMACSIVEncryptor;
 		friend class GMACSIVDecryptor;
 
 	  public:
 		ZT_INLINE CTR(const AES& aes) noexcept : _aes(aes)
-		{}
+		{
+		}
 
 		/**
 		 * Initialize this CTR instance to encrypt a new stream
@@ -325,8 +329,7 @@ public:
 	 * This supports encryption of a maximum of 2^31 bytes of data per
 	 * call to init().
 	 */
-	class GMACSIVEncryptor
+	class GMACSIVEncryptor {
-	{
 	  public:
 		/**
 		 * Create a new AES-GMAC-SIV encryptor keyed with the provided AES instances
@@ -334,10 +337,11 @@ public:
 		 * @param k0 First of two AES instances keyed with K0
 		 * @param k1 Second of two AES instances keyed with K1
 		 */
-		ZT_INLINE GMACSIVEncryptor(const AES &k0, const AES &k1) noexcept :
+		ZT_INLINE GMACSIVEncryptor(const AES& k0, const AES& k1) noexcept
-			_gmac(k0),
+			: _gmac(k0)
-			_ctr(k1)
+			, _ctr(k1)
-		{}
+		{
+		}
 
 		/**
 		 * Initialize AES-GMAC-SIV
@@ -386,7 +390,9 @@ public:
 		 * @param len Length of plaintext chunk
 		 */
 		ZT_INLINE void update1(const void* const input, const unsigned int len) noexcept
-		{ _gmac.update(input, len); }
+		{
+			_gmac.update(input, len);
+		}
 
 		/**
 		 * Finish first pass, compute CTR IV, initialize second pass.
@@ -428,7 +434,9 @@ public:
 		 * @param len Length of plaintext chunk
 		 */
 		ZT_INLINE void update2(const void* const input, const unsigned int len) noexcept
-		{ _ctr.crypt(input, len); }
+		{
+			_ctr.crypt(input, len);
+		}
 
 		/**
 		 * Finish second pass and return a pointer to the opaque 128-bit IV+MAC block
@@ -456,13 +464,13 @@ public:
 	 *
 	 * GMAC-SIV decryption is single-pass. AAD (if any) must be processed first.
 	 */
-	class GMACSIVDecryptor
+	class GMACSIVDecryptor {
-	{
 	  public:
-		ZT_INLINE GMACSIVDecryptor(const AES &k0, const AES &k1) noexcept:
+		ZT_INLINE GMACSIVDecryptor(const AES& k0, const AES& k1) noexcept
-			_ctr(k1),
+			: _ctr(k1)
-			_gmac(k0)
+			, _gmac(k0)
-		{}
+		{
+		}
 
 		/**
 		 * Initialize decryptor for a new message
@@ -550,11 +558,9 @@ private:
 	void p_encryptSW(const uint8_t* in, uint8_t* out) const noexcept;
 	void p_decryptSW(const uint8_t* in, uint8_t* out) const noexcept;
 
-	union
+	union {
-	{
 #ifdef ZT_AES_AESNI
-		struct
+		struct {
-		{
 			__m128i k[28];
 			__m128i h[4];	 // h, hh, hhh, hhhh
 			__m128i h2[4];	 // _mm_xor_si128(_mm_shuffle_epi32(h, 78), h), etc.
@@ -562,8 +568,7 @@ private:
 #endif
 
 #ifdef ZT_AES_NEON
-		struct
+		struct {
-		{
 			uint64_t hsw[2];   // in case it has AES but not PMULL, not sure if that ever happens
 			uint8x16_t ek[15];
 			uint8x16_t dk[15];
@@ -571,8 +576,7 @@ private:
 		} neon;
 #endif
 
-		struct
+		struct {
-		{
 			uint64_t h[2];
 			uint32_t ek[60];
 			uint32_t dk[60];

node/AES_aesni.cpp

@@ -11,8 +11,8 @@
  */
 /****/
 
-#include "Constants.hpp"
 #include "AES.hpp"
+#include "Constants.hpp"
 
 #ifdef ZT_AES_AESNI
 
@@ -29,7 +29,8 @@ const __m128i s_sseSwapBytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
 #ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul")))
 #endif
-__m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
+__m128i
+p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
 {
 	y = _mm_shuffle_epi8(y, s_sseSwapBytes);
 	__m128i t1 = _mm_clmulepi64_si128(h, y, 0x00);
@@ -81,11 +82,7 @@ void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, co
 	const __m512i kk14 = _mm512_broadcast_i32x4(k[14]);
 	do {
 		__m512i p0 = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(in));
-		__m512i d0 = _mm512_set_epi64(
+		__m512i d0 = _mm512_set_epi64((long long)Utils::hton(c1 + 3ULL), (long long)c0, (long long)Utils::hton(c1 + 2ULL), (long long)c0, (long long)Utils::hton(c1 + 1ULL), (long long)c0, (long long)Utils::hton(c1), (long long)c0);
-			(long long)Utils::hton(c1 + 3ULL), (long long)c0,
-			(long long)Utils::hton(c1 + 2ULL), (long long)c0,
-			(long long)Utils::hton(c1 + 1ULL), (long long)c0,
-			(long long)Utils::hton(c1), (long long)c0);
 		c1 += 4;
 		in += 64;
 		len -= 64;
@@ -134,12 +131,8 @@ void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, co
 	do {
 		__m256i p0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(in));
 		__m256i p1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(in + 32));
-		__m256i d0 = _mm256_set_epi64x(
+		__m256i d0 = _mm256_set_epi64x((long long)Utils::hton(c1 + 1ULL), (long long)c0, (long long)Utils::hton(c1), (long long)c0);
-			(long long)Utils::hton(c1 + 1ULL), (long long)c0,
+		__m256i d1 = _mm256_set_epi64x((long long)Utils::hton(c1 + 3ULL), (long long)c0, (long long)Utils::hton(c1 + 2ULL), (long long)c0);
-			(long long)Utils::hton(c1), (long long)c0);
-		__m256i d1 = _mm256_set_epi64x(
-			(long long)Utils::hton(c1 + 3ULL), (long long)c0,
-			(long long)Utils::hton(c1 + 2ULL), (long long)c0);
 		c1 += 4;
 		in += 64;
 		len -= 64;
@@ -184,7 +177,8 @@ void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, co
 #ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
 #endif
-__m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
+__m128i
+p_init256_1_aesni(__m128i a, __m128i b) noexcept
 {
 	__m128i x, y;
 	b = _mm_shuffle_epi32(b, 0xff);
@@ -201,7 +195,8 @@ __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
 #ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
 #endif
-__m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept
+__m128i
+p_init256_2_aesni(__m128i a, __m128i b) noexcept
 {
 	__m128i x, y, z;
 	y = _mm_aeskeygenassist_si128(a, 0x00);
@@ -260,7 +255,11 @@ void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept
 			in += 64;
 			__m128i a = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh, d1, 0x00), _mm_clmulepi64_si128(hhh, d2, 0x00)), _mm_xor_si128(_mm_clmulepi64_si128(hh, d3, 0x00), _mm_clmulepi64_si128(h, d4, 0x00)));
 			__m128i b = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh, d1, 0x11), _mm_clmulepi64_si128(hhh, d2, 0x11)), _mm_xor_si128(_mm_clmulepi64_si128(hh, d3, 0x11), _mm_clmulepi64_si128(h, d4, 0x11)));
-			__m128i c = _mm_xor_si128(_mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(hhhh2, _mm_xor_si128(_mm_shuffle_epi32(d1, 78), d1), 0x00), _mm_clmulepi64_si128(hhh2, _mm_xor_si128(_mm_shuffle_epi32(d2, 78), d2), 0x00)), _mm_xor_si128(_mm_clmulepi64_si128(hh2, _mm_xor_si128(_mm_shuffle_epi32(d3, 78), d3), 0x00), _mm_clmulepi64_si128(h2, _mm_xor_si128(_mm_shuffle_epi32(d4, 78), d4), 0x00))), _mm_xor_si128(a, b));
+			__m128i c = _mm_xor_si128(
+				_mm_xor_si128(
+					_mm_xor_si128(_mm_clmulepi64_si128(hhhh2, _mm_xor_si128(_mm_shuffle_epi32(d1, 78), d1), 0x00), _mm_clmulepi64_si128(hhh2, _mm_xor_si128(_mm_shuffle_epi32(d2, 78), d2), 0x00)),
+					_mm_xor_si128(_mm_clmulepi64_si128(hh2, _mm_xor_si128(_mm_shuffle_epi32(d3, 78), d3), 0x00), _mm_clmulepi64_si128(h2, _mm_xor_si128(_mm_shuffle_epi32(d4, 78), d4), 0x00))),
+				_mm_xor_si128(a, b));
 			a = _mm_xor_si128(_mm_slli_si128(c, 8), a);
 			b = _mm_xor_si128(_mm_srli_si128(c, 8), b);
 			c = _mm_srli_epi32(a, 31);
@@ -427,12 +426,12 @@ void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) n
 	_len = totalLen + len;
 
 	if (likely(len >= 64)) {
-
 #if defined(ZT_AES_VAES512) && defined(ZT_AES_VAES256)
 		if (Utils::CPUID.vaes && (len >= 256)) {
 			if (Utils::CPUID.avx512f) {
 				p_aesCtrInnerVAES512(len, _ctr[0], c1, in, out, k);
-			} else {
+			}
+			else {
 				p_aesCtrInnerVAES256(len, _ctr[0], c1, in, out, k);
 			}
 			goto skip_conventional_aesni_64;
@@ -526,7 +525,6 @@ void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) n
 			_mm_storeu_si128(reinterpret_cast<__m128i*>(out + 48), d3);
 			out += 64;
 		} while (likely(in != eof64));
-
 	}
 
 skip_conventional_aesni_64:

node/AES_armcrypto.cpp

@@ -11,8 +11,8 @@
  */
 /****/
 
-#include "Constants.hpp"
 #include "AES.hpp"
+#include "Constants.hpp"
 
 #ifdef ZT_AES_NEON
 
@@ -309,9 +309,14 @@ void AES::CTR::p_armCrypt(const uint8_t *in, uint8_t *out, unsigned int len) noe
 
 void AES::p_init_armneon_crypto(const uint8_t* key) noexcept
 {
-	static const uint8_t s_sbox[256] = {0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c,
+	static const uint8_t s_sbox[256] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
-	                                    0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea,
+										 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
-	                                    0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16};
+										 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+										 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+										 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+										 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+										 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+										 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 };
 
 	uint64_t h[2];
 	uint32_t* const w = reinterpret_cast<uint32_t*>(p_k.neon.ek);
@@ -326,7 +331,8 @@ void AES::p_init_armneon_crypto(const uint8_t *key) noexcept
 		const unsigned int imod = i & (ZT_INIT_ARMNEON_CRYPTO_NK - 1);
 		if (imod == 0) {
 			t = ZT_INIT_ARMNEON_CRYPTO_SUBWORD(ZT_INIT_ARMNEON_CRYPTO_ROTWORD(t)) ^ rcon[(i - 1) / ZT_INIT_ARMNEON_CRYPTO_NK];
-		} else if (imod == 4) {
+		}
+		else if (imod == 4) {
 			t = ZT_INIT_ARMNEON_CRYPTO_SUBWORD(t);
 		}
 		w[i] = w[i - ZT_INIT_ARMNEON_CRYPTO_NK] ^ t;

node/Address.hpp

@@ -14,37 +14,52 @@
 #ifndef ZT_ADDRESS_HPP
 #define ZT_ADDRESS_HPP
 
-#include <stdio.h>
+#include "Buffer.hpp"
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-
-#include <string>
-
 #include "Constants.hpp"
 #include "Utils.hpp"
-#include "Buffer.hpp"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string>
 
 namespace ZeroTier {
 
 /**
  * A ZeroTier address
  */
-class Address
+class Address {
-{
   public:
-	Address() : _a(0) {}
+	Address() : _a(0)
-	Address(const Address &a) : _a(a._a) {}
+	{
-	Address(uint64_t a) : _a(a & 0xffffffffffULL) {}
+	}
+	Address(const Address& a) : _a(a._a)
+	{
+	}
+	Address(uint64_t a) : _a(a & 0xffffffffffULL)
+	{
+	}
 
 	/**
 	 * @param bits Raw address -- 5 bytes, big-endian byte order
 	 * @param len Length of array
 	 */
-	Address(const void *bits,unsigned int len) { setTo(bits,len); }
+	Address(const void* bits, unsigned int len)
+	{
+		setTo(bits, len);
+	}
 
-	inline Address &operator=(const Address &a) { _a = a._a; return *this; }
+	inline Address& operator=(const Address& a)
-	inline Address &operator=(const uint64_t a) { _a = (a & 0xffffffffffULL); return *this; }
+	{
+		_a = a._a;
+		return *this;
+	}
+	inline Address& operator=(const uint64_t a)
+	{
+		_a = (a & 0xffffffffffULL);
+		return *this;
+	}
 
 	/**
 	 * @param bits Raw address -- 5 bytes, big-endian byte order
@@ -87,8 +102,7 @@ public:
 	 *
 	 * @param b Buffer to append to
 	 */
-	template<unsigned int C>
+	template <unsigned int C> inline void appendTo(Buffer<C>& b) const
-	inline void appendTo(Buffer<C> &b) const
 	{
 		unsigned char* p = (unsigned char*)b.appendField(ZT_ADDRESS_LENGTH);
 		*(p++) = (unsigned char)((_a >> 32) & 0xff);
@@ -101,22 +115,34 @@ public:
 	/**
 	 * @return Integer containing address (0 to 2^40)
 	 */
-	inline uint64_t toInt() const { return _a; }
+	inline uint64_t toInt() const
+	{
+		return _a;
+	}
 
 	/**
 	 * @return Hash code for use with Hashtable
 	 */
-	inline unsigned long hashCode() const { return (unsigned long)_a; }
+	inline unsigned long hashCode() const
+	{
+		return (unsigned long)_a;
+	}
 
 	/**
 	 * @return Hexadecimal string
 	 */
-	inline char *toString(char buf[11]) const { return Utils::hex10(_a,buf); }
+	inline char* toString(char buf[11]) const
+	{
+		return Utils::hex10(_a, buf);
+	}
 
 	/**
 	 * @return True if this address is not zero
 	 */
-	inline operator bool() const { return (_a != 0); }
+	inline operator bool() const
+	{
+		return (_a != 0);
+	}
 
 	/**
 	 * Check if this address is reserved
@@ -127,29 +153,74 @@ public:
 	 *
 	 * @return True if address is reserved and may not be used
 	 */
-	inline bool isReserved() const { return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); }
+	inline bool isReserved() const
+	{
+		return ((! _a) || ((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX));
+	}
 
 	/**
 	 * @param i Value from 0 to 4 (inclusive)
 	 * @return Byte at said position (address interpreted in big-endian order)
 	 */
-	inline uint8_t operator[](unsigned int i) const { return (uint8_t)(_a >> (32 - (i * 8))); }
+	inline uint8_t operator[](unsigned int i) const
+	{
+		return (uint8_t)(_a >> (32 - (i * 8)));
+	}
 
-	inline void zero() { _a = 0; }
+	inline void zero()
+	{
+		_a = 0;
+	}
 
-	inline bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); }
+	inline bool operator==(const uint64_t& a) const
-	inline bool operator!=(const uint64_t &a) const { return (_a != (a & 0xffffffffffULL)); }
+	{
-	inline bool operator>(const uint64_t &a) const { return (_a > (a & 0xffffffffffULL)); }
+		return (_a == (a & 0xffffffffffULL));
-	inline bool operator<(const uint64_t &a) const { return (_a < (a & 0xffffffffffULL)); }
+	}
-	inline bool operator>=(const uint64_t &a) const { return (_a >= (a & 0xffffffffffULL)); }
+	inline bool operator!=(const uint64_t& a) const
-	inline bool operator<=(const uint64_t &a) const { return (_a <= (a & 0xffffffffffULL)); }
+	{
+		return (_a != (a & 0xffffffffffULL));
+	}
+	inline bool operator>(const uint64_t& a) const
+	{
+		return (_a > (a & 0xffffffffffULL));
+	}
+	inline bool operator<(const uint64_t& a) const
+	{
+		return (_a < (a & 0xffffffffffULL));
+	}
+	inline bool operator>=(const uint64_t& a) const
+	{
+		return (_a >= (a & 0xffffffffffULL));
+	}
+	inline bool operator<=(const uint64_t& a) const
+	{
+		return (_a <= (a & 0xffffffffffULL));
+	}
 
-	inline bool operator==(const Address &a) const { return (_a == a._a); }
+	inline bool operator==(const Address& a) const
-	inline bool operator!=(const Address &a) const { return (_a != a._a); }
+	{
-	inline bool operator>(const Address &a) const { return (_a > a._a); }
+		return (_a == a._a);
-	inline bool operator<(const Address &a) const { return (_a < a._a); }
+	}
-	inline bool operator>=(const Address &a) const { return (_a >= a._a); }
+	inline bool operator!=(const Address& a) const
-	inline bool operator<=(const Address &a) const { return (_a <= a._a); }
+	{
+		return (_a != a._a);
+	}
+	inline bool operator>(const Address& a) const
+	{
+		return (_a > a._a);
+	}
+	inline bool operator<(const Address& a) const
+	{
+		return (_a < a._a);
+	}
+	inline bool operator>=(const Address& a) const
+	{
+		return (_a >= a._a);
+	}
+	inline bool operator<=(const Address& a) const
+	{
+		return (_a <= a._a);
+	}
 
   private:
 	uint64_t _a;

node/AtomicCounter.hpp

@@ -25,10 +25,12 @@ namespace ZeroTier {
 /**
  * Simple atomic counter supporting increment and decrement
  */
-class AtomicCounter
+class AtomicCounter {
-{
   public:
-	AtomicCounter() { _v = 0; }
+	AtomicCounter()
+	{
+		_v = 0;
+	}
 
 	inline int load() const
 	{
@@ -58,8 +60,13 @@ public:
 	}
 
   private:
-	AtomicCounter(const AtomicCounter &) {}
+	AtomicCounter(const AtomicCounter&)
-	const AtomicCounter &operator=(const AtomicCounter &) { return *this; }
+	{
+	}
+	const AtomicCounter& operator=(const AtomicCounter&)
+	{
+		return *this;
+	}
 
 #ifdef __GNUC__
 	int _v;

node/Bond.cpp

@@ -373,6 +373,7 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 	 */
 	if (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
 		if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && _paths[_abPathIdx].p) {
+			// fprintf(stderr, "trying to send via (_abPathIdx=%d) %s\n", _abPathIdx, pathToStr(_paths[_abPathIdx].p).c_str());
 			return _paths[_abPathIdx].p;
 		}
 	}
@@ -1032,6 +1033,13 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 		bool satisfiedUpDelay = (now - _paths[i].lastAliveToggle) >= _upDelay;
 		// How long since the last QoS was received (Must be less than ZT_PEER_PATH_EXPIRATION since the remote peer's _qosSendInterval isn't known)
 		bool acceptableQoSAge = (_paths[i].lastQoSReceived == 0 && inTrial) || ((now - _paths[i].lastQoSReceived) < ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD);
+
+		// Allow active-backup to operate without the receipt of QoS records
+		// This may be expanded to the other modes as an option
+		if (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
+			acceptableQoSAge = true;
+		}
+
 		currEligibility = _paths[i].allowed() && ((acceptableAge && satisfiedUpDelay && acceptableQoSAge) || inTrial);
 
 		if (currEligibility) {
@@ -1043,12 +1051,11 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 		 */
 		if (currEligibility != _paths[i].eligible) {
 			if (currEligibility == 0) {
-				log("link %s is no longer eligible", pathToStr(_paths[i].p).c_str());
+				log("link %s is no longer eligible (reason: allowed=%d, age=%d, ud=%d, qos=%d, trial=%d)", pathToStr(_paths[i].p).c_str(), _paths[i].allowed(), acceptableAge, satisfiedUpDelay, acceptableQoSAge, inTrial);
 			}
 			if (currEligibility == 1) {
 				log("link %s is eligible", pathToStr(_paths[i].p).c_str());
 			}
-			debug("\t[%d] allowed=%d, age=%d, qa=%d, ud=%d, trial=%d", i, _paths[i].allowed(), acceptableAge, acceptableQoSAge, satisfiedUpDelay, inTrial);
 			dumpPathStatus(now, i);
 			if (currEligibility) {
 				rebuildBond = true;
@@ -1496,7 +1503,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 {
 	int prevActiveBackupPathIdx = _abPathIdx;
 	int nonPreferredPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
-	bool bFoundPrimaryLink = false;
+	bool foundPathOnPrimaryLink = false;
+	bool foundPreferredPath = false;
 
 	if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && ! _paths[_abPathIdx].p) {
 		_abPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
@@ -1559,15 +1567,16 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 							if (! _paths[i].preferred()) {
 								// Found path on primary link, take note in case we don't find a preferred path
 								nonPreferredPathIdx = i;
-								bFoundPrimaryLink = true;
+								foundPathOnPrimaryLink = true;
 							}
 							if (_paths[i].preferred()) {
 								_abPathIdx = i;
-								bFoundPrimaryLink = true;
+								foundPathOnPrimaryLink = true;
 								if (_paths[_abPathIdx].p) {
 									SharedPtr<Link> abLink = RR->bc->getLinkBySocket(_policyAlias, _paths[_abPathIdx].p->localSocket());
 									if (abLink) {
-										log("found preferred primary link %s", pathToStr(_paths[_abPathIdx].p).c_str());
+										log("found preferred primary link (_abPathIdx=%d), %s", _abPathIdx, pathToStr(_paths[_abPathIdx].p).c_str());
+										foundPreferredPath = true;
 									}
 									break;	 // Found preferred path on primary link
 								}
@@ -1575,8 +1584,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 						}
 					}
 				}
-				if (bFoundPrimaryLink && (nonPreferredPathIdx != ZT_MAX_PEER_NETWORK_PATHS)) {
+				if (! foundPreferredPath && foundPathOnPrimaryLink && (nonPreferredPathIdx != ZT_MAX_PEER_NETWORK_PATHS)) {
-					log("found non-preferred primary link");
+					log("found non-preferred primary link (_abPathIdx=%d)", _abPathIdx);
 					_abPathIdx = nonPreferredPathIdx;
 				}
 			}
@@ -1614,10 +1623,10 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 		}
 		if (_paths[(*it)].p && ! _paths[(*it)].eligible) {
 			SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, _paths[(*it)].p->localSocket());
-			it = _abFailoverQueue.erase(it);
 			if (link) {
-				log("link %s is ineligible, removing from failover queue (%zu links remain in queue)", pathToStr(_paths[_abPathIdx].p).c_str(), _abFailoverQueue.size());
+				log("link %s is ineligible, removing from failover queue (%zu links remain in queue)", pathToStr(_paths[(*it)].p).c_str(), _abFailoverQueue.size());
 			}
+			it = _abFailoverQueue.erase(it);
 			continue;
 		}
 		else {
@@ -1684,7 +1693,7 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 						}
 					}
 					if (! bFoundPathInQueue) {
-						_abFailoverQueue.push_front(i);
+						_abFailoverQueue.push_back(i);
 						log("add link %s to failover queue (%zu links in queue)", pathToStr(_paths[i].p).c_str(), _abFailoverQueue.size());
 						addPathToBond(i, 0);
 					}
@@ -1734,7 +1743,7 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 					}
 				}
 				if (! bFoundPathInQueue) {
-					_abFailoverQueue.push_front(i);
+					_abFailoverQueue.push_back(i);
 					log("add link %s to failover queue (%zu links in queue)", pathToStr(_paths[i].p).c_str(), _abFailoverQueue.size());
 					addPathToBond(i, 0);
 				}
@@ -1742,17 +1751,11 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 		}
 	}
 	// Sort queue based on performance
-	if (! _abFailoverQueue.empty()) {
+	std::sort(_abFailoverQueue.begin(), _abFailoverQueue.end(),
-		for (int i = 0; i < _abFailoverQueue.size(); i++) {
+		[this](const int a, const int b) {
-			int value_to_insert = _abFailoverQueue[i];
+			// Sort by failover score in descending order (highest score first)
-			int hole_position = i;
+			return _paths[a].failoverScore > _paths[b].failoverScore;
-			while (hole_position > 0 && (_abFailoverQueue[hole_position - 1] > value_to_insert)) {
+		});
-				_abFailoverQueue[hole_position] = _abFailoverQueue[hole_position - 1];
-				hole_position = hole_position - 1;
-			}
-			_abFailoverQueue[hole_position] = value_to_insert;
-		}
-	}
 
 	/**
 	 * Short-circuit if we have no queued paths
@@ -1902,7 +1905,7 @@ void Bond::setBondParameters(int policy, SharedPtr<Bond> templateBond, bool useT
 	 * Policy defaults
 	 */
 	_abPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
-	_abLinkSelectMethod = ZT_BOND_RESELECTION_POLICY_OPTIMIZE;
+	_abLinkSelectMethod = ZT_BOND_RESELECTION_POLICY_ALWAYS;
 	_rrPacketsSentOnCurrLink = 0;
 	_rrIdx = 0;
 	_packetsPerLink = 64;
@@ -2021,7 +2024,8 @@ void Bond::dumpInfo(int64_t now, bool force)
 	_lastSummaryDump = now;
 	float overhead = (_overheadBytes / (timeSinceLastDump / 1000.0f) / 1000.0f);
 	_overheadBytes = 0;
-	log("bond: bp=%d, fi=%" PRIu64 ", mi=%d, ud=%d, dd=%d, flows=%zu, leaf=%d, overhead=%f KB/s, links=(%d/%d)",
+	log("bond: ready=%d, bp=%d, fi=%" PRIu64 ", mi=%d, ud=%d, dd=%d, flows=%zu, leaf=%d, overhead=%f KB/s, links=(%d/%d)",
+		isReady(),
 		_policy,
 		_failoverInterval,
 		_monitorInterval,

node/Bond.hpp

@@ -1144,6 +1144,7 @@ class Bond {
 		__attribute__((format(printf, 2, 3)))
 #endif
 	{
+		// if (_peerId != 0x0 && _peerId != 0x0) { return; }
 #ifdef ZT_TRACE
 		time_t rawtime;
 		struct tm* timeinfo;
@@ -1175,6 +1176,7 @@ class Bond {
 		__attribute__((format(printf, 2, 3)))
 #endif
 	{
+		// if (_peerId != 0x0 && _peerId != 0x0) { return; }
 #ifdef ZT_DEBUG
 		time_t rawtime;
 		struct tm* timeinfo;

node/Buffer.hpp

@@ -14,17 +14,16 @@
 #ifndef ZT_BUFFER_HPP
 #define ZT_BUFFER_HPP
 
-#include <string.h>
-#include <stdint.h>
-
-#include <stdexcept>
-#include <string>
-#include <algorithm>
-#include <utility>
-
 #include "Constants.hpp"
 #include "Utils.hpp"
 
+#include <algorithm>
+#include <stdexcept>
+#include <stdint.h>
+#include <string.h>
+#include <string>
+#include <utility>
+
 #if defined(__GNUC__) && (! defined(ZT_NO_TYPE_PUNNING))
 #define ZT_VAR_MAY_ALIAS __attribute__((__may_alias__))
 #else
@@ -46,9 +45,7 @@ namespace ZeroTier {
  *
  * @tparam C Total capacity
  */
-template<unsigned int C>
+template <unsigned int C> class Buffer {
-class Buffer
-{
 	// I love me!
 	template <unsigned int C2> friend class Buffer;
 
@@ -65,17 +62,40 @@ public:
 	typedef int difference_type;
 	typedef std::reverse_iterator<iterator> reverse_iterator;
 	typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
-	inline iterator begin() { return _b; }
+	inline iterator begin()
-	inline iterator end() { return (_b + _l); }
+	{
-	inline const_iterator begin() const { return _b; }
+		return _b;
-	inline const_iterator end() const { return (_b + _l); }
+	}
-	inline reverse_iterator rbegin() { return reverse_iterator(begin()); }
+	inline iterator end()
-	inline reverse_iterator rend() { return reverse_iterator(end()); }
+	{
-	inline const_reverse_iterator rbegin() const { return const_reverse_iterator(begin()); }
+		return (_b + _l);
-	inline const_reverse_iterator rend() const { return const_reverse_iterator(end()); }
+	}
+	inline const_iterator begin() const
+	{
+		return _b;
+	}
+	inline const_iterator end() const
+	{
+		return (_b + _l);
+	}
+	inline reverse_iterator rbegin()
+	{
+		return reverse_iterator(begin());
+	}
+	inline reverse_iterator rend()
+	{
+		return reverse_iterator(end());
+	}
+	inline const_reverse_iterator rbegin() const
+	{
+		return const_reverse_iterator(begin());
+	}
+	inline const_reverse_iterator rend() const
+	{
+		return const_reverse_iterator(end());
+	}
 
-	Buffer() :
+	Buffer() : _l(0)
-		_l(0)
 	{
 	}
 
@@ -87,8 +107,7 @@ public:
 		_l = l;
 	}
 
-	template<unsigned int C2>
+	template <unsigned int C2> Buffer(const Buffer<C2>& b)
-	Buffer(const Buffer<C2> &b)
 	{
 		*this = b;
 	}
@@ -98,15 +117,15 @@ public:
 		copyFrom(b, l);
 	}
 
-	template<unsigned int C2>
+	template <unsigned int C2> inline Buffer& operator=(const Buffer<C2>& b)
-	inline Buffer &operator=(const Buffer<C2> &b)
 	{
 		if (unlikely(b._l > C)) {
 			throw ZT_EXCEPTION_OUT_OF_BOUNDS;
 		}
 		if (C2 == C) {
 			memcpy(this, &b, sizeof(Buffer<C>));
-		} else {
+		}
+		else {
 			memcpy(_b, b._b, _l = b._l);
 		}
 		return *this;
@@ -172,8 +191,7 @@ public:
 	 * @param v Value
 	 * @tparam T Integer type (e.g. uint16_t, int64_t)
 	 */
-	template<typename T>
+	template <typename T> inline void setAt(unsigned int i, const T v)
-	inline void setAt(unsigned int i,const T v)
 	{
 		if (unlikely((i + sizeof(T)) > _l)) {
 			throw ZT_EXCEPTION_OUT_OF_BOUNDS;
@@ -196,8 +214,7 @@ public:
 	 * @tparam T Integer type (e.g. uint16_t, int64_t)
 	 * @return Integer value
 	 */
-	template<typename T>
+	template <typename T> inline T at(unsigned int i) const
-	inline T at(unsigned int i) const
 	{
 		if (unlikely((i + sizeof(T)) > _l)) {
 			throw ZT_EXCEPTION_OUT_OF_BOUNDS;
@@ -223,8 +240,7 @@ public:
 	 * @tparam T Integer type (e.g. uint16_t, int64_t)
 	 * @throws std::out_of_range Attempt to append beyond capacity
 	 */
-	template<typename T>
+	template <typename T> inline void append(const T v)
-	inline void append(const T v)
 	{
 		if (unlikely((_l + sizeof(T)) > C)) {
 			throw ZT_EXCEPTION_OUT_OF_BOUNDS;
@@ -313,8 +329,7 @@ public:
 	 * @tparam C2 Capacity of second buffer (typically inferred)
 	 * @throws std::out_of_range Attempt to append beyond capacity
 	 */
-	template<unsigned int C2>
+	template <unsigned int C2> inline void append(const Buffer<C2>& b)
-	inline void append(const Buffer<C2> &b)
 	{
 		append(b._b, b._l);
 	}
@@ -408,70 +423,88 @@ public:
 	/**
 	 * Set buffer data length to zero
 	 */
-	inline void clear() { _l = 0; }
+	inline void clear()
+	{
+		_l = 0;
+	}
 
 	/**
 	 * Zero buffer up to size()
 	 */
-	inline void zero() { memset(_b,0,_l); }
+	inline void zero()
+	{
+		memset(_b, 0, _l);
+	}
 
 	/**
 	 * Zero unused capacity area
 	 */
-	inline void zeroUnused() { memset(_b + _l,0,C - _l); }
+	inline void zeroUnused()
+	{
+		memset(_b + _l, 0, C - _l);
+	}
 
 	/**
 	 * Unconditionally and securely zero buffer's underlying memory
 	 */
-	inline void burn() { Utils::burn(_b,sizeof(_b)); }
+	inline void burn()
+	{
+		Utils::burn(_b, sizeof(_b));
+	}
 
 	/**
 	 * @return Constant pointer to data in buffer
 	 */
-	inline const void *data() const { return _b; }
+	inline const void* data() const
+	{
+		return _b;
+	}
 
 	/**
 	 * @return Non-constant pointer to data in buffer
 	 */
-	inline void *unsafeData() { return _b; }
+	inline void* unsafeData()
+	{
+		return _b;
+	}
 
 	/**
 	 * @return Size of data in buffer
 	 */
-	inline unsigned int size() const { return _l; }
+	inline unsigned int size() const
+	{
+		return _l;
+	}
 
 	/**
 	 * @return Capacity of buffer
 	 */
-	inline unsigned int capacity() const { return C; }
+	inline unsigned int capacity() const
+	{
+		return C;
+	}
 
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator==(const Buffer<C2>& b) const
-	inline bool operator==(const Buffer<C2> &b) const
 	{
 		return ((_l == b._l) && (! memcmp(_b, b._b, _l)));
 	}
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator!=(const Buffer<C2>& b) const
-	inline bool operator!=(const Buffer<C2> &b) const
 	{
 		return ((_l != b._l) || (memcmp(_b, b._b, _l)));
 	}
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator<(const Buffer<C2>& b) const
-	inline bool operator<(const Buffer<C2> &b) const
 	{
 		return (memcmp(_b, b._b, std::min(_l, b._l)) < 0);
 	}
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator>(const Buffer<C2>& b) const
-	inline bool operator>(const Buffer<C2> &b) const
 	{
 		return (b < *this);
 	}
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator<=(const Buffer<C2>& b) const
-	inline bool operator<=(const Buffer<C2> &b) const
 	{
 		return ! (b < *this);
 	}
-	template<unsigned int C2>
+	template <unsigned int C2> inline bool operator>=(const Buffer<C2>& b) const
-	inline bool operator>=(const Buffer<C2> &b) const
 	{
 		return ! (*this < b);
 	}

node/C25519.cpp

@@ -7,17 +7,18 @@ Derived from public domain code by D. J. Bernstein.
 // Modified very slightly for ZeroTier One by Adam Ierymenko
 // This code remains in the public domain.
 
+#include "C25519.hpp"
+
+#include "Buffer.hpp"
+#include "Constants.hpp"
+#include "Hashtable.hpp"
+#include "Mutex.hpp"
+#include "SHA512.hpp"
+
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "Constants.hpp"
-#include "C25519.hpp"
-#include "SHA512.hpp"
-#include "Buffer.hpp"
-#include "Hashtable.hpp"
-#include "Mutex.hpp"
-
 #ifdef __WINDOWS__
 #pragma warning(disable : 4146)
 #endif
@@ -41,7 +42,8 @@ typedef uint8_t u8;
 typedef int32_t s32;
 typedef int64_t limb;
 
-static inline void fsum(limb *output, const limb *in) {
+static inline void fsum(limb* output, const limb* in)
+{
 	unsigned i;
 	for (i = 0; i < 10; i += 2) {
 		output[0 + i] = output[0 + i] + in[0 + i];
@@ -49,124 +51,56 @@ static inline void fsum(limb *output, const limb *in) {
 	}
 }
 
-static inline void fdifference(limb *output, const limb *in) {
+static inline void fdifference(limb* output, const limb* in)
+{
 	unsigned i;
 	for (i = 0; i < 10; ++i) {
 		output[i] = in[i] - output[i];
 	}
 }
 
-static inline void fscalar_product(limb *output, const limb *in, const limb scalar) {
+static inline void fscalar_product(limb* output, const limb* in, const limb scalar)
+{
 	unsigned i;
 	for (i = 0; i < 10; ++i) {
 		output[i] = in[i] * scalar;
 	}
 }
 
-static inline void fproduct(limb *output, const limb *in2, const limb *in) {
+static inline void fproduct(limb* output, const limb* in2, const limb* in)
+{
 	output[0] = ((limb)((s32)in2[0])) * ((s32)in[0]);
-  output[1] =       ((limb) ((s32) in2[0])) * ((s32) in[1]) +
+	output[1] = ((limb)((s32)in2[0])) * ((s32)in[1]) + ((limb)((s32)in2[1])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[1])) * ((s32) in[0]);
+	output[2] = 2 * ((limb)((s32)in2[1])) * ((s32)in[1]) + ((limb)((s32)in2[0])) * ((s32)in[2]) + ((limb)((s32)in2[2])) * ((s32)in[0]);
-  output[2] =  2 *  ((limb) ((s32) in2[1])) * ((s32) in[1]) +
+	output[3] = ((limb)((s32)in2[1])) * ((s32)in[2]) + ((limb)((s32)in2[2])) * ((s32)in[1]) + ((limb)((s32)in2[0])) * ((s32)in[3]) + ((limb)((s32)in2[3])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[0])) * ((s32) in[2]) +
+	output[4] = ((limb)((s32)in2[2])) * ((s32)in[2]) + 2 * (((limb)((s32)in2[1])) * ((s32)in[3]) + ((limb)((s32)in2[3])) * ((s32)in[1])) + ((limb)((s32)in2[0])) * ((s32)in[4]) + ((limb)((s32)in2[4])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[2])) * ((s32) in[0]);
+	output[5] =
-  output[3] =       ((limb) ((s32) in2[1])) * ((s32) in[2]) +
+		((limb)((s32)in2[2])) * ((s32)in[3]) + ((limb)((s32)in2[3])) * ((s32)in[2]) + ((limb)((s32)in2[1])) * ((s32)in[4]) + ((limb)((s32)in2[4])) * ((s32)in[1]) + ((limb)((s32)in2[0])) * ((s32)in[5]) + ((limb)((s32)in2[5])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[2])) * ((s32) in[1]) +
+	output[6] = 2 * (((limb)((s32)in2[3])) * ((s32)in[3]) + ((limb)((s32)in2[1])) * ((s32)in[5]) + ((limb)((s32)in2[5])) * ((s32)in[1])) + ((limb)((s32)in2[2])) * ((s32)in[4]) + ((limb)((s32)in2[4])) * ((s32)in[2])
-                    ((limb) ((s32) in2[0])) * ((s32) in[3]) +
+				+ ((limb)((s32)in2[0])) * ((s32)in[6]) + ((limb)((s32)in2[6])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[3])) * ((s32) in[0]);
+	output[7] = ((limb)((s32)in2[3])) * ((s32)in[4]) + ((limb)((s32)in2[4])) * ((s32)in[3]) + ((limb)((s32)in2[2])) * ((s32)in[5]) + ((limb)((s32)in2[5])) * ((s32)in[2]) + ((limb)((s32)in2[1])) * ((s32)in[6])
-  output[4] =       ((limb) ((s32) in2[2])) * ((s32) in[2]) +
+				+ ((limb)((s32)in2[6])) * ((s32)in[1]) + ((limb)((s32)in2[0])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[0]);
-               2 * (((limb) ((s32) in2[1])) * ((s32) in[3]) +
+	output[8] = ((limb)((s32)in2[4])) * ((s32)in[4]) + 2 * (((limb)((s32)in2[3])) * ((s32)in[5]) + ((limb)((s32)in2[5])) * ((s32)in[3]) + ((limb)((s32)in2[1])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[1]))
-                    ((limb) ((s32) in2[3])) * ((s32) in[1])) +
+				+ ((limb)((s32)in2[2])) * ((s32)in[6]) + ((limb)((s32)in2[6])) * ((s32)in[2]) + ((limb)((s32)in2[0])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[0]);
-                    ((limb) ((s32) in2[0])) * ((s32) in[4]) +
+	output[9] = ((limb)((s32)in2[4])) * ((s32)in[5]) + ((limb)((s32)in2[5])) * ((s32)in[4]) + ((limb)((s32)in2[3])) * ((s32)in[6]) + ((limb)((s32)in2[6])) * ((s32)in[3]) + ((limb)((s32)in2[2])) * ((s32)in[7])
-                    ((limb) ((s32) in2[4])) * ((s32) in[0]);
+				+ ((limb)((s32)in2[7])) * ((s32)in[2]) + ((limb)((s32)in2[1])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[1]) + ((limb)((s32)in2[0])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[0]);
-  output[5] =       ((limb) ((s32) in2[2])) * ((s32) in[3]) +
+	output[10] = 2 * (((limb)((s32)in2[5])) * ((s32)in[5]) + ((limb)((s32)in2[3])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[3]) + ((limb)((s32)in2[1])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[1]))
-                    ((limb) ((s32) in2[3])) * ((s32) in[2]) +
+				 + ((limb)((s32)in2[4])) * ((s32)in[6]) + ((limb)((s32)in2[6])) * ((s32)in[4]) + ((limb)((s32)in2[2])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[2]);
-                    ((limb) ((s32) in2[1])) * ((s32) in[4]) +
+	output[11] = ((limb)((s32)in2[5])) * ((s32)in[6]) + ((limb)((s32)in2[6])) * ((s32)in[5]) + ((limb)((s32)in2[4])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[4]) + ((limb)((s32)in2[3])) * ((s32)in[8])
-                    ((limb) ((s32) in2[4])) * ((s32) in[1]) +
+				 + ((limb)((s32)in2[8])) * ((s32)in[3]) + ((limb)((s32)in2[2])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[2]);
-                    ((limb) ((s32) in2[0])) * ((s32) in[5]) +
+	output[12] = ((limb)((s32)in2[6])) * ((s32)in[6]) + 2 * (((limb)((s32)in2[5])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[5]) + ((limb)((s32)in2[3])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[3]))
-                    ((limb) ((s32) in2[5])) * ((s32) in[0]);
+				 + ((limb)((s32)in2[4])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[4]);
-  output[6] =  2 * (((limb) ((s32) in2[3])) * ((s32) in[3]) +
+	output[13] =
-                    ((limb) ((s32) in2[1])) * ((s32) in[5]) +
+		((limb)((s32)in2[6])) * ((s32)in[7]) + ((limb)((s32)in2[7])) * ((s32)in[6]) + ((limb)((s32)in2[5])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[5]) + ((limb)((s32)in2[4])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[4]);
-                    ((limb) ((s32) in2[5])) * ((s32) in[1])) +
+	output[14] = 2 * (((limb)((s32)in2[7])) * ((s32)in[7]) + ((limb)((s32)in2[5])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[5])) + ((limb)((s32)in2[6])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[6]);
-                    ((limb) ((s32) in2[2])) * ((s32) in[4]) +
+	output[15] = ((limb)((s32)in2[7])) * ((s32)in[8]) + ((limb)((s32)in2[8])) * ((s32)in[7]) + ((limb)((s32)in2[6])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[6]);
-                    ((limb) ((s32) in2[4])) * ((s32) in[2]) +
+	output[16] = ((limb)((s32)in2[8])) * ((s32)in[8]) + 2 * (((limb)((s32)in2[7])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[7]));
-                    ((limb) ((s32) in2[0])) * ((s32) in[6]) +
+	output[17] = ((limb)((s32)in2[8])) * ((s32)in[9]) + ((limb)((s32)in2[9])) * ((s32)in[8]);
-                    ((limb) ((s32) in2[6])) * ((s32) in[0]);
-  output[7] =       ((limb) ((s32) in2[3])) * ((s32) in[4]) +
-                    ((limb) ((s32) in2[4])) * ((s32) in[3]) +
-                    ((limb) ((s32) in2[2])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[5])) * ((s32) in[2]) +
-                    ((limb) ((s32) in2[1])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[1]) +
-                    ((limb) ((s32) in2[0])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[0]);
-  output[8] =       ((limb) ((s32) in2[4])) * ((s32) in[4]) +
-               2 * (((limb) ((s32) in2[3])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[5])) * ((s32) in[3]) +
-                    ((limb) ((s32) in2[1])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[1])) +
-                    ((limb) ((s32) in2[2])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[2]) +
-                    ((limb) ((s32) in2[0])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[0]);
-  output[9] =       ((limb) ((s32) in2[4])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[5])) * ((s32) in[4]) +
-                    ((limb) ((s32) in2[3])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[3]) +
-                    ((limb) ((s32) in2[2])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[2]) +
-                    ((limb) ((s32) in2[1])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[1]) +
-                    ((limb) ((s32) in2[0])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[0]);
-  output[10] = 2 * (((limb) ((s32) in2[5])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[3])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[3]) +
-                    ((limb) ((s32) in2[1])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[1])) +
-                    ((limb) ((s32) in2[4])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[4]) +
-                    ((limb) ((s32) in2[2])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[2]);
-  output[11] =      ((limb) ((s32) in2[5])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[4])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[4]) +
-                    ((limb) ((s32) in2[3])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[3]) +
-                    ((limb) ((s32) in2[2])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[2]);
-  output[12] =      ((limb) ((s32) in2[6])) * ((s32) in[6]) +
-               2 * (((limb) ((s32) in2[5])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[3])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[3])) +
-                    ((limb) ((s32) in2[4])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[4]);
-  output[13] =      ((limb) ((s32) in2[6])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[7])) * ((s32) in[6]) +
-                    ((limb) ((s32) in2[5])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[5]) +
-                    ((limb) ((s32) in2[4])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[4]);
-  output[14] = 2 * (((limb) ((s32) in2[7])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[5])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[5])) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[6]);
-  output[15] =      ((limb) ((s32) in2[7])) * ((s32) in[8]) +
-                    ((limb) ((s32) in2[8])) * ((s32) in[7]) +
-                    ((limb) ((s32) in2[6])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[6]);
-  output[16] =      ((limb) ((s32) in2[8])) * ((s32) in[8]) +
-               2 * (((limb) ((s32) in2[7])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[7]));
-  output[17] =      ((limb) ((s32) in2[8])) * ((s32) in[9]) +
-                    ((limb) ((s32) in2[9])) * ((s32) in[8]);
 	output[18] = 2 * ((limb)((s32)in2[9])) * ((s32)in[9]);
 }
 
-static inline void freduce_degree(limb *output) {
+static inline void freduce_degree(limb* output)
+{
 	output[8] += output[18] << 4;
 	output[8] += output[18] << 1;
 	output[8] += output[18];
@@ -224,7 +158,8 @@ static inline limb div_by_2_25(const limb v)
 	return (v + roundoff) >> 25;
 }
 
-static inline void freduce_coefficients(limb *output) {
+static inline void freduce_coefficients(limb* output)
+{
 	unsigned i;
 
 	output[10] = 0;
@@ -267,7 +202,8 @@ static inline void freduce_coefficients(limb *output) {
 	 * bound on |output[1]| is sufficient to meet our needs. */
 }
 
-static inline void fmul(limb *output, const limb *in, const limb *in2) {
+static inline void fmul(limb* output, const limb* in, const limb* in2)
+{
 	limb t[19];
 	fproduct(t, in, in2);
 	/* |t[i]| < 14*2^54 */
@@ -277,65 +213,31 @@ static inline void fmul(limb *output, const limb *in, const limb *in2) {
 	memcpy(output, t, sizeof(limb) * 10);
 }
 
-static inline void fsquare_inner(limb *output, const limb *in) {
+static inline void fsquare_inner(limb* output, const limb* in)
+{
 	output[0] = ((limb)((s32)in[0])) * ((s32)in[0]);
 	output[1] = 2 * ((limb)((s32)in[0])) * ((s32)in[1]);
-  output[2] =  2 * (((limb) ((s32) in[1])) * ((s32) in[1]) +
+	output[2] = 2 * (((limb)((s32)in[1])) * ((s32)in[1]) + ((limb)((s32)in[0])) * ((s32)in[2]));
-                    ((limb) ((s32) in[0])) * ((s32) in[2]));
+	output[3] = 2 * (((limb)((s32)in[1])) * ((s32)in[2]) + ((limb)((s32)in[0])) * ((s32)in[3]));
-  output[3] =  2 * (((limb) ((s32) in[1])) * ((s32) in[2]) +
+	output[4] = ((limb)((s32)in[2])) * ((s32)in[2]) + 4 * ((limb)((s32)in[1])) * ((s32)in[3]) + 2 * ((limb)((s32)in[0])) * ((s32)in[4]);
-                    ((limb) ((s32) in[0])) * ((s32) in[3]));
+	output[5] = 2 * (((limb)((s32)in[2])) * ((s32)in[3]) + ((limb)((s32)in[1])) * ((s32)in[4]) + ((limb)((s32)in[0])) * ((s32)in[5]));
-  output[4] =       ((limb) ((s32) in[2])) * ((s32) in[2]) +
+	output[6] = 2 * (((limb)((s32)in[3])) * ((s32)in[3]) + ((limb)((s32)in[2])) * ((s32)in[4]) + ((limb)((s32)in[0])) * ((s32)in[6]) + 2 * ((limb)((s32)in[1])) * ((s32)in[5]));
-               4 *  ((limb) ((s32) in[1])) * ((s32) in[3]) +
+	output[7] = 2 * (((limb)((s32)in[3])) * ((s32)in[4]) + ((limb)((s32)in[2])) * ((s32)in[5]) + ((limb)((s32)in[1])) * ((s32)in[6]) + ((limb)((s32)in[0])) * ((s32)in[7]));
-               2 *  ((limb) ((s32) in[0])) * ((s32) in[4]);
+	output[8] = ((limb)((s32)in[4])) * ((s32)in[4]) + 2 * (((limb)((s32)in[2])) * ((s32)in[6]) + ((limb)((s32)in[0])) * ((s32)in[8]) + 2 * (((limb)((s32)in[1])) * ((s32)in[7]) + ((limb)((s32)in[3])) * ((s32)in[5])));
-  output[5] =  2 * (((limb) ((s32) in[2])) * ((s32) in[3]) +
+	output[9] = 2 * (((limb)((s32)in[4])) * ((s32)in[5]) + ((limb)((s32)in[3])) * ((s32)in[6]) + ((limb)((s32)in[2])) * ((s32)in[7]) + ((limb)((s32)in[1])) * ((s32)in[8]) + ((limb)((s32)in[0])) * ((s32)in[9]));
-                    ((limb) ((s32) in[1])) * ((s32) in[4]) +
+	output[10] = 2 * (((limb)((s32)in[5])) * ((s32)in[5]) + ((limb)((s32)in[4])) * ((s32)in[6]) + ((limb)((s32)in[2])) * ((s32)in[8]) + 2 * (((limb)((s32)in[3])) * ((s32)in[7]) + ((limb)((s32)in[1])) * ((s32)in[9])));
-                    ((limb) ((s32) in[0])) * ((s32) in[5]));
+	output[11] = 2 * (((limb)((s32)in[5])) * ((s32)in[6]) + ((limb)((s32)in[4])) * ((s32)in[7]) + ((limb)((s32)in[3])) * ((s32)in[8]) + ((limb)((s32)in[2])) * ((s32)in[9]));
-  output[6] =  2 * (((limb) ((s32) in[3])) * ((s32) in[3]) +
+	output[12] = ((limb)((s32)in[6])) * ((s32)in[6]) + 2 * (((limb)((s32)in[4])) * ((s32)in[8]) + 2 * (((limb)((s32)in[5])) * ((s32)in[7]) + ((limb)((s32)in[3])) * ((s32)in[9])));
-                    ((limb) ((s32) in[2])) * ((s32) in[4]) +
+	output[13] = 2 * (((limb)((s32)in[6])) * ((s32)in[7]) + ((limb)((s32)in[5])) * ((s32)in[8]) + ((limb)((s32)in[4])) * ((s32)in[9]));
-                    ((limb) ((s32) in[0])) * ((s32) in[6]) +
+	output[14] = 2 * (((limb)((s32)in[7])) * ((s32)in[7]) + ((limb)((s32)in[6])) * ((s32)in[8]) + 2 * ((limb)((s32)in[5])) * ((s32)in[9]));
-               2 *  ((limb) ((s32) in[1])) * ((s32) in[5]));
+	output[15] = 2 * (((limb)((s32)in[7])) * ((s32)in[8]) + ((limb)((s32)in[6])) * ((s32)in[9]));
-  output[7] =  2 * (((limb) ((s32) in[3])) * ((s32) in[4]) +
+	output[16] = ((limb)((s32)in[8])) * ((s32)in[8]) + 4 * ((limb)((s32)in[7])) * ((s32)in[9]);
-                    ((limb) ((s32) in[2])) * ((s32) in[5]) +
-                    ((limb) ((s32) in[1])) * ((s32) in[6]) +
-                    ((limb) ((s32) in[0])) * ((s32) in[7]));
-  output[8] =       ((limb) ((s32) in[4])) * ((s32) in[4]) +
-               2 * (((limb) ((s32) in[2])) * ((s32) in[6]) +
-                    ((limb) ((s32) in[0])) * ((s32) in[8]) +
-               2 * (((limb) ((s32) in[1])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[3])) * ((s32) in[5])));
-  output[9] =  2 * (((limb) ((s32) in[4])) * ((s32) in[5]) +
-                    ((limb) ((s32) in[3])) * ((s32) in[6]) +
-                    ((limb) ((s32) in[2])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[1])) * ((s32) in[8]) +
-                    ((limb) ((s32) in[0])) * ((s32) in[9]));
-  output[10] = 2 * (((limb) ((s32) in[5])) * ((s32) in[5]) +
-                    ((limb) ((s32) in[4])) * ((s32) in[6]) +
-                    ((limb) ((s32) in[2])) * ((s32) in[8]) +
-               2 * (((limb) ((s32) in[3])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[1])) * ((s32) in[9])));
-  output[11] = 2 * (((limb) ((s32) in[5])) * ((s32) in[6]) +
-                    ((limb) ((s32) in[4])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[3])) * ((s32) in[8]) +
-                    ((limb) ((s32) in[2])) * ((s32) in[9]));
-  output[12] =      ((limb) ((s32) in[6])) * ((s32) in[6]) +
-               2 * (((limb) ((s32) in[4])) * ((s32) in[8]) +
-               2 * (((limb) ((s32) in[5])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[3])) * ((s32) in[9])));
-  output[13] = 2 * (((limb) ((s32) in[6])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[5])) * ((s32) in[8]) +
-                    ((limb) ((s32) in[4])) * ((s32) in[9]));
-  output[14] = 2 * (((limb) ((s32) in[7])) * ((s32) in[7]) +
-                    ((limb) ((s32) in[6])) * ((s32) in[8]) +
-               2 *  ((limb) ((s32) in[5])) * ((s32) in[9]));
-  output[15] = 2 * (((limb) ((s32) in[7])) * ((s32) in[8]) +
-                    ((limb) ((s32) in[6])) * ((s32) in[9]));
-  output[16] =      ((limb) ((s32) in[8])) * ((s32) in[8]) +
-               4 *  ((limb) ((s32) in[7])) * ((s32) in[9]);
 	output[17] = 2 * ((limb)((s32)in[8])) * ((s32)in[9]);
 	output[18] = 2 * ((limb)((s32)in[9])) * ((s32)in[9]);
 }
 
-static void fsquare(limb *output, const limb *in) {
+static void fsquare(limb* output, const limb* in)
+{
 	limb t[19];
 	fsquare_inner(t, in);
 	/* |t[i]| < 14*2^54 because the largest product of two limbs will be <
@@ -347,12 +249,9 @@ static void fsquare(limb *output, const limb *in) {
 	memcpy(output, t, sizeof(limb) * 10);
 }
 
-static inline void fexpand(limb *output, const u8 *input) {
+static inline void fexpand(limb* output, const u8* input)
-#define F(n,start,shift,mask) \
+{
-  output[n] = ((((limb) input[start + 0]) | \
+#define F(n, start, shift, mask) output[n] = ((((limb)input[start + 0]) | ((limb)input[start + 1]) << 8 | ((limb)input[start + 2]) << 16 | ((limb)input[start + 3]) << 24) >> shift) & mask;
-                ((limb) input[start + 1]) << 8 | \
-                ((limb) input[start + 2]) << 16 | \
-                ((limb) input[start + 3]) << 24) >> shift) & mask;
 	F(0, 0, 0, 0x3ffffff);
 	F(1, 3, 2, 0x1ffffff);
 	F(2, 6, 3, 0x3ffffff);
@@ -370,7 +269,8 @@ static inline void fexpand(limb *output, const u8 *input) {
 #error "This code only works when >> does sign-extension on negative numbers"
 #endif
 
-static inline s32 s32_eq(s32 a, s32 b) {
+static inline s32 s32_eq(s32 a, s32 b)
+{
 	a = ~(a ^ b);
 	a &= a << 16;
 	a &= a << 8;
@@ -380,13 +280,15 @@ static inline s32 s32_eq(s32 a, s32 b) {
 	return a >> 31;
 }
 
-static inline s32 s32_gte(s32 a, s32 b) {
+static inline s32 s32_gte(s32 a, s32 b)
+{
 	a -= b;
 	/* a >= 0 iff a >= b. */
 	return ~(a >> 31);
 }
 
-static inline void fcontract(u8 *output, limb *input_limbs) {
+static inline void fcontract(u8* output, limb* input_limbs)
+{
 	int i;
 	int j;
 	s32 input[10];
@@ -406,7 +308,8 @@ static inline void fcontract(u8 *output, limb *input_limbs) {
 				const s32 carry = -((input[i] & mask) >> 25);
 				input[i] = input[i] + (carry << 25);
 				input[i + 1] = input[i + 1] - carry;
-      } else {
+			}
+			else {
 				const s32 mask = input[i] >> 31;
 				const s32 carry = -((input[i] & mask) >> 26);
 				input[i] = input[i] + (carry << 26);
@@ -455,7 +358,8 @@ static inline void fcontract(u8 *output, limb *input_limbs) {
 				const s32 carry = input[i] >> 25;
 				input[i] &= 0x1ffffff;
 				input[i + 1] += carry;
-      } else {
+			}
+			else {
 				const s32 carry = input[i] >> 26;
 				input[i] &= 0x3ffffff;
 				input[i + 1] += carry;
@@ -483,7 +387,8 @@ static inline void fcontract(u8 *output, limb *input_limbs) {
 	for (i = 1; i < 10; i++) {
 		if ((i & 1) == 1) {
 			mask &= s32_eq(input[i], 0x1ffffff);
-    } else {
+		}
+		else {
 			mask &= s32_eq(input[i], 0x3ffffff);
 		}
 	}
@@ -495,7 +400,8 @@ static inline void fcontract(u8 *output, limb *input_limbs) {
 	for (i = 1; i < 10; i++) {
 		if ((i & 1) == 1) {
 			input[i] -= mask & 0x1ffffff;
-    } else {
+		}
+		else {
 			input[i] -= mask & 0x3ffffff;
 		}
 	}
@@ -528,13 +434,18 @@ static inline void fcontract(u8 *output, limb *input_limbs) {
 #undef F
 }
 
-static inline void fmonty(limb *x2, limb *z2,  /* output 2Q */
+static inline void fmonty(
-                   limb *x3, limb *z3,  /* output Q + Q' */
+	limb* x2,
-                   limb *x, limb *z,    /* input Q */
+	limb* z2, /* output 2Q */
-                   limb *xprime, limb *zprime,  /* input Q' */
+	limb* x3,
-                   const limb *qmqp /* input Q - Q' */) {
+	limb* z3, /* output Q + Q' */
-  limb origx[10], origxprime[10], zzz[19], xx[19], zz[19], xxprime[19],
+	limb* x,
-        zzprime[19], zzzprime[19], xxxprime[19];
+	limb* z, /* input Q */
+	limb* xprime,
+	limb* zprime, /* input Q' */
+	const limb* qmqp /* input Q - Q' */)
+{
+	limb origx[10], origxprime[10], zzz[19], xx[19], zz[19], xxprime[19], zzprime[19], zzzprime[19], xxxprime[19];
 
 	memcpy(origx, x, 10 * sizeof(limb));
 	fsum(x, z);
@@ -603,7 +514,8 @@ static inline void fmonty(limb *x2, limb *z2,  /* output 2Q */
 	/* |z2|i| < 2^26 */
 }
 
-static inline void swap_conditional(limb a[19], limb b[19], limb iswap) {
+static inline void swap_conditional(limb a[19], limb b[19], limb iswap)
+{
 	unsigned i;
 	const s32 swap = (s32)-iswap;
 
@@ -614,7 +526,8 @@ static inline void swap_conditional(limb a[19], limb b[19], limb iswap) {
 	}
 }
 
-static inline void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *q) {
+static inline void cmult(limb* resultx, limb* resultz, const u8* n, const limb* q)
+{
 	limb a[19] = { 0 }, b[19] = { 1 }, c[19] = { 1 }, d[19] = { 0 };
 	limb *nqpqx = a, *nqpqz = b, *nqx = c, *nqz = d, *t;
 	limb e[19] = { 0 }, f[19] = { 1 }, g[19] = { 0 }, h[19] = { 1 };
@@ -631,11 +544,7 @@ static inline void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *
 
 			swap_conditional(nqx, nqpqx, bit);
 			swap_conditional(nqz, nqpqz, bit);
-      fmonty(nqx2, nqz2,
+			fmonty(nqx2, nqz2, nqpqx2, nqpqz2, nqx, nqz, nqpqx, nqpqz, q);
-             nqpqx2, nqpqz2,
-             nqx, nqz,
-             nqpqx, nqpqz,
-             q);
 			swap_conditional(nqx2, nqpqx2, bit);
 			swap_conditional(nqz2, nqpqz2, bit);
 
@@ -660,7 +569,8 @@ static inline void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *
 	memcpy(resultz, nqz, sizeof(limb) * 10);
 }
 
-static inline void crecip(limb *out, const limb *z) {
+static inline void crecip(limb* out, const limb* z)
+{
 	limb z2[10];
 	limb z9[10];
 	limb z11[10];
@@ -690,32 +600,50 @@ static inline void crecip(limb *out, const limb *z) {
 
 	/* 2^11 - 2^1 */ fsquare(t0, z2_10_0);
 	/* 2^12 - 2^2 */ fsquare(t1, t0);
-  /* 2^20 - 2^10 */ for (i = 2;i < 10;i += 2) { fsquare(t0,t1); fsquare(t1,t0); }
+	/* 2^20 - 2^10 */ for (i = 2; i < 10; i += 2) {
+		fsquare(t0, t1);
+		fsquare(t1, t0);
+	}
 	/* 2^20 - 2^0 */ fmul(z2_20_0, t1, z2_10_0);
 
 	/* 2^21 - 2^1 */ fsquare(t0, z2_20_0);
 	/* 2^22 - 2^2 */ fsquare(t1, t0);
-  /* 2^40 - 2^20 */ for (i = 2;i < 20;i += 2) { fsquare(t0,t1); fsquare(t1,t0); }
+	/* 2^40 - 2^20 */ for (i = 2; i < 20; i += 2) {
+		fsquare(t0, t1);
+		fsquare(t1, t0);
+	}
 	/* 2^40 - 2^0 */ fmul(t0, t1, z2_20_0);
 
 	/* 2^41 - 2^1 */ fsquare(t1, t0);
 	/* 2^42 - 2^2 */ fsquare(t0, t1);
-  /* 2^50 - 2^10 */ for (i = 2;i < 10;i += 2) { fsquare(t1,t0); fsquare(t0,t1); }
+	/* 2^50 - 2^10 */ for (i = 2; i < 10; i += 2) {
+		fsquare(t1, t0);
+		fsquare(t0, t1);
+	}
 	/* 2^50 - 2^0 */ fmul(z2_50_0, t0, z2_10_0);
 
 	/* 2^51 - 2^1 */ fsquare(t0, z2_50_0);
 	/* 2^52 - 2^2 */ fsquare(t1, t0);
-  /* 2^100 - 2^50 */ for (i = 2;i < 50;i += 2) { fsquare(t0,t1); fsquare(t1,t0); }
+	/* 2^100 - 2^50 */ for (i = 2; i < 50; i += 2) {
+		fsquare(t0, t1);
+		fsquare(t1, t0);
+	}
 	/* 2^100 - 2^0 */ fmul(z2_100_0, t1, z2_50_0);
 
 	/* 2^101 - 2^1 */ fsquare(t1, z2_100_0);
 	/* 2^102 - 2^2 */ fsquare(t0, t1);
-  /* 2^200 - 2^100 */ for (i = 2;i < 100;i += 2) { fsquare(t1,t0); fsquare(t0,t1); }
+	/* 2^200 - 2^100 */ for (i = 2; i < 100; i += 2) {
+		fsquare(t1, t0);
+		fsquare(t0, t1);
+	}
 	/* 2^200 - 2^0 */ fmul(t1, t0, z2_100_0);
 
 	/* 2^201 - 2^1 */ fsquare(t0, t1);
 	/* 2^202 - 2^2 */ fsquare(t1, t0);
-  /* 2^250 - 2^50 */ for (i = 2;i < 50;i += 2) { fsquare(t0,t1); fsquare(t1,t0); }
+	/* 2^250 - 2^50 */ for (i = 2; i < 50; i += 2) {
+		fsquare(t0, t1);
+		fsquare(t1, t0);
+	}
 	/* 2^250 - 2^0 */ fmul(t0, t1, z2_50_0);
 
 	/* 2^251 - 2^1 */ fsquare(t1, t0);
@@ -726,7 +654,8 @@ static inline void crecip(limb *out, const limb *z) {
 	/* 2^255 - 21 */ fmul(out, t1, z11);
 }
 
-static void crypto_scalarmult(u8 *mypublic, const u8 *secret, const u8 *basepoint) {
+static void crypto_scalarmult(u8* mypublic, const u8* secret, const u8* basepoint)
+{
 	limb bp[10], x[10], z[11], zmone[10];
 	uint8_t e[32];
 	int i;
@@ -756,26 +685,19 @@ static inline void crypto_scalarmult_base(unsigned char *q,const unsigned char *
 
 // Ed25519 ref from: http://bench.cr.yp.to/supercop.html
 
-typedef struct
+typedef struct {
-{
 	crypto_uint32 v[32];
-}
+} fe25519;
-fe25519;
 
-typedef struct
+typedef struct {
-{
 	crypto_uint32 v[32];
-}
+} sc25519;
-sc25519;
 
-typedef struct
+typedef struct {
-{
 	crypto_uint32 v[16];
-}
+} shortsc25519;
-shortsc25519;
 
-typedef struct
+typedef struct {
-{
 	fe25519 x;
 	fe25519 y;
 	fe25519 z;
@@ -784,23 +706,20 @@ typedef struct
 
 #define ge25519_p3 ge25519
 
-typedef struct
+typedef struct {
-{
 	fe25519 x;
 	fe25519 z;
 	fe25519 y;
 	fe25519 t;
 } ge25519_p1p1;
 
-typedef struct
+typedef struct {
-{
 	fe25519 x;
 	fe25519 y;
 	fe25519 z;
 } ge25519_p2;
 
-typedef struct
+typedef struct {
-{
 	fe25519 x;
 	fe25519 y;
 } ge25519_aff;
@@ -1051,32 +970,50 @@ static inline void fe25519_invert(fe25519 *r, const fe25519 *x)
 
 	/* 2^11 - 2^1 */ fe25519_square(&t0, &z2_10_0);
 	/* 2^12 - 2^2 */ fe25519_square(&t1, &t0);
-	/* 2^20 - 2^10 */ for (i = 2;i < 10;i += 2) { fe25519_square(&t0,&t1); fe25519_square(&t1,&t0); }
+	/* 2^20 - 2^10 */ for (i = 2; i < 10; i += 2) {
+		fe25519_square(&t0, &t1);
+		fe25519_square(&t1, &t0);
+	}
 	/* 2^20 - 2^0 */ fe25519_mul(&z2_20_0, &t1, &z2_10_0);
 
 	/* 2^21 - 2^1 */ fe25519_square(&t0, &z2_20_0);
 	/* 2^22 - 2^2 */ fe25519_square(&t1, &t0);
-	/* 2^40 - 2^20 */ for (i = 2;i < 20;i += 2) { fe25519_square(&t0,&t1); fe25519_square(&t1,&t0); }
+	/* 2^40 - 2^20 */ for (i = 2; i < 20; i += 2) {
+		fe25519_square(&t0, &t1);
+		fe25519_square(&t1, &t0);
+	}
 	/* 2^40 - 2^0 */ fe25519_mul(&t0, &t1, &z2_20_0);
 
 	/* 2^41 - 2^1 */ fe25519_square(&t1, &t0);
 	/* 2^42 - 2^2 */ fe25519_square(&t0, &t1);
-	/* 2^50 - 2^10 */ for (i = 2;i < 10;i += 2) { fe25519_square(&t1,&t0); fe25519_square(&t0,&t1); }
+	/* 2^50 - 2^10 */ for (i = 2; i < 10; i += 2) {
+		fe25519_square(&t1, &t0);
+		fe25519_square(&t0, &t1);
+	}
 	/* 2^50 - 2^0 */ fe25519_mul(&z2_50_0, &t0, &z2_10_0);
 
 	/* 2^51 - 2^1 */ fe25519_square(&t0, &z2_50_0);
 	/* 2^52 - 2^2 */ fe25519_square(&t1, &t0);
-	/* 2^100 - 2^50 */ for (i = 2;i < 50;i += 2) { fe25519_square(&t0,&t1); fe25519_square(&t1,&t0); }
+	/* 2^100 - 2^50 */ for (i = 2; i < 50; i += 2) {
+		fe25519_square(&t0, &t1);
+		fe25519_square(&t1, &t0);
+	}
 	/* 2^100 - 2^0 */ fe25519_mul(&z2_100_0, &t1, &z2_50_0);
 
 	/* 2^101 - 2^1 */ fe25519_square(&t1, &z2_100_0);
 	/* 2^102 - 2^2 */ fe25519_square(&t0, &t1);
-	/* 2^200 - 2^100 */ for (i = 2;i < 100;i += 2) { fe25519_square(&t1,&t0); fe25519_square(&t0,&t1); }
+	/* 2^200 - 2^100 */ for (i = 2; i < 100; i += 2) {
+		fe25519_square(&t1, &t0);
+		fe25519_square(&t0, &t1);
+	}
 	/* 2^200 - 2^0 */ fe25519_mul(&t1, &t0, &z2_100_0);
 
 	/* 2^201 - 2^1 */ fe25519_square(&t0, &t1);
 	/* 2^202 - 2^2 */ fe25519_square(&t1, &t0);
-	/* 2^250 - 2^50 */ for (i = 2;i < 50;i += 2) { fe25519_square(&t0,&t1); fe25519_square(&t1,&t0); }
+	/* 2^250 - 2^50 */ for (i = 2; i < 50; i += 2) {
+		fe25519_square(&t0, &t1);
+		fe25519_square(&t1, &t0);
+	}
 	/* 2^250 - 2^0 */ fe25519_mul(&t0, &t1, &z2_50_0);
 
 	/* 2^251 - 2^1 */ fe25519_square(&t1, &t0);
@@ -1355,24 +1292,17 @@ static inline void sc25519_2interleave2(unsigned char r[127], const sc25519 *s1,
 }
 
 /* d */
-static const fe25519 ge25519_ecd = {{0xA3, 0x78, 0x59, 0x13, 0xCA, 0x4D, 0xEB, 0x75, 0xAB, 0xD8, 0x41, 0x41, 0x4D, 0x0A, 0x70, 0x00,
+static const fe25519 ge25519_ecd = { { 0xA3, 0x78, 0x59, 0x13, 0xCA, 0x4D, 0xEB, 0x75, 0xAB, 0xD8, 0x41, 0x41, 0x4D, 0x0A, 0x70, 0x00, 0x98, 0xE8, 0x79, 0x77, 0x79, 0x40, 0xC7, 0x8C, 0x73, 0xFE, 0x6F, 0x2B, 0xEE, 0x6C, 0x03, 0x52 } };
-											0x98, 0xE8, 0x79, 0x77, 0x79, 0x40, 0xC7, 0x8C, 0x73, 0xFE, 0x6F, 0x2B, 0xEE, 0x6C, 0x03, 0x52}};
 /* 2*d */
-static const fe25519 ge25519_ec2d = {{0x59, 0xF1, 0xB2, 0x26, 0x94, 0x9B, 0xD6, 0xEB, 0x56, 0xB1, 0x83, 0x82, 0x9A, 0x14, 0xE0, 0x00,
+static const fe25519 ge25519_ec2d = { { 0x59, 0xF1, 0xB2, 0x26, 0x94, 0x9B, 0xD6, 0xEB, 0x56, 0xB1, 0x83, 0x82, 0x9A, 0x14, 0xE0, 0x00, 0x30, 0xD1, 0xF3, 0xEE, 0xF2, 0x80, 0x8E, 0x19, 0xE7, 0xFC, 0xDF, 0x56, 0xDC, 0xD9, 0x06, 0x24 } };
-											 0x30, 0xD1, 0xF3, 0xEE, 0xF2, 0x80, 0x8E, 0x19, 0xE7, 0xFC, 0xDF, 0x56, 0xDC, 0xD9, 0x06, 0x24}};
 /* sqrt(-1) */
-static const fe25519 ge25519_sqrtm1 = {{0xB0, 0xA0, 0x0E, 0x4A, 0x27, 0x1B, 0xEE, 0xC4, 0x78, 0xE4, 0x2F, 0xAD, 0x06, 0x18, 0x43, 0x2F,
+static const fe25519 ge25519_sqrtm1 = { { 0xB0, 0xA0, 0x0E, 0x4A, 0x27, 0x1B, 0xEE, 0xC4, 0x78, 0xE4, 0x2F, 0xAD, 0x06, 0x18, 0x43, 0x2F, 0xA7, 0xD7, 0xFB, 0x3D, 0x99, 0x00, 0x4D, 0x2B, 0x0B, 0xDF, 0xC1, 0x4F, 0x80, 0x24, 0x83, 0x2B } };
-												 0xA7, 0xD7, 0xFB, 0x3D, 0x99, 0x00, 0x4D, 0x2B, 0x0B, 0xDF, 0xC1, 0x4F, 0x80, 0x24, 0x83, 0x2B}};
 
 /* Packed coordinates of the base point */
-static const ge25519 ge25519_base = {{{0x1A, 0xD5, 0x25, 0x8F, 0x60, 0x2D, 0x56, 0xC9, 0xB2, 0xA7, 0x25, 0x95, 0x60, 0xC7, 0x2C, 0x69,
+static const ge25519 ge25519_base = { { { 0x1A, 0xD5, 0x25, 0x8F, 0x60, 0x2D, 0x56, 0xC9, 0xB2, 0xA7, 0x25, 0x95, 0x60, 0xC7, 0x2C, 0x69, 0x5C, 0xDC, 0xD6, 0xFD, 0x31, 0xE2, 0xA4, 0xC0, 0xFE, 0x53, 0x6E, 0xCD, 0xD3, 0x36, 0x69, 0x21 } },
-																0x5C, 0xDC, 0xD6, 0xFD, 0x31, 0xE2, 0xA4, 0xC0, 0xFE, 0x53, 0x6E, 0xCD, 0xD3, 0x36, 0x69, 0x21}},
+									  { { 0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } },
-															{{0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66,
+									  { { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-																0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66}},
+									  { { 0xA3, 0xDD, 0xB7, 0xA5, 0xB3, 0x8A, 0xDE, 0x6D, 0xF5, 0x52, 0x51, 0x77, 0x80, 0x9F, 0xF0, 0x20, 0x7D, 0xE3, 0xAB, 0x64, 0x8E, 0x4E, 0xEA, 0x66, 0x65, 0x76, 0x8B, 0xD7, 0x0F, 0x5F, 0x87, 0x67 } } };
-															{{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-																0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
-															{{0xA3, 0xDD, 0xB7, 0xA5, 0xB3, 0x8A, 0xDE, 0x6D, 0xF5, 0x52, 0x51, 0x77, 0x80, 0x9F, 0xF0, 0x20,
-																0x7D, 0xE3, 0xAB, 0x64, 0x8E, 0x4E, 0xEA, 0x66, 0x65, 0x76, 0x8B, 0xD7, 0x0F, 0x5F, 0x87, 0x67}}};
 
 /* Multiples of the base point in affine representation */
 static const ge25519_aff ge25519_base_multiples_affine[425] = {
@@ -2429,20 +2359,33 @@ static inline void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge2551
 	/* precomputation                                                        s2 s1 */
 	setneutral(pre); /* 00 00 */
 	pre[1] = *p1;	 /* 00 01 */
-	dbl_p1p1(&tp1p1,(ge25519_p2 *)p1);      p1p1_to_p3( &pre[2], &tp1p1); /* 00 10 */
+	dbl_p1p1(&tp1p1, (ge25519_p2*)p1);
-	add_p1p1(&tp1p1,&pre[1], &pre[2]);      p1p1_to_p3( &pre[3], &tp1p1); /* 00 11 */
+	p1p1_to_p3(&pre[2], &tp1p1); /* 00 10 */
+	add_p1p1(&tp1p1, &pre[1], &pre[2]);
+	p1p1_to_p3(&pre[3], &tp1p1); /* 00 11 */
 	pre[4] = *p2;				 /* 01 00 */
-	add_p1p1(&tp1p1,&pre[1], &pre[4]);      p1p1_to_p3( &pre[5], &tp1p1); /* 01 01 */
+	add_p1p1(&tp1p1, &pre[1], &pre[4]);
-	add_p1p1(&tp1p1,&pre[2], &pre[4]);      p1p1_to_p3( &pre[6], &tp1p1); /* 01 10 */
+	p1p1_to_p3(&pre[5], &tp1p1); /* 01 01 */
-	add_p1p1(&tp1p1,&pre[3], &pre[4]);      p1p1_to_p3( &pre[7], &tp1p1); /* 01 11 */
+	add_p1p1(&tp1p1, &pre[2], &pre[4]);
-	dbl_p1p1(&tp1p1,(ge25519_p2 *)p2);      p1p1_to_p3( &pre[8], &tp1p1); /* 10 00 */
+	p1p1_to_p3(&pre[6], &tp1p1); /* 01 10 */
-	add_p1p1(&tp1p1,&pre[1], &pre[8]);      p1p1_to_p3( &pre[9], &tp1p1); /* 10 01 */
+	add_p1p1(&tp1p1, &pre[3], &pre[4]);
-	dbl_p1p1(&tp1p1,(ge25519_p2 *)pre5);    p1p1_to_p3(&pre[10], &tp1p1); /* 10 10 */
+	p1p1_to_p3(&pre[7], &tp1p1); /* 01 11 */
-	add_p1p1(&tp1p1,&pre[3], &pre[8]);      p1p1_to_p3(&pre[11], &tp1p1); /* 10 11 */
+	dbl_p1p1(&tp1p1, (ge25519_p2*)p2);
-	add_p1p1(&tp1p1,&pre[4], &pre[8]);      p1p1_to_p3(&pre[12], &tp1p1); /* 11 00 */
+	p1p1_to_p3(&pre[8], &tp1p1); /* 10 00 */
-	add_p1p1(&tp1p1,&pre[1],&pre[12]);      p1p1_to_p3(&pre[13], &tp1p1); /* 11 01 */
+	add_p1p1(&tp1p1, &pre[1], &pre[8]);
-	add_p1p1(&tp1p1,&pre[2],&pre[12]);      p1p1_to_p3(&pre[14], &tp1p1); /* 11 10 */
+	p1p1_to_p3(&pre[9], &tp1p1); /* 10 01 */
-	add_p1p1(&tp1p1,&pre[3],&pre[12]);      p1p1_to_p3(&pre[15], &tp1p1); /* 11 11 */
+	dbl_p1p1(&tp1p1, (ge25519_p2*)pre5);
+	p1p1_to_p3(&pre[10], &tp1p1); /* 10 10 */
+	add_p1p1(&tp1p1, &pre[3], &pre[8]);
+	p1p1_to_p3(&pre[11], &tp1p1); /* 10 11 */
+	add_p1p1(&tp1p1, &pre[4], &pre[8]);
+	p1p1_to_p3(&pre[12], &tp1p1); /* 11 00 */
+	add_p1p1(&tp1p1, &pre[1], &pre[12]);
+	p1p1_to_p3(&pre[13], &tp1p1); /* 11 01 */
+	add_p1p1(&tp1p1, &pre[2], &pre[12]);
+	p1p1_to_p3(&pre[14], &tp1p1); /* 11 10 */
+	add_p1p1(&tp1p1, &pre[3], &pre[12]);
+	p1p1_to_p3(&pre[15], &tp1p1); /* 11 11 */
 
 	sc25519_2interleave2(b, s1, s2);
 
@@ -2458,7 +2401,8 @@ static inline void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge2551
 		}
 		if (i != 0) {
 			p1p1_to_p2((ge25519_p2*)r, &tp1p1);
-		} else {
+		}
+		else {
 			p1p1_to_p3(r, &tp1p1);
 		}
 	}

node/C25519.hpp

@@ -25,13 +25,21 @@ namespace ZeroTier {
 /**
  * A combined Curve25519 ECDH and Ed25519 signature engine
  */
-class C25519
+class C25519 {
-{
   public:
-	struct Public { uint8_t data[ZT_C25519_PUBLIC_KEY_LEN]; };
+	struct Public {
-	struct Private { uint8_t data[ZT_C25519_PRIVATE_KEY_LEN]; };
+		uint8_t data[ZT_C25519_PUBLIC_KEY_LEN];
-	struct Signature { uint8_t data[ZT_C25519_SIGNATURE_LEN]; };
+	};
-	struct Pair { Public pub; Private priv; };
+	struct Private {
+		uint8_t data[ZT_C25519_PRIVATE_KEY_LEN];
+	};
+	struct Signature {
+		uint8_t data[ZT_C25519_SIGNATURE_LEN];
+	};
+	struct Pair {
+		Public pub;
+		Private priv;
+	};
 
 	/**
 	 * Generate a C25519 elliptic curve key pair
@@ -58,8 +66,7 @@ public:
 	 * @return Key pair where cond(kp) returns true
 	 * @tparam F Type of 'cond'
 	 */
-	template<typename F>
+	template <typename F> static inline Pair generateSatisfying(F cond)
-	static inline Pair generateSatisfying(F cond)
 	{
 		Pair kp;
 		void* const priv = (void*)kp.priv.data;
@@ -85,7 +92,10 @@ public:
 	 * @param keylen Number of key bytes to generate
 	 */
 	static void agree(const Private& mine, const Public& their, void* keybuf, unsigned int keylen);
-	static inline void agree(const Pair &mine,const Public &their,void *keybuf,unsigned int keylen) { agree(mine.priv,their,keybuf,keylen); }
+	static inline void agree(const Pair& mine, const Public& their, void* keybuf, unsigned int keylen)
+	{
+		agree(mine.priv, their, keybuf, keylen);
+	}
 
 	/**
 	 * Sign a message with a sender's key pair
@@ -107,7 +117,10 @@ public:
 	 * @param signature Buffer to fill with signature -- MUST be 96 bytes in length
 	 */
 	static void sign(const Private& myPrivate, const Public& myPublic, const void* msg, unsigned int len, void* signature);
-	static inline void sign(const Pair &mine,const void *msg,unsigned int len,void *signature) { sign(mine.priv,mine.pub,msg,len,signature); }
+	static inline void sign(const Pair& mine, const void* msg, unsigned int len, void* signature)
+	{
+		sign(mine.priv, mine.pub, msg, len, signature);
+	}
 
 	/**
 	 * Sign a message with a sender's key pair

node/Capability.cpp

@@ -12,12 +12,13 @@
 /****/
 
 #include "Capability.hpp"
-#include "RuntimeEnvironment.hpp"
+
 #include "Identity.hpp"
-#include "Topology.hpp"
-#include "Switch.hpp"
 #include "Network.hpp"
 #include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
 
 namespace ZeroTier {
 
@@ -37,10 +38,12 @@ int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const
 				if ((! _custody[c].to) || (! _custody[c].from) || (_custody[c].from != Network::controllerFor(_nwid))) {
 					return -1;	 // the first entry must be present and from the network's controller
 				}
-			} else {
+			}
+			else {
 				if (! _custody[c].to) {
 					return 0;	// all previous entries were valid, so we are valid
-				} else if ((!_custody[c].from)||(_custody[c].from != _custody[c-1].to)) {
+				}
+				else if ((! _custody[c].from) || (_custody[c].from != _custody[c - 1].to)) {
 					return -1;	 // otherwise if we have another entry it must be from the previous holder in the chain
 				}
 			}
@@ -50,7 +53,8 @@ int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const
 				if (! id.verify(tmp.data(), tmp.size(), _custody[c].signature)) {
 					return -1;
 				}
-			} else {
+			}
+			else {
 				RR->sw->requestWhois(tPtr, RR->node->now(), _custody[c].from);
 				return 1;
 			}
@@ -58,7 +62,9 @@ int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const
 
 		// We reached max custody chain length and everything was valid
 		return 0;
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 	return -1;
 }
 

node/Capability.hpp

@@ -14,19 +14,19 @@
 #ifndef ZT_CAPABILITY_HPP
 #define ZT_CAPABILITY_HPP
 
+#include "../include/ZeroTierOne.h"
+#include "Address.hpp"
+#include "Buffer.hpp"
+#include "C25519.hpp"
+#include "Constants.hpp"
+#include "Credential.hpp"
+#include "Identity.hpp"
+#include "Utils.hpp"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "Constants.hpp"
-#include "Credential.hpp"
-#include "Address.hpp"
-#include "C25519.hpp"
-#include "Utils.hpp"
-#include "Buffer.hpp"
-#include "Identity.hpp"
-#include "../include/ZeroTierOne.h"
-
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -54,17 +54,14 @@ class RuntimeEnvironment;
  * handed off between nodes. Limited transferability of capabilities is
  * a feature of true capability based security.
  */
-class Capability : public Credential
+class Capability : public Credential {
-{
   public:
-	static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_CAPABILITY; }
+	static inline Credential::Type credentialType()
+	{
+		return Credential::CREDENTIAL_TYPE_CAPABILITY;
+	}
 
-	Capability() :
+	Capability() : _nwid(0), _ts(0), _id(0), _maxCustodyChainLength(0), _ruleCount(0)
-		_nwid(0),
-		_ts(0),
-		_id(0),
-		_maxCustodyChainLength(0),
-		_ruleCount(0)
 	{
 		memset(_rules, 0, sizeof(_rules));
 		memset(_custody, 0, sizeof(_custody));
@@ -78,12 +75,12 @@ public:
 	 * @param rules Network flow rules for this capability
 	 * @param ruleCount Number of flow rules
 	 */
-	Capability(uint32_t id,uint64_t nwid,int64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) :
+	Capability(uint32_t id, uint64_t nwid, int64_t ts, unsigned int mccl, const ZT_VirtualNetworkRule* rules, unsigned int ruleCount)
-		_nwid(nwid),
+		: _nwid(nwid)
-		_ts(ts),
+		, _ts(ts)
-		_id(id),
+		, _id(id)
-		_maxCustodyChainLength((mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1),
+		, _maxCustodyChainLength((mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1)
-		_ruleCount((ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES)
+		, _ruleCount((ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES)
 	{
 		if (_ruleCount > 0) {
 			memcpy(_rules, rules, sizeof(ZT_VirtualNetworkRule) * _ruleCount);
@@ -93,27 +90,42 @@ public:
 	/**
 	 * @return Rules -- see ruleCount() for size of array
 	 */
-	inline const ZT_VirtualNetworkRule *rules() const { return _rules; }
+	inline const ZT_VirtualNetworkRule* rules() const
+	{
+		return _rules;
+	}
 
 	/**
 	 * @return Number of rules in rules()
 	 */
-	inline unsigned int ruleCount() const { return _ruleCount; }
+	inline unsigned int ruleCount() const
+	{
+		return _ruleCount;
+	}
 
 	/**
 	 * @return ID and evaluation order of this capability in network
 	 */
-	inline uint32_t id() const { return _id; }
+	inline uint32_t id() const
+	{
+		return _id;
+	}
 
 	/**
 	 * @return Network ID for which this capability was issued
 	 */
-	inline uint64_t networkId() const { return _nwid; }
+	inline uint64_t networkId() const
+	{
+		return _nwid;
+	}
 
 	/**
 	 * @return Timestamp
 	 */
-	inline int64_t timestamp() const { return _ts; }
+	inline int64_t timestamp() const
+	{
+		return _ts;
+	}
 
 	/**
 	 * @return Last 'to' address in chain of custody
@@ -124,7 +136,8 @@ public:
 		for (unsigned int i = 0; i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH; ++i) {
 			if (! _custody[i].to) {
 				return i2;
-			} else {
+			}
+			else {
 				i2 = _custody[i].to;
 			}
 		}
@@ -157,7 +170,9 @@ public:
 					return true;
 				}
 			}
-		} catch ( ... ) {}
+		}
+		catch (...) {
+		}
 		return false;
 	}
 
@@ -169,8 +184,7 @@ public:
 	 */
 	int verify(const RuntimeEnvironment* RR, void* tPtr) const;
 
-	template<unsigned int C>
+	template <unsigned int C> static inline void serializeRules(Buffer<C>& b, const ZT_VirtualNetworkRule* rules, unsigned int ruleCount)
-	static inline void serializeRules(Buffer<C> &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount)
 	{
 		for (unsigned int i = 0; i < ruleCount; ++i) {
 			// Each rule consists of its 8-bit type followed by the size of that type's
@@ -284,8 +298,7 @@ public:
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> static inline void deserializeRules(const Buffer<C>& b, unsigned int& p, ZT_VirtualNetworkRule* rules, unsigned int& ruleCount, const unsigned int maxRuleCount)
-	static inline void deserializeRules(const Buffer<C> &b,unsigned int &p,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,const unsigned int maxRuleCount)
 	{
 		while ((ruleCount < maxRuleCount) && (p < b.size())) {
 			rules[ruleCount].t = (uint8_t)b[p++];
@@ -380,8 +393,7 @@ public:
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline void serialize(Buffer<C>& b, const bool forSign = false) const
-	inline void serialize(Buffer<C> &b,const bool forSign = false) const
 	{
 		if (forSign) {
 			b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL);
@@ -404,7 +416,8 @@ public:
 					b.append((uint8_t)1);						   // 1 == Ed25519 signature
 					b.append((uint16_t)ZT_C25519_SIGNATURE_LEN);   // length of signature
 					b.append(_custody[i].signature.data, ZT_C25519_SIGNATURE_LEN);
-				} else {
+				}
+				else {
 					b.append((unsigned char)0, ZT_ADDRESS_LENGTH);	 // zero 'to' terminates chain
 					break;
 				}
@@ -419,8 +432,7 @@ public:
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline unsigned int deserialize(const Buffer<C>& b, unsigned int startAt = 0)
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		*this = Capability();
 
@@ -464,7 +476,8 @@ public:
 				p += 2;
 				memcpy(_custody[i].signature.data, b.field(p, ZT_C25519_SIGNATURE_LEN), ZT_C25519_SIGNATURE_LEN);
 				p += ZT_C25519_SIGNATURE_LEN;
-			} else {
+			}
+			else {
 				p += 2 + b.template at<uint16_t>(p);
 			}
 		}
@@ -478,10 +491,19 @@ public:
 	}
 
 	// Provides natural sort order by ID
-	inline bool operator<(const Capability &c) const { return (_id < c._id); }
+	inline bool operator<(const Capability& c) const
+	{
+		return (_id < c._id);
+	}
 
-	inline bool operator==(const Capability &c) const { return (memcmp(this,&c,sizeof(Capability)) == 0); }
+	inline bool operator==(const Capability& c) const
-	inline bool operator!=(const Capability &c) const { return (memcmp(this,&c,sizeof(Capability)) != 0); }
+	{
+		return (memcmp(this, &c, sizeof(Capability)) == 0);
+	}
+	inline bool operator!=(const Capability& c) const
+	{
+		return (memcmp(this, &c, sizeof(Capability)) != 0);
+	}
 
   private:
 	uint64_t _nwid;

node/CertificateOfMembership.cpp

@@ -12,11 +12,12 @@
 /****/
 
 #include "CertificateOfMembership.hpp"
-#include "RuntimeEnvironment.hpp"
+
-#include "Topology.hpp"
-#include "Switch.hpp"
 #include "Network.hpp"
 #include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
 
 namespace ZeroTier {
 
@@ -107,7 +108,8 @@ bool CertificateOfMembership::sign(const Identity &with)
 		_signature = with.sign(buf, ptr * sizeof(uint64_t));
 		_signedBy = with.address();
 		return true;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		_signedBy.zero();
 		return false;
 	}

node/CertificateOfMembership.hpp

@@ -14,21 +14,20 @@
 #ifndef ZT_CERTIFICATEOFMEMBERSHIP_HPP
 #define ZT_CERTIFICATEOFMEMBERSHIP_HPP
 
-#include <stdint.h>
+#include "Address.hpp"
-#include <string.h>
+#include "Buffer.hpp"
-
+#include "C25519.hpp"
-#include <string>
-#include <stdexcept>
-#include <algorithm>
-
 #include "Constants.hpp"
 #include "Credential.hpp"
-#include "Buffer.hpp"
-#include "Address.hpp"
-#include "C25519.hpp"
 #include "Identity.hpp"
 #include "Utils.hpp"
 
+#include <algorithm>
+#include <stdexcept>
+#include <stdint.h>
+#include <string.h>
+#include <string>
+
 /**
  * Maximum number of qualifiers allowed in a COM (absolute max: 65535)
  */
@@ -64,10 +63,12 @@ class RuntimeEnvironment;
  * This is a memcpy()'able structure and is safe (in a crash sense) to modify
  * without locks.
  */
-class CertificateOfMembership : public Credential
+class CertificateOfMembership : public Credential {
-{
   public:
-	static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COM; }
+	static inline Credential::Type credentialType()
+	{
+		return Credential::CREDENTIAL_TYPE_COM;
+	}
 
 	/**
 	 * Reserved qualifier IDs
@@ -78,8 +79,7 @@ public:
 	 * Addition of new required fields requires that code in hasRequiredFields
 	 * be updated as well.
 	 */
-	enum ReservedId
+	enum ReservedId {
-	{
 		/**
 		 * Timestamp of certificate
 		 */
@@ -101,8 +101,9 @@ public:
 	/**
 	 * Create an empty certificate of membership
 	 */
-	CertificateOfMembership() :
+	CertificateOfMembership() : _qualifierCount(0)
-		_qualifierCount(0) {}
+	{
+	}
 
 	/**
 	 * Create from required fields common to all networks
@@ -120,8 +121,7 @@ public:
 	 * @param b Buffer to deserialize from
 	 * @param startAt Position to start in buffer
 	 */
-	template<unsigned int C>
+	template <unsigned int C> CertificateOfMembership(const Buffer<C>& b, unsigned int startAt = 0)
-	CertificateOfMembership(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		deserialize(b, startAt);
 	}
@@ -129,12 +129,18 @@ public:
 	/**
 	 * @return True if there's something here
 	 */
-	inline operator bool() const { return (_qualifierCount != 0); }
+	inline operator bool() const
+	{
+		return (_qualifierCount != 0);
+	}
 
 	/**
 	 * @return Credential ID, always 0 for COMs
 	 */
-	inline uint32_t id() const { return 0; }
+	inline uint32_t id() const
+	{
+		return 0;
+	}
 
 	/**
 	 * @return Timestamp for this cert and maximum delta for timestamp
@@ -211,15 +217,20 @@ public:
 	/**
 	 * @return True if signed
 	 */
-	inline bool isSigned() const { return (_signedBy); }
+	inline bool isSigned() const
+	{
+		return (_signedBy);
+	}
 
 	/**
 	 * @return Address that signed this certificate or null address if none
 	 */
-	inline const Address &signedBy() const { return _signedBy; }
+	inline const Address& signedBy() const
+	{
+		return _signedBy;
+	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline void serialize(Buffer<C>& b) const
-	inline void serialize(Buffer<C> &b) const
 	{
 		b.append((uint8_t)1);
 		b.append((uint16_t)_qualifierCount);
@@ -234,8 +245,7 @@ public:
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline unsigned int deserialize(const Buffer<C>& b, unsigned int startAt = 0)
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
 
@@ -253,7 +263,8 @@ public:
 			const uint64_t qid = b.template at<uint64_t>(p);
 			if (qid < lastId) {
 				throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_BAD_ENCODING;
-			} else {
+			}
+			else {
 				lastId = qid;
 			}
 			if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
@@ -262,7 +273,8 @@ public:
 				_qualifiers[_qualifierCount].maxDelta = b.template at<uint64_t>(p + 16);
 				p += 24;
 				++_qualifierCount;
-			} else {
+			}
+			else {
 				throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW;
 			}
 		}
@@ -295,16 +307,23 @@ public:
 		}
 		return (memcmp(_signature.data, c._signature.data, ZT_C25519_SIGNATURE_LEN) == 0);
 	}
-	inline bool operator!=(const CertificateOfMembership &c) const { return (!(*this == c)); }
+	inline bool operator!=(const CertificateOfMembership& c) const
+	{
+		return (! (*this == c));
+	}
 
   private:
-	struct _Qualifier
+	struct _Qualifier {
+		_Qualifier() : id(0), value(0), maxDelta(0)
 		{
-		_Qualifier() : id(0),value(0),maxDelta(0) {}
+		}
 		uint64_t id;
 		uint64_t value;
 		uint64_t maxDelta;
-		inline bool operator<(const _Qualifier &q) const { return (id < q.id); } // sort order
+		inline bool operator<(const _Qualifier& q) const
+		{
+			return (id < q.id);
+		}	// sort order
 	};
 
 	Address _signedBy;

node/CertificateOfOwnership.cpp

@@ -12,12 +12,13 @@
 /****/
 
 #include "CertificateOfOwnership.hpp"
-#include "RuntimeEnvironment.hpp"
+
 #include "Identity.hpp"
-#include "Topology.hpp"
-#include "Switch.hpp"
 #include "Network.hpp"
 #include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
 
 namespace ZeroTier {
 
@@ -35,7 +36,8 @@ int CertificateOfOwnership::verify(const RuntimeEnvironment *RR,void *tPtr) cons
 		Buffer<(sizeof(CertificateOfOwnership) + 64)> tmp;
 		this->serialize(tmp, true);
 		return (id.verify(tmp.data(), tmp.size(), _signature) ? 0 : -1);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return -1;
 	}
 }

node/CertificateOfOwnership.hpp

@@ -14,20 +14,20 @@
 #ifndef ZT_CERTIFICATEOFOWNERSHIP_HPP
 #define ZT_CERTIFICATEOFOWNERSHIP_HPP
 
+#include "Address.hpp"
+#include "Buffer.hpp"
+#include "C25519.hpp"
+#include "Constants.hpp"
+#include "Credential.hpp"
+#include "Identity.hpp"
+#include "InetAddress.hpp"
+#include "MAC.hpp"
+
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "Constants.hpp"
-#include "Credential.hpp"
-#include "C25519.hpp"
-#include "Address.hpp"
-#include "Identity.hpp"
-#include "Buffer.hpp"
-#include "InetAddress.hpp"
-#include "MAC.hpp"
-
 // Max things per CertificateOfOwnership
 #define ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS 16
 
@@ -41,18 +41,14 @@ class RuntimeEnvironment;
 /**
  * Certificate indicating ownership of a network identifier
  */
-class CertificateOfOwnership : public Credential
+class CertificateOfOwnership : public Credential {
-{
   public:
-	static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COO; }
+	static inline Credential::Type credentialType()
-
-	enum Thing
 	{
-		THING_NULL = 0,
+		return Credential::CREDENTIAL_TYPE_COO;
-		THING_MAC_ADDRESS = 1,
+	}
-		THING_IPV4_ADDRESS = 2,
+
-		THING_IPV6_ADDRESS = 3
+	enum Thing { THING_NULL = 0, THING_MAC_ADDRESS = 1, THING_IPV4_ADDRESS = 2, THING_IPV6_ADDRESS = 3 };
-	};
 
 	CertificateOfOwnership()
 	{
@@ -68,15 +64,36 @@ public:
 		_issuedTo = issuedTo;
 	}
 
-	inline uint64_t networkId() const { return _networkId; }
+	inline uint64_t networkId() const
-	inline int64_t timestamp() const { return _ts; }
+	{
-	inline uint32_t id() const { return _id; }
+		return _networkId;
-	inline unsigned int thingCount() const { return (unsigned int)_thingCount; }
+	}
+	inline int64_t timestamp() const
+	{
+		return _ts;
+	}
+	inline uint32_t id() const
+	{
+		return _id;
+	}
+	inline unsigned int thingCount() const
+	{
+		return (unsigned int)_thingCount;
+	}
 
-	inline Thing thingType(const unsigned int i) const { return (Thing)_thingTypes[i]; }
+	inline Thing thingType(const unsigned int i) const
-	inline const uint8_t *thingValue(const unsigned int i) const { return _thingValues[i]; }
+	{
+		return (Thing)_thingTypes[i];
+	}
+	inline const uint8_t* thingValue(const unsigned int i) const
+	{
+		return _thingValues[i];
+	}
 
-	inline const Address &issuedTo() const { return _issuedTo; }
+	inline const Address& issuedTo() const
+	{
+		return _issuedTo;
+	}
 
 	inline bool owns(const InetAddress& ip) const
 	{
@@ -105,7 +122,8 @@ public:
 			_thingTypes[_thingCount] = THING_IPV4_ADDRESS;
 			memcpy(_thingValues[_thingCount], &(reinterpret_cast<const struct sockaddr_in*>(&ip)->sin_addr.s_addr), 4);
 			++_thingCount;
-		} else if (ip.ss_family == AF_INET6) {
+		}
+		else if (ip.ss_family == AF_INET6) {
 			_thingTypes[_thingCount] = THING_IPV6_ADDRESS;
 			memcpy(_thingValues[_thingCount], reinterpret_cast<const struct sockaddr_in6*>(&ip)->sin6_addr.s6_addr, 16);
 			++_thingCount;
@@ -145,8 +163,7 @@ public:
 	 */
 	int verify(const RuntimeEnvironment* RR, void* tPtr) const;
 
-	template<unsigned int C>
+	template <unsigned int C> inline void serialize(Buffer<C>& b, const bool forSign = false) const
-	inline void serialize(Buffer<C> &b,const bool forSign = false) const
 	{
 		if (forSign) {
 			b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL);
@@ -177,8 +194,7 @@ public:
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline unsigned int deserialize(const Buffer<C>& b, unsigned int startAt = 0)
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
 
@@ -213,7 +229,8 @@ public:
 			p += 2;
 			memcpy(_signature.data, b.field(p, ZT_C25519_SIGNATURE_LEN), ZT_C25519_SIGNATURE_LEN);
 			p += ZT_C25519_SIGNATURE_LEN;
-		} else {
+		}
+		else {
 			p += 2 + b.template at<uint16_t>(p);
 		}
 
@@ -226,10 +243,19 @@ public:
 	}
 
 	// Provides natural sort order by ID
-	inline bool operator<(const CertificateOfOwnership &coo) const { return (_id < coo._id); }
+	inline bool operator<(const CertificateOfOwnership& coo) const
+	{
+		return (_id < coo._id);
+	}
 
-	inline bool operator==(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) == 0); }
+	inline bool operator==(const CertificateOfOwnership& coo) const
-	inline bool operator!=(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) != 0); }
+	{
+		return (memcmp(this, &coo, sizeof(CertificateOfOwnership)) == 0);
+	}
+	inline bool operator!=(const CertificateOfOwnership& coo) const
+	{
+		return (memcmp(this, &coo, sizeof(CertificateOfOwnership)) != 0);
+	}
 
   private:
 	bool _owns(const Thing& t, const void* v, unsigned int l) const;

node/Constants.hpp

@@ -106,8 +106,8 @@
 #pragma warning(disable : 4101)
 #undef __UNIX_LIKE__
 #undef __BSD__
-#include <winsock2.h>
 #include <windows.h>
+#include <winsock2.h>
 #endif
 
 #ifdef __NetBSD__
@@ -118,9 +118,9 @@
 
 #if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64))
 #define ZT_ARCH_X64 1
-#include <xmmintrin.h>
 #include <emmintrin.h>
 #include <immintrin.h>
+#include <xmmintrin.h>
 #endif
 
 #if (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(ZT_ARCH_ARM_HAS_NEON))
@@ -145,7 +145,8 @@
 #endif
 
 // Define ZT_NO_TYPE_PUNNING to disable reckless casts on anything other than x86/x64.
-#if (!(defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_AMD64) || defined(_M_X64) || defined(i386) || defined(__i386) || defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) || defined(__X86__) || defined(_X86_) || defined(__I86__) || defined(__INTEL__) || defined(__386)))
+#if (! (defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_AMD64) || defined(_M_X64) || defined(i386) || defined(__i386) || defined(__i386__) || defined(__i486__) || defined(__i586__)          \
+		|| defined(__i686__) || defined(_M_IX86) || defined(__X86__) || defined(_X86_) || defined(__I86__) || defined(__INTEL__) || defined(__386)))
 #ifndef ZT_NO_TYPE_PUNNING
 #define ZT_NO_TYPE_PUNNING 1
 #endif
@@ -202,6 +203,72 @@
 #define ZT_PACKED_STRUCT(D) D __attribute__((packed))
 #endif
 
+#if defined(_WIN32)
+#define ZT_PLATFORM_NAME "windows"	 // Windows
+#elif defined(_WIN64)
+#define ZT_PLATFORM_NAME "windows"	 // Windows
+#elif defined(__CYGWIN__)
+#define ZT_PLATFORM_NAME "windows"	 // Windows (Cygwin POSIX under Microsoft Window)
+#elif defined(__ANDROID__)
+#define ZT_PLATFORM_NAME "android"	 // Android (implies Linux, so it must come first)
+#elif defined(__linux__)
+#define ZT_PLATFORM_NAME "linux"   // Debian, Ubuntu, Gentoo, Fedora, openSUSE, RedHat, Centos and other
+#elif defined(__unix__) || ! defined(__APPLE__) && defined(__MACH__)
+#include <sys/param.h>
+#if defined(BSD)
+#define ZT_PLATFORM_NAME "bsd"	 // FreeBSD, NetBSD, OpenBSD, DragonFly BSD
+#endif
+#elif defined(__hpux)
+#define ZT_PLATFORM_NAME "hp-ux"   // HP-UX
+#elif defined(_AIX)
+#define ZT_PLATFORM_NAME "aix"					// IBM AIX
+#elif defined(__APPLE__) && defined(__MACH__)	// Apple OSX and iOS (Darwin)
+#include <TargetConditionals.h>
+#if defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR == 1
+#define ZT_PLATFORM_NAME "ios_sim"	 // Apple iOS
+#elif defined(TARGET_OS_IPAD) && TARGET_OS_IPAD == 1
+#define ZT_PLATFORM_NAME "ios_ipad"
+#elif defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1
+#define ZT_PLATFORM_NAME "ios_iphone"	// Apple iOS
+#elif defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1
+#define ZT_PLATFORM_NAME "macos"   // Apple OSX
+#endif
+#elif defined(__sun) && defined(__SVR4)
+#define ZT_PLATFORM_NAME "solaris"	 // Oracle Solaris, Open Indiana
+#else
+#define ZT_PLATFORM_NAME "unknown"
+#endif
+#ifndef ZT_PLATFORM_NAME
+#define ZT_PLATFORM_NAME "unknown"
+#endif
+
+#if defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64) || defined(_M_AMD64)
+#define ZT_ARCH_NAME "x86_64"
+#elif defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_X86_) || defined(_M_IX86) || defined(__X86__) || defined(__I86__) || defined(_M_I86)
+#define ZT_ARCH_NAME "x86"
+#elif defined(__aarch64__) || defined(__AARCH64EL__) || defined(_M_ARM64)
+#define ZT_ARCH_NAME "arm64"
+#elif defined(__arm__) || defined(__TARGET_ARCH_ARM) || defined(_ARM) || defined(_M_ARM) || defined(_M_ARMT) || defined(__arm) || defined(__thumb__)
+#define ZT_ARCH_NAME "arm"
+#elif defined(__loongarch__) || defined(_LOONGARCH_ARCH)
+#define ZT_ARCH_NAME "loongarch"
+#elif defined(__mips__) || defined(__MIPS__)
+#define ZT_ARCH_NAME "mips"
+#elif defined(__riscv) || defined(__riscv_xlen)
+#define ZT_ARCH_NAME "riscv"
+#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) || defined(__ppc64__) || defined(_M_PPC)
+#define ZT_ARCH_NAME "powerpc"
+#elif defined(__s390__) || defined(__s390x__) || defined(__zarch__)
+#define ZT_ARCH_NAME "s390"
+#else
+#define ZT_ARCH_NAME "unknown"
+#endif
+#ifndef ZT_ARCH_NAME
+#define ZT_ARCH_NAME "unknown"
+#endif
+
+#define ZT_TARGET_NAME (ZT_PLATFORM_NAME "/" ZT_ARCH_NAME)
+
 /**
  * Length of a ZeroTier address in bytes
  */
@@ -566,7 +633,6 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 1000
 
-
 /**
  * Minimum allowed amount of time between flow/path optimizations (anti-flapping)
  */

node/Credential.hpp

@@ -14,30 +14,27 @@
 #ifndef ZT_CREDENTIAL_HPP
 #define ZT_CREDENTIAL_HPP
 
-#include <string>
+#include "Constants.hpp"
+
 #include <memory>
 #include <stdexcept>
-
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
 #include <string.h>
-
+#include <string>
-#include "Constants.hpp"
 
 namespace ZeroTier {
 
 /**
  * Base class for credentials
  */
-class Credential
+class Credential {
-{
   public:
 	/**
 	 * Do not change type code IDs -- these are used in Revocation objects and elsewhere
 	 */
-	enum Type
+	enum Type {
-	{
 		CREDENTIAL_TYPE_NULL = 0,
 		CREDENTIAL_TYPE_COM = 1,   // CertificateOfMembership
 		CREDENTIAL_TYPE_CAPABILITY = 2,

node/DNS.hpp

@@ -13,14 +13,14 @@
 
 #ifndef ZT_DNS_HPP
 #define ZT_DNS_HPP
+#include "../include/ZeroTierOne.h"
+#include "Buffer.hpp"
+#include "InetAddress.hpp"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "Buffer.hpp"
-#include "InetAddress.hpp"
-#include "../include/ZeroTierOne.h"
-
 namespace ZeroTier {
 
 /**
@@ -28,8 +28,7 @@ namespace ZeroTier {
  */
 class DNS {
   public:
-    template<unsigned int C>
+	template <unsigned int C> static inline void serializeDNS(Buffer<C>& b, const ZT_VirtualNetworkDNS* dns)
-    static inline void serializeDNS(Buffer<C> &b, const ZT_VirtualNetworkDNS *dns)
 	{
 		b.append(dns->domain, 128);
 		for (unsigned int j = 0; j < ZT_MAX_DNS_SERVERS; ++j) {
@@ -38,8 +37,7 @@ public:
 		}
 	}
 
-    template<unsigned int C>
+	template <unsigned int C> static inline void deserializeDNS(const Buffer<C>& b, unsigned int& p, ZT_VirtualNetworkDNS* dns)
-    static inline void deserializeDNS(const Buffer<C> &b, unsigned int &p, ZT_VirtualNetworkDNS *dns)
 	{
 		char* d = (char*)b.data() + p;
 		memset(dns, 0, sizeof(ZT_VirtualNetworkDNS));
@@ -52,6 +50,6 @@ public:
 	}
 };
 
-}
+}	// namespace ZeroTier
 
 #endif	 // ZT_DNS_HPP

node/Dictionary.hpp

@@ -14,10 +14,10 @@
 #ifndef ZT_DICTIONARY_HPP
 #define ZT_DICTIONARY_HPP
 
+#include "Address.hpp"
+#include "Buffer.hpp"
 #include "Constants.hpp"
 #include "Utils.hpp"
-#include "Buffer.hpp"
-#include "Address.hpp"
 
 #include <stdint.h>
 
@@ -45,28 +45,37 @@ namespace ZeroTier {
  *
  * @tparam C Dictionary max capacity in bytes
  */
-template<unsigned int C>
+template <unsigned int C> class Dictionary {
-class Dictionary
-{
   public:
-	Dictionary() { memset(_d,0,sizeof(_d)); }
+	Dictionary()
-	Dictionary(const char *s) { this->load(s); }
+	{
+		memset(_d, 0, sizeof(_d));
+	}
+	Dictionary(const char* s)
+	{
+		this->load(s);
+	}
 	Dictionary(const char* s, unsigned int len)
 	{
 		for (unsigned int i = 0; i < C; ++i) {
 			if ((s) && (i < len)) {
 				if (! (_d[i] = *s)) {
 					s = (const char*)0;
-				} else {
+				}
+				else {
 					++s;
 				}
-			} else {
+			}
+			else {
 				_d[i] = (char)0;
 			}
 		}
 		_d[C - 1] = (char)0;
 	}
-	Dictionary(const Dictionary &d) { memcpy(_d,d._d,C); }
+	Dictionary(const Dictionary& d)
+	{
+		memcpy(_d, d._d, C);
+	}
 
 	inline Dictionary& operator=(const Dictionary& d)
 	{
@@ -74,7 +83,10 @@ public:
 		return *this;
 	}
 
-	inline operator bool() const { return (_d[0] != 0); }
+	inline operator bool() const
+	{
+		return (_d[0] != 0);
+	}
 
 	/**
 	 * Load a dictionary from a C-string
@@ -88,10 +100,12 @@ public:
 			if (s) {
 				if (! (_d[i] = *s)) {
 					s = (const char*)0;
-				} else {
+				}
+				else {
 					++s;
 				}
-			} else {
+			}
+			else {
 				_d[i] = (char)0;
 			}
 		}
@@ -195,9 +209,11 @@ public:
 							dest[j - 1] = (char)0;
 							return j - 1;
 						}
-					} else if (*p == '\\') {
+					}
+					else if (*p == '\\') {
 						esc = true;
-					} else {
+					}
+					else {
 						dest[j++] = *p;
 						if (j == (int)destlen) {
 							dest[j - 1] = (char)0;
@@ -211,7 +227,8 @@ public:
 				}
 				dest[j] = (char)0;
 				return j;
-			} else {
+			}
+			else {
 				while ((*p) && (*p != 13) && (*p != 10)) {
 					if (++p == eof) {
 						dest[0] = (char)0;
@@ -223,7 +240,8 @@ public:
 						dest[0] = (char)0;
 						return -1;
 					}
-				} else {
+				}
+				else {
 					break;
 				}
 			}
@@ -241,14 +259,14 @@ public:
 	 * @return True if key was found (if false, dest will be empty)
 	 * @tparam BC Buffer capacity (usually inferred)
 	 */
-	template<unsigned int BC>
+	template <unsigned int BC> inline bool get(const char* key, Buffer<BC>& dest) const
-	inline bool get(const char *key,Buffer<BC> &dest) const
 	{
 		const int r = this->get(key, const_cast<char*>(reinterpret_cast<const char*>(dest.data())), BC);
 		if (r >= 0) {
 			dest.setSize((unsigned int)r);
 			return true;
-		} else {
+		}
+		else {
 			dest.clear();
 			return false;
 		}
@@ -426,7 +444,8 @@ public:
 		char tmp[32];
 		if (value >= 0) {
 			return this->add(key, Utils::hex((uint64_t)value, tmp), -1);
-		} else {
+		}
+		else {
 			tmp[0] = '-';
 			return this->add(key, Utils::hex((uint64_t)(value * -1), tmp + 1), -1);
 		}
@@ -446,8 +465,7 @@ public:
 	 *
 	 * @tparam BC Buffer capacity (usually inferred)
 	 */
-	template<unsigned int BC>
+	template <unsigned int BC> inline bool add(const char* key, const Buffer<BC>& value)
-	inline bool add(const char *key,const Buffer<BC> &value)
 	{
 		return this->add(key, (const char*)value.data(), (int)value.size());
 	}
@@ -465,10 +483,19 @@ public:
 	/**
 	 * @return Value of C template parameter
 	 */
-	inline unsigned int capacity() const { return C; }
+	inline unsigned int capacity() const
+	{
+		return C;
+	}
 
-	inline const char *data() const { return _d; }
+	inline const char* data() const
-	inline char *unsafeData() { return _d; }
+	{
+		return _d;
+	}
+	inline char* unsafeData()
+	{
+		return _d;
+	}
 
   private:
 	char _d[C];

node/Hashtable.hpp

@@ -16,30 +16,37 @@
 
 #include "Constants.hpp"
 
+#include <algorithm>
+#include <stdexcept>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-
-#include <stdexcept>
-#include <vector>
 #include <utility>
-#include <algorithm>
+#include <vector>
 
 namespace ZeroTier {
 
 /**
  * A minimal hash table implementation for the ZeroTier core
  */
-template<typename K,typename V>
+template <typename K, typename V> class Hashtable {
-class Hashtable
-{
   private:
-	struct _Bucket
+	struct _Bucket {
+		_Bucket(const K& k, const V& v) : k(k), v(v)
 		{
-		_Bucket(const K &k,const V &v) : k(k),v(v) {}
+		}
-		_Bucket(const K &k) : k(k),v() {}
+		_Bucket(const K& k) : k(k), v()
-		_Bucket(const _Bucket &b) : k(b.k),v(b.v) {}
+		{
-		inline _Bucket &operator=(const _Bucket &b) { k = b.k; v = b.v; return *this; }
+		}
+		_Bucket(const _Bucket& b) : k(b.k), v(b.v)
+		{
+		}
+		inline _Bucket& operator=(const _Bucket& b)
+		{
+			k = b.k;
+			v = b.v;
+			return *this;
+		}
 		K k;
 		V v;
 		_Bucket* next;	 // must be set manually for each _Bucket
@@ -53,16 +60,12 @@ public:
 	 * may rehash and invalidate the iterator. Note the erasing the key will destroy
 	 * the targets of the pointers returned by next().
 	 */
-	class Iterator
+	class Iterator {
-	{
 	  public:
 		/**
 		 * @param ht Hash table to iterate over
 		 */
-		Iterator(Hashtable &ht) :
+		Iterator(Hashtable& ht) : _idx(0), _ht(&ht), _b(ht._t[0])
-			_idx(0),
-			_ht(&ht),
-			_b(ht._t[0])
 		{
 		}
 
@@ -98,10 +101,7 @@ public:
 	/**
 	 * @param bc Initial capacity in buckets (default: 64, must be nonzero)
 	 */
-	Hashtable(unsigned long bc = 64) :
+	Hashtable(unsigned long bc = 64) : _t(reinterpret_cast<_Bucket**>(::malloc(sizeof(_Bucket*) * bc))), _bc(bc), _s(0)
-		_t(reinterpret_cast<_Bucket **>(::malloc(sizeof(_Bucket *) * bc))),
-		_bc(bc),
-		_s(0)
 	{
 		if (! _t) {
 			throw ZT_EXCEPTION_OUT_OF_MEMORY;
@@ -111,10 +111,7 @@ public:
 		}
 	}
 
-	Hashtable(const Hashtable<K,V> &ht) :
+	Hashtable(const Hashtable<K, V>& ht) : _t(reinterpret_cast<_Bucket**>(::malloc(sizeof(_Bucket*) * ht._bc))), _bc(ht._bc), _s(ht._s)
-		_t(reinterpret_cast<_Bucket **>(::malloc(sizeof(_Bucket *) * ht._bc))),
-		_bc(ht._bc),
-		_s(ht._s)
 	{
 		if (! _t) {
 			throw ZT_EXCEPTION_OUT_OF_MEMORY;
@@ -198,8 +195,7 @@ public:
 	 * @param v Vector, list, or other compliant container
 	 * @tparam Type of V (generally inferred)
 	 */
-	template<typename C>
+	template <typename C> inline void appendKeys(C& v) const
-	inline void appendKeys(C &v) const
 	{
 		if (_s) {
 			for (unsigned long i = 0; i < _bc; ++i) {
@@ -246,7 +242,10 @@ public:
 		}
 		return (V*)0;
 	}
-	inline const V *get(const K &k) const { return const_cast<Hashtable *>(this)->get(k); }
+	inline const V* get(const K& k) const
+	{
+		return const_cast<Hashtable*>(this)->get(k);
+	}
 
 	/**
 	 * @param k Key
@@ -295,7 +294,8 @@ public:
 			if (b->k == k) {
 				if (lastb) {
 					lastb->next = b->next;
-				} else {
+				}
+				else {
 					_t[bidx] = b->next;
 				}
 				delete b;
@@ -371,16 +371,21 @@ public:
 	/**
 	 * @return Number of entries
 	 */
-	inline unsigned long size() const { return _s; }
+	inline unsigned long size() const
+	{
+		return _s;
+	}
 
 	/**
 	 * @return True if table is empty
 	 */
-	inline bool empty() const { return (_s == 0); }
+	inline bool empty() const
+	{
+		return (_s == 0);
+	}
 
   private:
-	template<typename O>
+	template <typename O> static inline unsigned long _hc(const O& obj)
-	static inline unsigned long _hc(const O &obj)
 	{
 		return (unsigned long)obj.hashCode();
 	}

node/Identity.cpp

@@ -11,17 +11,18 @@
  */
 /****/
 
-#include <stdio.h>
+#include "Identity.hpp"
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
 
 #include "Constants.hpp"
-#include "Identity.hpp"
 #include "SHA512.hpp"
 #include "Salsa20.hpp"
 #include "Utils.hpp"
 
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
 // These can't be changed without a new identity type. They define the
 // parameters of the hashcash hashing/searching algorithm.
 
@@ -68,10 +69,13 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub
 
 // Hashcash generation halting condition -- halt when first byte is less than
 // threshold value.
-struct _Identity_generate_cond
+struct _Identity_generate_cond {
+	_Identity_generate_cond()
 	{
-	_Identity_generate_cond() {}
+	}
-	_Identity_generate_cond(unsigned char *sb,char *gm) : digest(sb),genmem(gm) {}
+	_Identity_generate_cond(unsigned char* sb, char* gm) : digest(sb), genmem(gm)
+	{
+	}
 	inline bool operator()(const C25519::Pair& kp) const
 	{
 		_computeMemoryHardHash(kp.pub.data, ZT_C25519_PUBLIC_KEY_LEN, digest, genmem);
@@ -115,13 +119,7 @@ bool Identity::locallyValidate() const
 	unsigned char addrb[5];
 	_address.copyTo(addrb, 5);
 
-	return (
+	return ((digest[0] < ZT_IDENTITY_GEN_HASHCASH_FIRST_BYTE_LESS_THAN) && (digest[59] == addrb[0]) && (digest[60] == addrb[1]) && (digest[61] == addrb[2]) && (digest[62] == addrb[3]) && (digest[63] == addrb[4]));
-		(digest[0] < ZT_IDENTITY_GEN_HASHCASH_FIRST_BYTE_LESS_THAN)&&
-		(digest[59] == addrb[0])&&
-		(digest[60] == addrb[1])&&
-		(digest[61] == addrb[2])&&
-		(digest[62] == addrb[3])&&
-		(digest[63] == addrb[4]));
 }
 
 char* Identity::toString(bool includePrivate, char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const

node/Identity.hpp

@@ -14,16 +14,16 @@
 #ifndef ZT_IDENTITY_HPP
 #define ZT_IDENTITY_HPP
 
+#include "Address.hpp"
+#include "Buffer.hpp"
+#include "C25519.hpp"
+#include "Constants.hpp"
+#include "SHA512.hpp"
+#include "Utils.hpp"
+
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "Constants.hpp"
-#include "Utils.hpp"
-#include "Address.hpp"
-#include "C25519.hpp"
-#include "Buffer.hpp"
-#include "SHA512.hpp"
-
 #define ZT_IDENTITY_STRING_BUFFER_LENGTH 384
 
 namespace ZeroTier {
@@ -38,32 +38,24 @@ namespace ZeroTier {
  * search for a different public key that duplicates an existing address. (See
  * code for deriveAddress() for this algorithm.)
  */
-class Identity
+class Identity {
-{
   public:
-	Identity() :
+	Identity() : _privateKey((C25519::Private*)0)
-		_privateKey((C25519::Private *)0)
 	{
 	}
 
-	Identity(const Identity &id) :
+	Identity(const Identity& id) : _address(id._address), _publicKey(id._publicKey), _privateKey((id._privateKey) ? new C25519::Private(*(id._privateKey)) : (C25519::Private*)0)
-		_address(id._address),
-		_publicKey(id._publicKey),
-		_privateKey((id._privateKey) ? new C25519::Private(*(id._privateKey)) : (C25519::Private *)0)
 	{
 	}
 
-	Identity(const char *str) :
+	Identity(const char* str) : _privateKey((C25519::Private*)0)
-		_privateKey((C25519::Private *)0)
 	{
 		if (! fromString(str)) {
 			throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE;
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> Identity(const Buffer<C>& b, unsigned int startAt = 0) : _privateKey((C25519::Private*)0)
-	Identity(const Buffer<C> &b,unsigned int startAt = 0) :
-		_privateKey((C25519::Private *)0)
 	{
 		deserialize(b, startAt);
 	}
@@ -85,7 +77,8 @@ public:
 				_privateKey = new C25519::Private();
 			}
 			*_privateKey = *(id._privateKey);
-		} else {
+		}
+		else {
 			delete _privateKey;
 			_privateKey = (C25519::Private*)0;
 		}
@@ -109,7 +102,10 @@ public:
 	/**
 	 * @return True if this identity contains a private key
 	 */
-	inline bool hasPrivate() const { return (_privateKey != (C25519::Private *)0); }
+	inline bool hasPrivate() const
+	{
+		return (_privateKey != (C25519::Private*)0);
+	}
 
 	/**
 	 * Compute a SHA384 hash of this identity's address and public key(s).
@@ -203,7 +199,10 @@ public:
 	/**
 	 * @return This identity's address
 	 */
-	inline const Address &address() const { return _address; }
+	inline const Address& address() const
+	{
+		return _address;
+	}
 
 	/**
 	 * Serialize this identity (binary)
@@ -212,8 +211,7 @@ public:
 	 * @param includePrivate If true, include private key component (if present) (default: false)
 	 * @throws std::out_of_range Buffer too small
 	 */
-	template<unsigned int C>
+	template <unsigned int C> inline void serialize(Buffer<C>& b, bool includePrivate = false) const
-	inline void serialize(Buffer<C> &b,bool includePrivate = false) const
 	{
 		_address.appendTo(b);
 		b.append((uint8_t)0);	// C25519/Ed25519 identity type
@@ -221,7 +219,8 @@ public:
 		if ((_privateKey) && (includePrivate)) {
 			b.append((unsigned char)ZT_C25519_PRIVATE_KEY_LEN);
 			b.append(_privateKey->data, ZT_C25519_PRIVATE_KEY_LEN);
-		} else {
+		}
+		else {
 			b.append((unsigned char)0);
 		}
 	}
@@ -238,8 +237,7 @@ public:
 	 * @throws std::out_of_range Serialized data invalid
 	 * @throws std::invalid_argument Serialized data invalid
 	 */
-	template<unsigned int C>
+	template <unsigned int C> inline unsigned int deserialize(const Buffer<C>& b, unsigned int startAt = 0)
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		delete _privateKey;
 		_privateKey = (C25519::Private*)0;
@@ -292,7 +290,10 @@ public:
 	/**
 	 * @return C25519 public key
 	 */
-	inline const C25519::Public &publicKey() const { return _publicKey; }
+	inline const C25519::Public& publicKey() const
+	{
+		return _publicKey;
+	}
 
 	/**
 	 * @return C25519 key pair (only returns valid pair if private key is present in this Identity object)
@@ -303,7 +304,8 @@ public:
 		pair.pub = _publicKey;
 		if (_privateKey) {
 			pair.priv = *_privateKey;
-		} else {
+		}
+		else {
 			memset(pair.priv.data, 0, ZT_C25519_PRIVATE_KEY_LEN);
 		}
 		return pair;
@@ -312,14 +314,35 @@ public:
 	/**
 	 * @return True if this identity contains something
 	 */
-	inline operator bool() const { return (_address); }
+	inline operator bool() const
+	{
+		return (_address);
+	}
 
-	inline bool operator==(const Identity &id) const { return ((_address == id._address)&&(memcmp(_publicKey.data,id._publicKey.data,ZT_C25519_PUBLIC_KEY_LEN) == 0)); }
+	inline bool operator==(const Identity& id) const
-	inline bool operator<(const Identity &id) const { return ((_address < id._address)||((_address == id._address)&&(memcmp(_publicKey.data,id._publicKey.data,ZT_C25519_PUBLIC_KEY_LEN) < 0))); }
+	{
-	inline bool operator!=(const Identity &id) const { return !(*this == id); }
+		return ((_address == id._address) && (memcmp(_publicKey.data, id._publicKey.data, ZT_C25519_PUBLIC_KEY_LEN) == 0));
-	inline bool operator>(const Identity &id) const { return (id < *this); }
+	}
-	inline bool operator<=(const Identity &id) const { return !(id < *this); }
+	inline bool operator<(const Identity& id) const
-	inline bool operator>=(const Identity &id) const { return !(*this < id); }
+	{
+		return ((_address < id._address) || ((_address == id._address) && (memcmp(_publicKey.data, id._publicKey.data, ZT_C25519_PUBLIC_KEY_LEN) < 0)));
+	}
+	inline bool operator!=(const Identity& id) const
+	{
+		return ! (*this == id);
+	}
+	inline bool operator>(const Identity& id) const
+	{
+		return (id < *this);
+	}
+	inline bool operator<=(const Identity& id) const
+	{
+		return ! (id < *this);
+	}
+	inline bool operator>=(const Identity& id) const
+	{
+		return ! (*this < id);
+	}
 
   private:
 	Address _address;

node/IncomingPacket.cpp

@@ -11,33 +11,34 @@
  */
 /****/
 
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include "../version.h"
-#include "../include/ZeroTierOne.h"
-
-#include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
 #include "IncomingPacket.hpp"
-#include "Topology.hpp"
+
-#include "Switch.hpp"
+#include "../include/ZeroTierOne.h"
-#include "Peer.hpp"
+#include "../version.h"
-#include "NetworkController.hpp"
-#include "SelfAwareness.hpp"
-#include "Salsa20.hpp"
-#include "SHA512.hpp"
-#include "World.hpp"
-#include "Node.hpp"
-#include "CertificateOfMembership.hpp"
-#include "Capability.hpp"
-#include "Tag.hpp"
-#include "Revocation.hpp"
-#include "Trace.hpp"
-#include "Path.hpp"
 #include "Bond.hpp"
+#include "Capability.hpp"
+#include "CertificateOfMembership.hpp"
+#include "Constants.hpp"
 #include "Metrics.hpp"
+#include "NetworkController.hpp"
+#include "Node.hpp"
+#include "PacketMultiplexer.hpp"
+#include "Path.hpp"
+#include "Peer.hpp"
+#include "Revocation.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "SHA512.hpp"
+#include "Salsa20.hpp"
+#include "SelfAwareness.hpp"
+#include "Switch.hpp"
+#include "Tag.hpp"
+#include "Topology.hpp"
+#include "Trace.hpp"
+#include "World.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 namespace ZeroTier {
 
@@ -55,11 +56,13 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr,int32_t f
 			const uint64_t tpid = trustedPathId();
 			if (RR->topology->shouldInboundPathBeTrusted(_path->address(), tpid)) {
 				_authenticated = true;
-			} else {
+			}
+			else {
 				RR->t->incomingPacketMessageAuthenticationFailure(tPtr, _path, packetId(), sourceAddress, hops(), "path not trusted");
 				return true;
 			}
-		} else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) {
+		}
+		else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE) && (verb() == Packet::VERB_HELLO)) {
 			// Only HELLO is allowed in the clear, but will still have a MAC
 			return _doHELLO(RR, tPtr, false);
 		}
@@ -155,11 +158,13 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr,int32_t f
 				return true;
 			}
 			return false;
-		} else {
+		}
+		else {
 			RR->sw->requestWhois(tPtr, RR->node->now(), sourceAddress);
 			return false;
 		}
-	} catch ( ... ) {
+	}
+	catch (...) {
 		RR->t->incomingPacketInvalid(tPtr, _path, packetId(), sourceAddress, hops(), verb(), "unexpected exception in tryDecode()");
 		return true;
 	}
@@ -181,7 +186,6 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 	 * circumstances. */
 
 	switch (errorCode) {
-
 		case Packet::ERROR_OBJ_NOT_FOUND:
 			// Object not found, currently only meaningful from network controllers.
 			if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
@@ -266,7 +270,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 								authenticationURL[sizeof(authenticationURL) - 1] = 0;	// ensure always zero terminated
 								network->setAuthenticationRequired(tPtr, authenticationURL);
 							}
-						} else if (authVer == 1) {
+						}
+						else if (authVer == 1) {
 							char issuerURL[2048] = { 0 };
 							char centralAuthURL[2048] = { 0 };
 							char ssoNonce[64] = { 0 };
@@ -291,14 +296,16 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 							}
 							if (authInfo.get(ZT_AUTHINFO_DICT_KEY_SSO_PROVIDER, ssoProvider, sizeof(ssoProvider)) > 0) {
 								ssoProvider[sizeof(ssoProvider) - 1] = 0;
-							} else {
+							}
+							else {
 								strncpy(ssoProvider, "default", sizeof(ssoProvider));
 							}
 
 							network->setAuthenticationRequired(tPtr, issuerURL, centralAuthURL, ssoClientID, ssoProvider, ssoNonce, ssoState);
 						}
 					}
-				} else {
+				}
+				else {
 					network->setAuthenticationRequired(tPtr, "");
 				}
 			}
@@ -334,7 +341,6 @@ bool IncomingPacket::_doACK(const RuntimeEnvironment* RR, void* tPtr, const Shar
 bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr, const SharedPtr<Peer>& peer)
 {
 	Metrics::pkt_qos_in++;
-	SharedPtr<Bond> bond = peer->bond();
 	if (! peer->rateGateQoS(RR->node->now(), _path)) {
 		return true;
 	}
@@ -408,15 +414,18 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 						Metrics::pkt_error_out++;
 						Metrics::pkt_error_identity_collision_out++;
 						_path->send(RR, tPtr, outp.data(), outp.size(), RR->node->now());
-					} else {
+					}
+					else {
 						RR->t->incomingPacketMessageAuthenticationFailure(tPtr, _path, pid, fromAddress, hops(), "invalid MAC");
 					}
-				} else {
+				}
+				else {
 					RR->t->incomingPacketMessageAuthenticationFailure(tPtr, _path, pid, fromAddress, hops(), "invalid identity");
 				}
 
 				return true;
-			} else {
+			}
+			else {
 				// Identity is the same as the one we already have -- check packet integrity
 
 				if (! dearmor(peer->key(), peer->aesKeysIfSupported())) {
@@ -427,7 +436,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 				// Continue at // VALID
 			}
 		}	// else if alreadyAuthenticated then continue at // VALID
-	} else {
+	}
+	else {
 		// We don't already have an identity with this address -- validate and learn it
 
 		// Sanity check: this basically can't happen
@@ -513,7 +523,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 
 	if (protoVersion >= 5) {
 		_path->address().serialize(outp);
-	} else {
+	}
+	else {
 		/* LEGACY COMPATIBILITY HACK:
 		 *
 		 * For a while now (since 1.0.3), ZeroTier has recognized changes in
@@ -585,7 +596,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP
 	}
 
 	switch (inReVerb) {
-
 		case Packet::VERB_HELLO: {
 			const uint64_t latency = RR->node->now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP);
 			const unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION];
@@ -615,7 +625,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP
 						ptr += w.deserialize(*this, ptr);
 						RR->topology->addWorld(tPtr, w, false);
 					}
-				} else {
+				}
+				else {
 					ptr += worldsLen;
 				}
 			}
@@ -716,7 +727,8 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar
 		if (id) {
 			id.serialize(outp, false);
 			++count;
-		} else {
+		}
+		else {
 			// Request unknown WHOIS from upstream from us (if we have one)
 			RR->sw->requestWhois(tPtr, RR->node->now(), addr);
 		}
@@ -793,7 +805,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 {
 	Metrics::pkt_frame_in++;
 	int32_t _flowId = ZT_QOS_NO_FLOW;
-	if (peer->flowHashingSupported()) {
+
 	if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 		const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 		const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
@@ -855,7 +867,6 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			}
 		}
 	}
-	}
 
 	const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID);
 	const SharedPtr<Network> network(RR->node->network(nwid));
@@ -869,10 +880,11 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 				const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 				const uint8_t* const frameData = reinterpret_cast<const uint8_t*>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 				if (network->filterIncomingPacket(tPtr, peer, RR->identity.address(), sourceMac, network->mac(), frameData, frameLen, etherType, 0) > 0) {
-					RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen);
+					RR->pm->putFrame(tPtr, nwid, network->userPtr(), sourceMac, network->mac(), etherType, 0, (const void*)frameData, frameLen, _flowId);
 				}
 			}
-		} else {
+		}
+		else {
 			_sendErrorNeedCredentials(RR, tPtr, peer, nwid);
 			return false;
 		}
@@ -922,19 +934,22 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 					if (from != MAC(peer->address(), nwid)) {
 						if (network->config().permitsBridging(peer->address())) {
 							network->learnBridgeRoute(from, peer->address());
-						} else {
+						}
+						else {
 							RR->t->incomingNetworkFrameDropped(tPtr, network, _path, packetId(), size(), peer->address(), Packet::VERB_EXT_FRAME, from, to, "bridging not allowed (remote)");
 							peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_EXT_FRAME, 0, Packet::VERB_NOP, true, nwid, flowId);	 // trustEstablished because COM is okay
 							return true;
 						}
-					} else if (to != network->mac()) {
+					}
+					else if (to != network->mac()) {
 						if (to.isMulticast()) {
 							if (network->config().multicastLimit == 0) {
 								RR->t->incomingNetworkFrameDropped(tPtr, network, _path, packetId(), size(), peer->address(), Packet::VERB_EXT_FRAME, from, to, "multicast disabled");
 								peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_EXT_FRAME, 0, Packet::VERB_NOP, true, nwid, flowId);	 // trustEstablished because COM is okay
 								return true;
 							}
-						} else if (!network->config().permitsBridging(RR->identity.address())) {
+						}
+						else if (! network->config().permitsBridging(RR->identity.address())) {
 							RR->t->incomingNetworkFrameDropped(tPtr, network, _path, packetId(), size(), peer->address(), Packet::VERB_EXT_FRAME, from, to, "bridging not allowed (local)");
 							peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_EXT_FRAME, 0, Packet::VERB_NOP, true, nwid, flowId);	 // trustEstablished because COM is okay
 							return true;
@@ -942,7 +957,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 					}
 					// fall through -- 2 means accept regardless of bridging checks or other restrictions
 				case 2:
-					RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen);
+					RR->pm->putFrame(tPtr, nwid, network->userPtr(), from, to, etherType, 0, (const void*)frameData, frameLen, flowId);
 					break;
 			}
 		}
@@ -960,7 +975,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 		}
 
 		peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_EXT_FRAME, 0, Packet::VERB_NOP, true, nwid, flowId);
-	} else {
+	}
+	else {
 		peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_EXT_FRAME, 0, Packet::VERB_NOP, false, nwid, flowId);
 	}
 
@@ -1172,7 +1188,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void
 		const char* metaDataBytes = (metaDataLength != 0) ? (const char*)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT, metaDataLength) : (const char*)0;
 		const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> metaData(metaDataBytes, metaDataLength);
 		RR->localNetworkController->request(nwid, (hopCount > 0) ? InetAddress() : _path->address(), requestPacketId, peer->identity(), metaData);
-	} else {
+	}
+	else {
 		Packet outp(peer->address(), RR->identity.address(), Packet::VERB_ERROR);
 		outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
 		outp.append(requestPacketId);
@@ -1231,7 +1248,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr
 			if ((com) && (network)) {
 				network->addCredential(tPtr, com);
 			}
-		} catch ( ... ) {} // discard invalid COMs
+		}
+		catch (...) {
+		}	// discard invalid COMs
 	}
 
 	const bool trustEstablished = (network) ? network->gate(tPtr, peer) : false;
@@ -1292,7 +1311,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 		if ((flags & 0x04) != 0) {
 			from.setTo(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC, 6), 6);
 			offset += 6;
-		} else {
+		}
+		else {
 			from.fromAddress(peer->address(), nwid);
 		}
 
@@ -1327,7 +1347,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 			if (from != MAC(peer->address(), nwid)) {
 				if (network->config().permitsBridging(peer->address())) {
 					network->learnBridgeRoute(from, peer->address());
-				} else {
+				}
+				else {
 					RR->t->incomingNetworkFrameDropped(tPtr, network, _path, packetId(), size(), peer->address(), Packet::VERB_MULTICAST_FRAME, from, to.mac(), "bridging not allowed (remote)");
 					peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_MULTICAST_FRAME, 0, Packet::VERB_NOP, true, nwid, ZT_QOS_NO_FLOW);   // trustEstablished because COM is okay
 					return true;
@@ -1390,29 +1411,28 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 		switch (addrType) {
 			case 4: {
 				const InetAddress a(field(ptr, 4), 4, at<uint16_t>(ptr + 4));
-				if (
+				if (((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) &&												  // not being told to forget
-					((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget
 					(! (((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now, a)))) &&	  // not already known
 					(RR->node->shouldUsePathForZeroTierTraffic(tPtr, peer->address(), _path->localSocket(), a)))			  // should use path
 				{
 					if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) {
 						peer->clusterRedirect(tPtr, _path, a, now);
-					} else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+					}
+					else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 						peer->attemptToContactAt(tPtr, InetAddress(), a, now, false);
 					}
 				}
 			} break;
 			case 6: {
-
 				const InetAddress a(field(ptr, 16), 16, at<uint16_t>(ptr + 16));
-				if (
+				if (((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) &&												  // not being told to forget
-					((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget
 					(! (((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now, a)))) &&	  // not already known
 					(RR->node->shouldUsePathForZeroTierTraffic(tPtr, peer->address(), _path->localSocket(), a)))			  // should use path
 				{
 					if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) {
 						peer->clusterRedirect(tPtr, _path, a, now);
-					} else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+					}
+					else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 						peer->attemptToContactAt(tPtr, InetAddress(), a, now, false);
 					}
 				}
@@ -1458,7 +1478,8 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con
 				RR->node->postEvent(tPtr, ZT_EVENT_REMOTE_TRACE, &rt);
 			}
 			rt.data = const_cast<char*>(++ptr);	  // start of next string, if any
-		} else {
+		}
+		else {
 			++ptr;
 		}
 	}

node/IncomingPacket.hpp

@@ -14,13 +14,13 @@
 #ifndef ZT_INCOMINGPACKET_HPP
 #define ZT_INCOMINGPACKET_HPP
 
-#include <stdexcept>
+#include "MulticastGroup.hpp"
-
 #include "Packet.hpp"
 #include "Path.hpp"
-#include "Utils.hpp"
-#include "MulticastGroup.hpp"
 #include "Peer.hpp"
+#include "Utils.hpp"
+
+#include <stdexcept>
 
 /*
  * The big picture:
@@ -46,14 +46,9 @@ class Network;
 /**
  * Subclass of packet that handles the decoding of it
  */
-class IncomingPacket : public Packet
+class IncomingPacket : public Packet {
-{
   public:
-	IncomingPacket() :
+	IncomingPacket() : Packet(), _receiveTime(0), _path(), _authenticated(false)
-		Packet(),
-		_receiveTime(0),
-		_path(),
-		_authenticated(false)
 	{
 	}
 
@@ -66,11 +61,7 @@ public:
 	 * @param now Current time
 	 * @throws std::out_of_range Range error processing packet
 	 */
-	IncomingPacket(const void *data,unsigned int len,const SharedPtr<Path> &path,int64_t now) :
+	IncomingPacket(const void* data, unsigned int len, const SharedPtr<Path>& path, int64_t now) : Packet(data, len), _receiveTime(now), _path(path), _authenticated(false)
-		Packet(data,len),
-		_receiveTime(now),
-		_path(path),
-		_authenticated(false)
 	{
 	}
 
@@ -109,7 +100,10 @@ public:
 	/**
 	 * @return Time of packet receipt / start of decode
 	 */
-	inline uint64_t receiveTime() const { return _receiveTime; }
+	inline uint64_t receiveTime() const
+	{
+		return _receiveTime;
+	}
 
   private:
 	// These are called internally to handle packet contents once it has

node/InetAddress.cpp

@@ -11,16 +11,16 @@
  */
 /****/
 
-#include <stdio.h>
+#include "InetAddress.hpp"
-#include <string.h>
-#include <stdint.h>
-
-#include <string>
 
 #include "Constants.hpp"
-#include "InetAddress.hpp"
 #include "Utils.hpp"
 
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <string>
+
 namespace ZeroTier {
 
 const InetAddress InetAddress::LO4((const void*)("\x7f\x00\x00\x01"), 4, 0);
@@ -29,7 +29,6 @@ const InetAddress InetAddress::LO6((const void *)("\x00\x00\x00\x00\x00\x00\x00\
 InetAddress::IpScope InetAddress::ipScope() const
 {
 	switch (ss_family) {
-
 		case AF_INET: {
 			const uint32_t ip = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in*>(this)->sin_addr.s_addr);
 			switch (ip >> 24) {
@@ -124,7 +123,8 @@ InetAddress::IpScope InetAddress::ipScope() const
 					}
 					if ((k == 15) && (ip[15] == 0x01)) {
 						return IP_SCOPE_LOOPBACK;	// fe80::1/128
-					} else {
+					}
+					else {
 						return IP_SCOPE_LINK_LOCAL;	  // fe80::/10
 					}
 				}
@@ -159,7 +159,6 @@ InetAddress::IpScope InetAddress::ipScope() const
 			}
 			return IP_SCOPE_GLOBAL;
 		} break;
-
 	}
 
 	return IP_SCOPE_NONE;
@@ -174,7 +173,8 @@ void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port)
 		ss_family = AF_INET;
 		reinterpret_cast<struct sockaddr_in*>(this)->sin_addr.s_addr = ipb[0];
 		reinterpret_cast<struct sockaddr_in*>(this)->sin_port = Utils::hton((uint16_t)port);
-	} else if (ipLen == 16) {
+	}
+	else if (ipLen == 16) {
 		ss_family = AF_INET6;
 		memcpy(reinterpret_cast<struct sockaddr_in6*>(this)->sin6_addr.s6_addr, ipBytes, 16);
 		reinterpret_cast<struct sockaddr_in6*>(this)->sin6_port = Utils::hton((uint16_t)port);
@@ -246,13 +246,15 @@ bool InetAddress::fromString(const char *ipSlashPort)
 		in6->sin6_family = AF_INET6;
 		in6->sin6_port = Utils::hton((uint16_t)port);
 		return true;
-	} else if (strchr(buf,'.')) {
+	}
+	else if (strchr(buf, '.')) {
 		struct sockaddr_in* const in = reinterpret_cast<struct sockaddr_in*>(this);
 		inet_pton(AF_INET, buf, &in->sin_addr.s_addr);
 		in->sin_family = AF_INET;
 		in->sin_port = Utils::hton((uint16_t)port);
 		return true;
-	} else {
+	}
+	else {
 		return false;
 	}
 }
@@ -270,7 +272,8 @@ InetAddress InetAddress::netmask() const
 			if (bits) {
 				nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits))));
 				nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits))));
-			} else {
+			}
+			else {
 				nm[0] = 0;
 				nm[1] = 0;
 			}
@@ -341,7 +344,8 @@ bool InetAddress::containsAddress(const InetAddress &addr) const
 				if (bits == 0) {
 					return true;
 				}
-				return ( (Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_addr.s_addr) >> (32 - bits)) == (Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr) >> (32 - bits)) );
+				return (
+					(Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in*>(&addr)->sin_addr.s_addr) >> (32 - bits)) == (Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in*>(this)->sin_addr.s_addr) >> (32 - bits)));
 			}
 			case AF_INET6: {
 				const InetAddress mask(netmask());
@@ -404,15 +408,15 @@ bool InetAddress::operator==(const InetAddress &a) const
 		switch (ss_family) {
 			case AF_INET:
 				return (
-					(reinterpret_cast<const struct sockaddr_in *>(this)->sin_port == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_port)&&
+					(reinterpret_cast<const struct sockaddr_in*>(this)->sin_port == reinterpret_cast<const struct sockaddr_in*>(&a)->sin_port)
-					(reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr));
+					&& (reinterpret_cast<const struct sockaddr_in*>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in*>(&a)->sin_addr.s_addr));
 				break;
 			case AF_INET6:
 				return (
-					(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_port == reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_port)&&
+					(reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_port == reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_port)
-					(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_flowinfo == reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_flowinfo)&&
+					&& (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_flowinfo == reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_flowinfo)
-					(memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0)&&
+					&& (memcmp(reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_addr.s6_addr, reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_addr.s6_addr, 16) == 0)
-					(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_scope_id == reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_scope_id));
+					&& (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_scope_id == reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_scope_id));
 				break;
 			default:
 				return (memcmp(this, &a, sizeof(InetAddress)) == 0);
@@ -425,12 +429,14 @@ bool InetAddress::operator<(const InetAddress &a) const
 {
 	if (ss_family < a.ss_family) {
 		return true;
-	} else if (ss_family == a.ss_family) {
+	}
+	else if (ss_family == a.ss_family) {
 		switch (ss_family) {
 			case AF_INET:
 				if (reinterpret_cast<const struct sockaddr_in*>(this)->sin_port < reinterpret_cast<const struct sockaddr_in*>(&a)->sin_port) {
 					return true;
-				} else if (reinterpret_cast<const struct sockaddr_in *>(this)->sin_port == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_port) {
+				}
+				else if (reinterpret_cast<const struct sockaddr_in*>(this)->sin_port == reinterpret_cast<const struct sockaddr_in*>(&a)->sin_port) {
 					if (reinterpret_cast<const struct sockaddr_in*>(this)->sin_addr.s_addr < reinterpret_cast<const struct sockaddr_in*>(&a)->sin_addr.s_addr) {
 						return true;
 					}
@@ -439,13 +445,16 @@ bool InetAddress::operator<(const InetAddress &a) const
 			case AF_INET6:
 				if (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_port < reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_port) {
 					return true;
-				} else if (reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_port == reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_port) {
+				}
+				else if (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_port == reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_port) {
 					if (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_flowinfo < reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_flowinfo) {
 						return true;
-					} else if (reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_flowinfo == reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_flowinfo) {
+					}
+					else if (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_flowinfo == reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_flowinfo) {
 						if (memcmp(reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_addr.s6_addr, reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_addr.s6_addr, 16) < 0) {
 							return true;
-						} else if (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0) {
+						}
+						else if (memcmp(reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_addr.s6_addr, reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_addr.s6_addr, 16) == 0) {
 							if (reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_scope_id < reinterpret_cast<const struct sockaddr_in6*>(&a)->sin6_scope_id) {
 								return true;
 							}

node/InetAddress.hpp

@@ -14,15 +14,15 @@
 #ifndef ZT_INETADDRESS_HPP
 #define ZT_INETADDRESS_HPP
 
+#include "../include/ZeroTierOne.h"
+#include "Buffer.hpp"
+#include "Constants.hpp"
+#include "MAC.hpp"
+#include "Utils.hpp"
+
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdint.h>
-
-#include "Constants.hpp"
-#include "../include/ZeroTierOne.h"
-#include "Utils.hpp"
-#include "MAC.hpp"
-#include "Buffer.hpp"
 
 namespace ZeroTier {
 
@@ -39,8 +39,7 @@ namespace ZeroTier {
  * sockaddr_storage and used interchangeably. DO NOT change this by e.g.
  * adding non-static fields, since much code depends on this identity.
  */
-struct InetAddress : public sockaddr_storage
+struct InetAddress : public sockaddr_storage {
-{
 	/**
 	 * Loopback IPv4 address (no port)
 	 */
@@ -58,8 +57,7 @@ struct InetAddress : public sockaddr_storage
 	 * MUST remain that way or Path must be changed to reflect. Also be sure
 	 * to change ZT_INETADDRESS_MAX_SCOPE if the max changes.
 	 */
-	enum IpScope
+	enum IpScope {
-	{
 		IP_SCOPE_NONE = 0,			  // NULL or not an IP address
 		IP_SCOPE_MULTICAST = 1,		  // 224.0.0.0 and other V4/V6 multicast IPs
 		IP_SCOPE_LOOPBACK = 2,		  // 127.0.0.1, ::1, etc.
@@ -72,25 +70,69 @@ struct InetAddress : public sockaddr_storage
 
 	// Can be used with the unordered maps and sets in c++11. We don't use C++11 in the core
 	// but this is safe to put here.
-	struct Hasher
+	struct Hasher {
+		inline std::size_t operator()(const InetAddress& a) const
 		{
-		inline std::size_t operator()(const InetAddress &a) const { return (std::size_t)a.hashCode(); }
+			return (std::size_t)a.hashCode();
+		}
 	};
 
-	InetAddress() { memset(this,0,sizeof(InetAddress)); }
+	InetAddress()
-	InetAddress(const InetAddress &a) { memcpy(this,&a,sizeof(InetAddress)); }
+	{
-	InetAddress(const InetAddress *a) { memcpy(this,a,sizeof(InetAddress)); }
+		memset(this, 0, sizeof(InetAddress));
-	InetAddress(const struct sockaddr_storage &ss) { *this = ss; }
+	}
-	InetAddress(const struct sockaddr_storage *ss) { *this = ss; }
+	InetAddress(const InetAddress& a)
-	InetAddress(const struct sockaddr &sa) { *this = sa; }
+	{
-	InetAddress(const struct sockaddr *sa) { *this = sa; }
+		memcpy(this, &a, sizeof(InetAddress));
-	InetAddress(const struct sockaddr_in &sa) { *this = sa; }
+	}
-	InetAddress(const struct sockaddr_in *sa) { *this = sa; }
+	InetAddress(const InetAddress* a)
-	InetAddress(const struct sockaddr_in6 &sa) { *this = sa; }
+	{
-	InetAddress(const struct sockaddr_in6 *sa) { *this = sa; }
+		memcpy(this, a, sizeof(InetAddress));
-	InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) { this->set(ipBytes,ipLen,port); }
+	}
-	InetAddress(const uint32_t ipv4,unsigned int port) { this->set(&ipv4,4,port); }
+	InetAddress(const struct sockaddr_storage& ss)
-	InetAddress(const char *ipSlashPort) { this->fromString(ipSlashPort); }
+	{
+		*this = ss;
+	}
+	InetAddress(const struct sockaddr_storage* ss)
+	{
+		*this = ss;
+	}
+	InetAddress(const struct sockaddr& sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const struct sockaddr* sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const struct sockaddr_in& sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const struct sockaddr_in* sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const struct sockaddr_in6& sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const struct sockaddr_in6* sa)
+	{
+		*this = sa;
+	}
+	InetAddress(const void* ipBytes, unsigned int ipLen, unsigned int port)
+	{
+		this->set(ipBytes, ipLen, port);
+	}
+	InetAddress(const uint32_t ipv4, unsigned int port)
+	{
+		this->set(&ipv4, 4, port);
+	}
+	InetAddress(const char* ipSlashPort)
+	{
+		this->fromString(ipSlashPort);
+	}
 
 	inline InetAddress& operator=(const InetAddress& a)
 	{
@@ -283,7 +325,10 @@ struct InetAddress : public sockaddr_storage
 	 *
 	 * @return Netmask bits
 	 */
-	inline unsigned int netmaskBits() const { return port(); }
+	inline unsigned int netmaskBits() const
+	{
+		return port();
+	}
 
 	/**
 	 * @return True if netmask bits is valid for the address type
@@ -308,7 +353,10 @@ struct InetAddress : public sockaddr_storage
 	 *
 	 * @return Gateway metric
 	 */
-	inline unsigned int metric() const { return port(); }
+	inline unsigned int metric() const
+	{
+		return port();
+	}
 
 	/**
 	 * Construct a full netmask as an InetAddress
@@ -353,12 +401,18 @@ struct InetAddress : public sockaddr_storage
 	/**
 	 * @return True if this is an IPv4 address
 	 */
-	inline bool isV4() const { return (ss_family == AF_INET); }
+	inline bool isV4() const
+	{
+		return (ss_family == AF_INET);
+	}
 
 	/**
 	 * @return True if this is an IPv6 address
 	 */
-	inline bool isV6() const { return (ss_family == AF_INET6); }
+	inline bool isV6() const
+	{
+		return (ss_family == AF_INET6);
+	}
 
 	/**
 	 * @return pointer to raw address bytes or NULL if not available
@@ -440,14 +494,16 @@ struct InetAddress : public sockaddr_storage
 	{
 		if (ss_family == AF_INET) {
 			return ((unsigned long)reinterpret_cast<const struct sockaddr_in*>(this)->sin_addr.s_addr + (unsigned long)reinterpret_cast<const struct sockaddr_in*>(this)->sin_port);
-		} else if (ss_family == AF_INET6) {
+		}
+		else if (ss_family == AF_INET6) {
 			unsigned long tmp = reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_port;
 			const uint8_t* a = reinterpret_cast<const uint8_t*>(reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_addr.s6_addr);
 			for (long i = 0; i < 16; ++i) {
 				reinterpret_cast<uint8_t*>(&tmp)[i % sizeof(tmp)] ^= a[i];
 			}
 			return tmp;
-		} else {
+		}
+		else {
 			unsigned long tmp = reinterpret_cast<const struct sockaddr_in6*>(this)->sin6_port;
 			const uint8_t* a = reinterpret_cast<const uint8_t*>(this);
 			for (long i = 0; i < (long)sizeof(InetAddress); ++i) {
@@ -460,7 +516,10 @@ struct InetAddress : public sockaddr_storage
 	/**
 	 * Set to null/zero
 	 */
-	inline void zero() { memset(this,0,sizeof(InetAddress)); }
+	inline void zero()
+	{
+		memset(this, 0, sizeof(InetAddress));
+	}
 
 	/**
 	 * Check whether this is a network/route rather than an IP assignment
@@ -500,7 +559,8 @@ struct InetAddress : public sockaddr_storage
 					for (unsigned int i = 0; i < 16; ++i) {
 						if (ip0[i] == ip1[i]) {
 							c += 8;
-						} else {
+						}
+						else {
 							uint8_t ip0b = ip0[i];
 							uint8_t ip1b = ip1[i];
 							uint8_t bit = 0x80;
@@ -552,10 +612,12 @@ struct InetAddress : public sockaddr_storage
 	/**
 	 * @return True if address family is non-zero
 	 */
-	inline operator bool() const { return (ss_family != 0); }
+	inline operator bool() const
+	{
+		return (ss_family != 0);
+	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline void serialize(Buffer<C>& b) const
-	inline void serialize(Buffer<C> &b) const
 	{
 		// This is used in the protocol and must be the same as describe in places
 		// like VERB_HELLO in Packet.hpp.
@@ -576,8 +638,7 @@ struct InetAddress : public sockaddr_storage
 		}
 	}
 
-	template<unsigned int C>
+	template <unsigned int C> inline unsigned int deserialize(const Buffer<C>& b, unsigned int startAt = 0)
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		memset(this, 0, sizeof(InetAddress));
 		unsigned int p = startAt;
@@ -616,10 +677,22 @@ struct InetAddress : public sockaddr_storage
 
 	bool operator==(const InetAddress& a) const;
 	bool operator<(const InetAddress& a) const;
-	inline bool operator!=(const InetAddress &a) const { return !(*this == a); }
+	inline bool operator!=(const InetAddress& a) const
-	inline bool operator>(const InetAddress &a) const { return (a < *this); }
+	{
-	inline bool operator<=(const InetAddress &a) const { return !(a < *this); }
+		return ! (*this == a);
-	inline bool operator>=(const InetAddress &a) const { return !(*this < a); }
+	}
+	inline bool operator>(const InetAddress& a) const
+	{
+		return (a < *this);
+	}
+	inline bool operator<=(const InetAddress& a) const
+	{
+		return ! (a < *this);
+	}
+	inline bool operator>=(const InetAddress& a) const
+	{
+		return ! (*this < a);
+	}
 
 	/**
 	 * @param mac MAC address seed

node/MAC.hpp

@@ -14,51 +14,68 @@
 #ifndef ZT_MAC_HPP
 #define ZT_MAC_HPP
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-#include "Constants.hpp"
-#include "Utils.hpp"
 #include "Address.hpp"
 #include "Buffer.hpp"
+#include "Constants.hpp"
+#include "Utils.hpp"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 namespace ZeroTier {
 
 /**
  * 48-byte Ethernet MAC address
  */
-class MAC
+class MAC {
-{
   public:
-	MAC() : _m(0ULL) {}
+	MAC() : _m(0ULL)
-	MAC(const MAC &m) : _m(m._m) {}
+	{
+	}
+	MAC(const MAC& m) : _m(m._m)
+	{
+	}
 
-	MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) :
+	MAC(const unsigned char a, const unsigned char b, const unsigned char c, const unsigned char d, const unsigned char e, const unsigned char f)
-		_m( ((((uint64_t)a) & 0xffULL) << 40) |
+		: _m(((((uint64_t)a) & 0xffULL) << 40) | ((((uint64_t)b) & 0xffULL) << 32) | ((((uint64_t)c) & 0xffULL) << 24) | ((((uint64_t)d) & 0xffULL) << 16) | ((((uint64_t)e) & 0xffULL) << 8) | (((uint64_t)f) & 0xffULL))
-		    ((((uint64_t)b) & 0xffULL) << 32) |
+	{
-		    ((((uint64_t)c) & 0xffULL) << 24) |
+	}
-		    ((((uint64_t)d) & 0xffULL) << 16) |
+	MAC(const void* bits, unsigned int len)
-		    ((((uint64_t)e) & 0xffULL) << 8) |
+	{
-		    (((uint64_t)f) & 0xffULL) ) {}
+		setTo(bits, len);
-	MAC(const void *bits,unsigned int len) { setTo(bits,len); }
+	}
-	MAC(const Address &ztaddr,uint64_t nwid) { fromAddress(ztaddr,nwid); }
+	MAC(const Address& ztaddr, uint64_t nwid)
-	MAC(const uint64_t m) : _m(m & 0xffffffffffffULL) {}
+	{
+		fromAddress(ztaddr, nwid);
+	}
+	MAC(const uint64_t m) : _m(m & 0xffffffffffffULL)
+	{
+	}
 
 	/**
 	 * @return MAC in 64-bit integer
 	 */
-	inline uint64_t toInt() const { return _m; }
+	inline uint64_t toInt() const
+	{
+		return _m;
+	}
 
 	/**
 	 * Set MAC to zero
 	 */
-	inline void zero() { _m = 0ULL; }
+	inline void zero()
+	{
+		_m = 0ULL;
+	}
 
 	/**
 	 * @return True if MAC is non-zero
 	 */
-	inline operator bool() const { return (_m != 0ULL); }
+	inline operator bool() const
+	{
+		return (_m != 0ULL);
+	}
 
 	/**
 	 * @param bits Raw MAC in big-endian byte order
@@ -107,8 +124,7 @@ public:
 	 *
 	 * @param b Buffer to append to
 	 */
-	template<unsigned int C>
+	template <unsigned int C> inline void appendTo(Buffer<C>& b) const
-	inline void appendTo(Buffer<C> &b) const
 	{
 		unsigned char* p = (unsigned char*)b.appendField(6);
 		*(p++) = (unsigned char)((_m >> 40) & 0xff);
@@ -122,17 +138,26 @@ public:
 	/**
 	 * @return True if this is broadcast (all 0xff)
 	 */
-	inline bool isBroadcast() const { return (_m == 0xffffffffffffULL); }
+	inline bool isBroadcast() const
+	{
+		return (_m == 0xffffffffffffULL);
+	}
 
 	/**
 	 * @return True if this is a multicast MAC
 	 */
-	inline bool isMulticast() const { return ((_m & 0x010000000000ULL) != 0ULL); }
+	inline bool isMulticast() const
+	{
+		return ((_m & 0x010000000000ULL) != 0ULL);
+	}
 
 	/**
 	 * @param True if this is a locally-administered MAC
 	 */
-	inline bool isLocallyAdministered() const { return ((_m & 0x020000000000ULL) != 0ULL); }
+	inline bool isLocallyAdministered() const
+	{
+		return ((_m & 0x020000000000ULL) != 0ULL);
+	}
 
 	/**
 	 * Set this MAC to a MAC derived from an address and a network ID
@@ -184,14 +209,23 @@ public:
 	 * @param i Value from 0 to 5 (inclusive)
 	 * @return Byte at said position (address interpreted in big-endian order)
 	 */
-	inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_m >> (40 - (i * 8))) & 0xff); }
+	inline unsigned char operator[](unsigned int i) const
+	{
+		return (unsigned char)((_m >> (40 - (i * 8))) & 0xff);
+	}
 
 	/**
 	 * @return 6, which is the number of bytes in a MAC, for container compliance
 	 */
-	inline unsigned int size() const { return 6; }
+	inline unsigned int size() const
+	{
+		return 6;
+	}
 
-	inline unsigned long hashCode() const { return (unsigned long)_m; }
+	inline unsigned long hashCode() const
+	{
+		return (unsigned long)_m;
+	}
 
 	inline char* toString(char buf[18]) const
 	{
@@ -227,12 +261,30 @@ public:
 		return *this;
 	}
 
-	inline bool operator==(const MAC &m) const { return (_m == m._m); }
+	inline bool operator==(const MAC& m) const
-	inline bool operator!=(const MAC &m) const { return (_m != m._m); }
+	{
-	inline bool operator<(const MAC &m) const { return (_m < m._m); }
+		return (_m == m._m);
-	inline bool operator<=(const MAC &m) const { return (_m <= m._m); }
+	}
-	inline bool operator>(const MAC &m) const { return (_m > m._m); }
+	inline bool operator!=(const MAC& m) const
-	inline bool operator>=(const MAC &m) const { return (_m >= m._m); }
+	{
+		return (_m != m._m);
+	}
+	inline bool operator<(const MAC& m) const
+	{
+		return (_m < m._m);
+	}
+	inline bool operator<=(const MAC& m) const
+	{
+		return (_m <= m._m);
+	}
+	inline bool operator>(const MAC& m) const
+	{
+		return (_m > m._m);
+	}
+	inline bool operator>=(const MAC& m) const
+	{
+		return (_m >= m._m);
+	}
 
   private:
 	uint64_t _m;

node/Membership.cpp

@@ -11,27 +11,21 @@
  */
 /****/
 
-#include <algorithm>
-
 #include "Membership.hpp"
-#include "RuntimeEnvironment.hpp"
+
-#include "Peer.hpp"
-#include "Topology.hpp"
-#include "Switch.hpp"
-#include "Packet.hpp"
 #include "Node.hpp"
+#include "Packet.hpp"
+#include "Peer.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
 #include "Trace.hpp"
 
+#include <algorithm>
+
 namespace ZeroTier {
 
-Membership::Membership() :
+Membership::Membership() : _lastUpdatedMulticast(0), _comRevocationThreshold(0), _lastPushedCredentials(0), _revocations(4), _remoteTags(4), _remoteCaps(4), _remoteCoos(4)
-	_lastUpdatedMulticast(0),
-	_comRevocationThreshold(0),
-	_lastPushedCredentials(0),
-	_revocations(4),
-	_remoteTags(4),
-	_remoteCaps(4),
-	_remoteCoos(4)
 {
 }
 
@@ -172,9 +166,18 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable<uint32_t,C> &remot
 	}
 }
 
-Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { return _addCredImpl<Tag>(_remoteTags,_revocations,RR,tPtr,nconf,tag); }
+Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment* RR, void* tPtr, const NetworkConfig& nconf, const Tag& tag)
-Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { return _addCredImpl<Capability>(_remoteCaps,_revocations,RR,tPtr,nconf,cap); }
+{
-Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { return _addCredImpl<CertificateOfOwnership>(_remoteCoos,_revocations,RR,tPtr,nconf,coo); }
+	return _addCredImpl<Tag>(_remoteTags, _revocations, RR, tPtr, nconf, tag);
+}
+Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment* RR, void* tPtr, const NetworkConfig& nconf, const Capability& cap)
+{
+	return _addCredImpl<Capability>(_remoteCaps, _revocations, RR, tPtr, nconf, cap);
+}
+Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment* RR, void* tPtr, const NetworkConfig& nconf, const CertificateOfOwnership& coo)
+{
+	return _addCredImpl<CertificateOfOwnership>(_remoteCoos, _revocations, RR, tPtr, nconf, coo);
+}
 
 Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment* RR, void* tPtr, const NetworkConfig& nconf, const Revocation& rev)
 {

node/Membership.hpp

@@ -14,17 +14,17 @@
 #ifndef ZT_MEMBERSHIP_HPP
 #define ZT_MEMBERSHIP_HPP
 
-#include <stdint.h>
-
-#include "Constants.hpp"
 #include "../include/ZeroTierOne.h"
+#include "Capability.hpp"
+#include "CertificateOfMembership.hpp"
+#include "Constants.hpp"
 #include "Credential.hpp"
 #include "Hashtable.hpp"
-#include "CertificateOfMembership.hpp"
-#include "Capability.hpp"
-#include "Tag.hpp"
-#include "Revocation.hpp"
 #include "NetworkConfig.hpp"
+#include "Revocation.hpp"
+#include "Tag.hpp"
+
+#include <stdint.h>
 
 #define ZT_MEMBERSHIP_CRED_ID_UNUSED 0xffffffffffffffffULL
 
@@ -40,16 +40,9 @@ class Network;
  *
  * This class is not thread safe. It must be locked externally.
  */
-class Membership
+class Membership {
-{
   public:
-	enum AddCredentialResult
+	enum AddCredentialResult { ADD_REJECTED, ADD_ACCEPTED_NEW, ADD_ACCEPTED_REDUNDANT, ADD_DEFERRED_FOR_WHOIS };
-	{
-		ADD_REJECTED,
-		ADD_ACCEPTED_NEW,
-		ADD_ACCEPTED_REDUNDANT,
-		ADD_DEFERRED_FOR_WHOIS
-	};
 
 	Membership();
 
@@ -64,9 +57,18 @@ public:
 	 */
 	void pushCredentials(const RuntimeEnvironment* RR, void* tPtr, const int64_t now, const Address& peerAddress, const NetworkConfig& nconf);
 
-	inline int64_t lastPushedCredentials() { return _lastPushedCredentials; }
+	inline int64_t lastPushedCredentials()
-	inline int64_t comTimestamp() { return _com.timestamp(); }
+	{
-	inline int64_t comRevocationThreshold() { return _comRevocationThreshold; }
+		return _lastPushedCredentials;
+	}
+	inline int64_t comTimestamp()
+	{
+		return _com.timestamp();
+	}
+	inline int64_t comRevocationThreshold()
+	{
+		return _comRevocationThreshold;
+	}
 
 	/**
 	 * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true
@@ -108,8 +110,7 @@ public:
 	 * @param r Resource to check
 	 * @return True if this peer has a certificate of ownership for the given resource
 	 */
-	template<typename T>
+	template <typename T> inline bool hasCertificateOfOwnershipFor(const NetworkConfig& nconf, const T& r) const
-	inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const
 	{
 		uint32_t* k = (uint32_t*)0;
 		CertificateOfOwnership* v = (CertificateOfOwnership*)0;
@@ -171,10 +172,16 @@ public:
 	/**
 	 * Generates a key for the internal use in indexing credentials by type and credential ID
 	 */
-	static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); }
+	static uint64_t credentialKey(const Credential::Type& t, const uint32_t i)
+	{
+		return (((uint64_t)t << 32) | (uint64_t)i);
+	}
 
   private:
-	inline bool _isV6NDPEmulated(const NetworkConfig &nconf,const MAC &m) const { return false; }
+	inline bool _isV6NDPEmulated(const NetworkConfig& nconf, const MAC& m) const
+	{
+		return false;
+	}
 	inline bool _isV6NDPEmulated(const NetworkConfig& nconf, const InetAddress& ip) const
 	{
 		if ((ip.isV6()) && (nconf.ndpEmulation())) {
@@ -215,8 +222,7 @@ private:
 		return false;
 	}
 
-	template<typename C>
+	template <typename C> inline bool _isCredentialTimestampValid(const NetworkConfig& nconf, const C& remoteCredential) const
-	inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &remoteCredential) const
 	{
 		const int64_t ts = remoteCredential.timestamp();
 		if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) {
@@ -226,8 +232,7 @@ private:
 		return false;
 	}
 
-	template<typename C>
+	template <typename C> inline void _cleanCredImpl(const NetworkConfig& nconf, Hashtable<uint32_t, C>& remoteCreds)
-	inline void _cleanCredImpl(const NetworkConfig &nconf,Hashtable<uint32_t,C> &remoteCreds)
 	{
 		uint32_t* k = (uint32_t*)0;
 		C* v = (C*)0;
@@ -260,15 +265,9 @@ private:
 	Hashtable<uint32_t, CertificateOfOwnership> _remoteCoos;
 
   public:
-	class CapabilityIterator
+	class CapabilityIterator {
-	{
 	  public:
-		CapabilityIterator(Membership &m,const NetworkConfig &nconf) :
+		CapabilityIterator(Membership& m, const NetworkConfig& nconf) : _hti(m._remoteCaps), _k((uint32_t*)0), _c((Capability*)0), _m(m), _nconf(nconf)
-			_hti(m._remoteCaps),
-			_k((uint32_t *)0),
-			_c((Capability *)0),
-			_m(m),
-			_nconf(nconf)
 		{
 		}
 

node/Metrics.cpp

@@ -10,263 +10,157 @@
  * of this software will be governed by version 2.0 of the Apache License.
  */
 
+// clang-format off
 #include <prometheus/simpleapi.h>
 #include <prometheus/histogram.h>
+// clang-format on
 
 namespace prometheus {
 namespace simpleapi {
 std::shared_ptr<Registry> registry_ptr = std::make_shared<Registry>();
 Registry& registry = *registry_ptr;
 SaveToFile saver;
-    }
+}	// namespace simpleapi
-}
+}	// namespace prometheus
 
 namespace ZeroTier {
 namespace Metrics {
 // Packet Type Counts
-        prometheus::simpleapi::counter_family_t packets
+prometheus::simpleapi::counter_family_t packets { "zt_packet", "ZeroTier packet type counts" };
-        { "zt_packet", "ZeroTier packet type counts"};
 
 // Incoming packets
-        prometheus::simpleapi::counter_metric_t pkt_nop_in
+prometheus::simpleapi::counter_metric_t pkt_nop_in { packets.Add({ { "packet_type", "nop" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "nop"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_in { packets.Add({ { "packet_type", "error" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_in
+prometheus::simpleapi::counter_metric_t pkt_ack_in { packets.Add({ { "packet_type", "ack" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "error"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_qos_in { packets.Add({ { "packet_type", "qos" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ack_in
+prometheus::simpleapi::counter_metric_t pkt_hello_in { packets.Add({ { "packet_type", "hello" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "ack"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_ok_in { packets.Add({ { "packet_type", "ok" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_qos_in
+prometheus::simpleapi::counter_metric_t pkt_whois_in { packets.Add({ { "packet_type", "whois" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "qos"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_rendezvous_in { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_hello_in
+prometheus::simpleapi::counter_metric_t pkt_frame_in { packets.Add({ { "packet_type", "frame" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "hello"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_ext_frame_in { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ok_in
+prometheus::simpleapi::counter_metric_t pkt_echo_in { packets.Add({ { "packet_type", "echo" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "ok"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_multicast_like_in { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_whois_in
+prometheus::simpleapi::counter_metric_t pkt_network_credentials_in { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "whois"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_network_config_request_in { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_rendezvous_in
+prometheus::simpleapi::counter_metric_t pkt_network_config_in { packets.Add({ { "packet_type", "network_config" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "rendezvous"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_frame_in
+prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "frame"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ext_frame_in
+prometheus::simpleapi::counter_metric_t pkt_user_message_in { packets.Add({ { "packet_type", "user_message" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "ext_frame"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_remote_trace_in { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_echo_in
+prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "rx" } }) };
-        { packets.Add({{"packet_type", "echo"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_like_in
-        { packets.Add({{"packet_type", "multicast_like"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_credentials_in
-        { packets.Add({{"packet_type", "network_credentials"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_config_request_in
-        { packets.Add({{"packet_type", "network_config_request"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_config_in
-        { packets.Add({{"packet_type", "network_config"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in
-        { packets.Add({{"packet_type", "multicast_gather"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in
-        { packets.Add({{"packet_type", "multicast_frame"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in
-        { packets.Add({{"packet_type", "push_direct_paths"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_user_message_in
-        { packets.Add({{"packet_type", "user_message"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_remote_trace_in
-        { packets.Add({{"packet_type", "remote_trace"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in
-        { packets.Add({{"packet_type", "path_negotiation_request"}, {"direction", "rx"}}) };
 
 // Outgoing packets
-        prometheus::simpleapi::counter_metric_t pkt_nop_out
+prometheus::simpleapi::counter_metric_t pkt_nop_out { packets.Add({ { "packet_type", "nop" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "nop"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_out { packets.Add({ { "packet_type", "error" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_out
+prometheus::simpleapi::counter_metric_t pkt_ack_out { packets.Add({ { "packet_type", "ack" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "error"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_qos_out { packets.Add({ { "packet_type", "qos" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ack_out
+prometheus::simpleapi::counter_metric_t pkt_hello_out { packets.Add({ { "packet_type", "hello" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "ack"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_ok_out { packets.Add({ { "packet_type", "ok" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_qos_out
+prometheus::simpleapi::counter_metric_t pkt_whois_out { packets.Add({ { "packet_type", "whois" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "qos"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_rendezvous_out { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_hello_out
+prometheus::simpleapi::counter_metric_t pkt_frame_out { packets.Add({ { "packet_type", "frame" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "hello"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_ext_frame_out { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ok_out
+prometheus::simpleapi::counter_metric_t pkt_echo_out { packets.Add({ { "packet_type", "echo" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "ok"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_multicast_like_out { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_whois_out
+prometheus::simpleapi::counter_metric_t pkt_network_credentials_out { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "whois"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_network_config_request_out { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_rendezvous_out
+prometheus::simpleapi::counter_metric_t pkt_network_config_out { packets.Add({ { "packet_type", "network_config" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "rendezvous"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_frame_out
+prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "frame"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_ext_frame_out
+prometheus::simpleapi::counter_metric_t pkt_user_message_out { packets.Add({ { "packet_type", "user_message" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "ext_frame"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_remote_trace_out { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_echo_out
+prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "tx" } }) };
-        { packets.Add({{"packet_type", "echo"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_like_out
-        { packets.Add({{"packet_type", "multicast_like"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_credentials_out
-        { packets.Add({{"packet_type", "network_credentials"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_config_request_out
-        { packets.Add({{"packet_type", "network_config_request"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_network_config_out
-        { packets.Add({{"packet_type", "network_config"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out
-        { packets.Add({{"packet_type", "multicast_gather"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out
-        { packets.Add({{"packet_type", "multicast_frame"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out
-        { packets.Add({{"packet_type", "push_direct_paths"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_user_message_out
-        { packets.Add({{"packet_type", "user_message"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_remote_trace_out
-        { packets.Add({{"packet_type", "remote_trace"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out
-        { packets.Add({{"packet_type", "path_negotiation_request"}, {"direction", "tx"}}) };
-
 
 // Packet Error Counts
-        prometheus::simpleapi::counter_family_t packet_errors
+prometheus::simpleapi::counter_family_t packet_errors { "zt_packet_error", "ZeroTier packet errors" };
-        { "zt_packet_error", "ZeroTier packet errors"};
 
 // Incoming Error Counts
-        prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in
+prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "rx" } }) };
-        { packet_errors.Add({{"error_type", "obj_not_found"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in
+prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "rx" } }) };
-        { packet_errors.Add({{"error_type", "unsupported_operation"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in
+prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "rx" } }) };
-        { packet_errors.Add({{"error_type", "identity_collision"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in
+prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "rx" } }) };
-        { packet_errors.Add({{"error_type", "need_membership_certificate"}, {"direction", "rx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in
-        { packet_errors.Add({{"error_type", "network_access_denied"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in
-        { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in
-        { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "rx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in
-        { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "rx"}}) };
 
 // Outgoing Error Counts
-        prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out
+prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "tx" } }) };
-        { packet_errors.Add({{"error_type", "obj_not_found"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out
+prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "tx" } }) };
-        { packet_errors.Add({{"error_type", "unsupported_operation"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out
+prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "tx" } }) };
-        { packet_errors.Add({{"error_type", "identity_collision"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out
+prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "tx" } }) };
-        { packet_errors.Add({{"error_type", "need_membership_certificate"}, {"direction", "tx"}}) };
+prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out
-        { packet_errors.Add({{"error_type", "network_access_denied"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out
-        { packet_errors.Add({{"error_type", "unwanted_multicast"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out
-        { packet_errors.Add({{"error_type", "authentication_required"}, {"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out
-        { packet_errors.Add({{"error_type", "internal_server_error"}, {"direction", "tx"}}) };
 
 // Data Sent/Received Metrics
-        prometheus::simpleapi::counter_family_t data
+prometheus::simpleapi::counter_family_t data { "zt_data", "number of bytes ZeroTier has transmitted or received" };
-        { "zt_data", "number of bytes ZeroTier has transmitted or received" };
+prometheus::simpleapi::counter_metric_t udp_recv { data.Add({ { "protocol", "udp" }, { "direction", "rx" } }) };
-        prometheus::simpleapi::counter_metric_t udp_recv
+prometheus::simpleapi::counter_metric_t udp_send { data.Add({ { "protocol", "udp" }, { "direction", "tx" } }) };
-        { data.Add({{"protocol","udp"},{"direction","rx"}}) };
+prometheus::simpleapi::counter_metric_t tcp_send { data.Add({ { "protocol", "tcp" }, { "direction", "tx" } }) };
-        prometheus::simpleapi::counter_metric_t udp_send
+prometheus::simpleapi::counter_metric_t tcp_recv { data.Add({ { "protocol", "tcp" }, { "direction", "rx" } }) };
-        { data.Add({{"protocol","udp"},{"direction","tx"}}) };
-        prometheus::simpleapi::counter_metric_t tcp_send
-        { data.Add({{"protocol","tcp"},{"direction", "tx"}}) };
-        prometheus::simpleapi::counter_metric_t tcp_recv
-        { data.Add({{"protocol","tcp"},{"direction", "rx"}}) };
 
 // Network Metrics
-        prometheus::simpleapi::gauge_metric_t network_num_joined
+prometheus::simpleapi::gauge_metric_t network_num_joined { "zt_num_networks", "number of networks this instance is joined to" };
-        { "zt_num_networks", "number of networks this instance is joined to" };
+prometheus::simpleapi::gauge_family_t network_num_multicast_groups { "zt_network_multicast_groups_subscribed", "number of multicast groups networks are subscribed to" };
-        prometheus::simpleapi::gauge_family_t network_num_multicast_groups
+prometheus::simpleapi::counter_family_t network_packets { "zt_network_packets", "number of incoming/outgoing packets per network" };
-        { "zt_network_multicast_groups_subscribed", "number of multicast groups networks are subscribed to" };
-        prometheus::simpleapi::counter_family_t network_packets
-        { "zt_network_packets", "number of incoming/outgoing packets per network" };
 
 #ifndef ZT_NO_PEER_METRICS
 // PeerMetrics
-        prometheus::CustomFamily<prometheus::Histogram<uint64_t>> &peer_latency = 
+prometheus::CustomFamily<prometheus::Histogram<uint64_t> >& peer_latency = prometheus::Builder<prometheus::Histogram<uint64_t> >().Name("zt_peer_latency").Help("peer latency (ms)").Register(prometheus::simpleapi::registry);
-        prometheus::Builder<prometheus::Histogram<uint64_t>>()
-            .Name("zt_peer_latency")
-            .Help("peer latency (ms)")
-            .Register(prometheus::simpleapi::registry);
 
-        prometheus::simpleapi::gauge_family_t peer_path_count
+prometheus::simpleapi::gauge_family_t peer_path_count { "zt_peer_path_count", "number of paths to peer" };
-        { "zt_peer_path_count", "number of paths to peer" };
+prometheus::simpleapi::counter_family_t peer_packets { "zt_peer_packets", "number of packets to/from a peer" };
-        prometheus::simpleapi::counter_family_t peer_packets
+prometheus::simpleapi::counter_family_t peer_packet_errors { "zt_peer_packet_errors", "number of incoming packet errors from a peer" };
-        { "zt_peer_packets", "number of packets to/from a peer" };
-        prometheus::simpleapi::counter_family_t peer_packet_errors
-        { "zt_peer_packet_errors" , "number of incoming packet errors from a peer" };
 #endif
 
 // General Controller Metrics
-        prometheus::simpleapi::gauge_metric_t   network_count
+prometheus::simpleapi::gauge_metric_t network_count { "controller_network_count", "number of networks the controller is serving" };
-        {"controller_network_count", "number of networks the controller is serving"};
+prometheus::simpleapi::gauge_metric_t member_count { "controller_member_count", "number of network members the controller is serving" };
-        prometheus::simpleapi::gauge_metric_t   member_count
+prometheus::simpleapi::counter_metric_t network_changes { "controller_network_change_count", "number of times a network configuration is changed" };
-        {"controller_member_count", "number of network members the controller is serving"};
+prometheus::simpleapi::counter_metric_t member_changes { "controller_member_change_count", "number of times a network member configuration is changed" };
-        prometheus::simpleapi::counter_metric_t network_changes
+prometheus::simpleapi::counter_metric_t member_auths { "controller_member_auth_count", "number of network member auths" };
-        {"controller_network_change_count", "number of times a network configuration is changed"};
+prometheus::simpleapi::counter_metric_t member_deauths { "controller_member_deauth_count", "number of network member deauths" };
-        prometheus::simpleapi::counter_metric_t member_changes
-        {"controller_member_change_count", "number of times a network member configuration is changed"};
-        prometheus::simpleapi::counter_metric_t member_auths
-        {"controller_member_auth_count", "number of network member auths"};
-        prometheus::simpleapi::counter_metric_t member_deauths
-        {"controller_member_deauth_count", "number of network member deauths"};
 
-        prometheus::simpleapi::gauge_metric_t network_config_request_queue_size
+prometheus::simpleapi::gauge_metric_t network_config_request_queue_size { "controller_network_config_request_queue", "number of entries in the request queue for network configurations" };
-        { "controller_network_config_request_queue", "number of entries in the request queue for network configurations" };
 
-        prometheus::simpleapi::counter_metric_t sso_expiration_checks
+prometheus::simpleapi::counter_metric_t sso_expiration_checks { "controller_sso_expiration_checks", "number of sso expiration checks done" };
-        { "controller_sso_expiration_checks", "number of sso expiration checks done" };
 
-        prometheus::simpleapi::counter_metric_t sso_member_deauth
+prometheus::simpleapi::counter_metric_t sso_member_deauth { "controller_sso_timeouts", "number of sso timeouts" };
-        { "controller_sso_timeouts", "number of sso timeouts" };
 
-        prometheus::simpleapi::counter_metric_t network_config_request
+prometheus::simpleapi::counter_metric_t network_config_request { "controller_network_config_request", "count of config requests handled" };
-        { "controller_network_config_request", "count of config requests handled" };
+prometheus::simpleapi::gauge_metric_t network_config_request_threads { "controller_network_config_request_threads", "number of active network config handling threads" };
-        prometheus::simpleapi::gauge_metric_t network_config_request_threads
+prometheus::simpleapi::counter_metric_t db_get_network { "controller_db_get_network", "counter" };
-        { "controller_network_config_request_threads", "number of active network config handling threads" };
+prometheus::simpleapi::counter_metric_t db_get_network_and_member { "controller_db_get_network_and_member", "counter" };
-        prometheus::simpleapi::counter_metric_t db_get_network
+prometheus::simpleapi::counter_metric_t db_get_network_and_member_and_summary { "controller_db_get_networK_and_member_summary", "counter" };
-        { "controller_db_get_network", "counter" };
+prometheus::simpleapi::counter_metric_t db_get_member_list { "controller_db_get_member_list", "counter" };
-        prometheus::simpleapi::counter_metric_t db_get_network_and_member
+prometheus::simpleapi::counter_metric_t db_get_network_list { "controller_db_get_network_list", "counter" };
-        { "controller_db_get_network_and_member", "counter" };
+prometheus::simpleapi::counter_metric_t db_member_change { "controller_db_member_change", "counter" };
-        prometheus::simpleapi::counter_metric_t db_get_network_and_member_and_summary
+prometheus::simpleapi::counter_metric_t db_network_change { "controller_db_network_change", "counter" };
-        { "controller_db_get_networK_and_member_summary", "counter" };
-        prometheus::simpleapi::counter_metric_t db_get_member_list
-        { "controller_db_get_member_list", "counter" };
-        prometheus::simpleapi::counter_metric_t db_get_network_list
-        { "controller_db_get_network_list", "counter" };
-        prometheus::simpleapi::counter_metric_t db_member_change
-        { "controller_db_member_change", "counter" };
-        prometheus::simpleapi::counter_metric_t db_network_change
-        { "controller_db_network_change", "counter" };
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 // Central Controller Metrics
-        prometheus::simpleapi::counter_metric_t pgsql_mem_notification
+prometheus::simpleapi::counter_metric_t pgsql_mem_notification { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" };
-        { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" };
+prometheus::simpleapi::counter_metric_t pgsql_net_notification { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
-        prometheus::simpleapi::counter_metric_t pgsql_net_notification
+prometheus::simpleapi::counter_metric_t pgsql_node_checkin { "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" };
-        { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
+prometheus::simpleapi::counter_metric_t pgsql_commit_ticks { "controller_pgsql_commit_ticks", "number of commit ticks run (pgsql)" };
-        prometheus::simpleapi::counter_metric_t pgsql_node_checkin
+prometheus::simpleapi::counter_metric_t db_get_sso_info { "controller_db_get_sso_info", "counter" };
-        { "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" };
-        prometheus::simpleapi::counter_metric_t pgsql_commit_ticks
-        { "controller_pgsql_commit_ticks", "number of commit ticks run (pgsql)" };
-        prometheus::simpleapi::counter_metric_t db_get_sso_info
-        { "controller_db_get_sso_info", "counter" };
 
-        prometheus::simpleapi::counter_metric_t redis_mem_notification
+prometheus::simpleapi::counter_metric_t redis_mem_notification { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
-        { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
+prometheus::simpleapi::counter_metric_t redis_net_notification { "controller_redis_network_notifications_received", "number of network change notifications received via redis" };
-        prometheus::simpleapi::counter_metric_t redis_net_notification
+prometheus::simpleapi::counter_metric_t redis_node_checkin { "controller_redis_node_checkin_count", "number of node check-ins (redis)" };
-        { "controller_redis_network_notifications_received", "number of network change notifications received via redis" };
-        prometheus::simpleapi::counter_metric_t redis_node_checkin
-        { "controller_redis_node_checkin_count", "number of node check-ins (redis)" };
 
 // Central DB Pool Metrics
-        prometheus::simpleapi::counter_metric_t conn_counter
+prometheus::simpleapi::counter_metric_t conn_counter { "controller_pgsql_connections_created", "number of pgsql connections created" };
-        { "controller_pgsql_connections_created", "number of pgsql connections created"};
+prometheus::simpleapi::counter_metric_t max_pool_size { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres" };
-        prometheus::simpleapi::counter_metric_t max_pool_size
+prometheus::simpleapi::counter_metric_t min_pool_size { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" };
-        { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres"};
+prometheus::simpleapi::gauge_metric_t pool_avail { "controller_pgsql_available_conns", "number of available postgres connections" };
-        prometheus::simpleapi::counter_metric_t min_pool_size
+prometheus::simpleapi::gauge_metric_t pool_in_use { "controller_pgsql_in_use_conns", "number of postgres database connections in use" };
-        { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" };
+prometheus::simpleapi::counter_metric_t pool_errors { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" };
-        prometheus::simpleapi::gauge_metric_t   pool_avail
-        { "controller_pgsql_available_conns", "number of available postgres connections" };
-        prometheus::simpleapi::gauge_metric_t   pool_in_use
-        { "controller_pgsql_in_use_conns", "number of postgres database connections in use" };
-        prometheus::simpleapi::counter_metric_t pool_errors
-        { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" };
 #endif
-    }
+}	// namespace Metrics
-}
+}	// namespace ZeroTier

node/Metrics.hpp

@@ -12,14 +12,16 @@
 #ifndef METRICS_H_
 #define METRICS_H_
 
+// clang-format off
 #include <prometheus/simpleapi.h>
 #include <prometheus/histogram.h>
+// clang-format on
 
 namespace prometheus {
 namespace simpleapi {
 extern std::shared_ptr<Registry> registry_ptr;
 }
-}
+}	// namespace prometheus
 
 namespace ZeroTier {
 namespace Metrics {
@@ -137,7 +139,6 @@ namespace ZeroTier {
 extern prometheus::simpleapi::counter_metric_t db_member_change;
 extern prometheus::simpleapi::counter_metric_t db_network_change;
 
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 // Central Controller Metrics
 extern prometheus::simpleapi::counter_metric_t pgsql_mem_notification;
@@ -150,8 +151,6 @@ namespace ZeroTier {
 extern prometheus::simpleapi::counter_metric_t redis_net_notification;
 extern prometheus::simpleapi::counter_metric_t redis_node_checkin;
 
-        
-
 // Central DB Pool Metrics
 extern prometheus::simpleapi::counter_metric_t conn_counter;
 extern prometheus::simpleapi::counter_metric_t max_pool_size;

node/MulticastGroup.hpp

@@ -14,10 +14,10 @@
 #ifndef ZT_MULTICASTGROUP_HPP
 #define ZT_MULTICASTGROUP_HPP
 
-#include <stdint.h>
-
-#include "MAC.hpp"
 #include "InetAddress.hpp"
+#include "MAC.hpp"
+
+#include <stdint.h>
 
 namespace ZeroTier {
 
@@ -36,18 +36,13 @@ namespace ZeroTier {
  *
  * MulticastGroup behaves as an immutable value object.
  */
-class MulticastGroup
+class MulticastGroup {
-{
   public:
-	MulticastGroup() :
+	MulticastGroup() : _mac(), _adi(0)
-		_mac(),
-		_adi(0)
 	{
 	}
 
-	MulticastGroup(const MAC &m,uint32_t a) :
+	MulticastGroup(const MAC& m, uint32_t a) : _mac(m), _adi(a)
-		_mac(m),
-		_adi(a)
 	{
 	}
 
@@ -65,7 +60,8 @@ public:
 			// ADI was added, as well as handling other things that want mindless
 			// Ethernet broadcast to all.
 			return MulticastGroup(MAC(0xffffffffffffULL), Utils::ntoh(*((const uint32_t*)ip.rawIpData())));
-		} else if (ip.isV6()) {
+		}
+		else if (ip.isV6()) {
 			// IPv6 is better designed in this respect. We can compute the IPv6
 			// multicast address directly from the IP address, and it gives us
 			// 24 bits of uniqueness. Collisions aren't likely to be common enough
@@ -79,29 +75,54 @@ public:
 	/**
 	 * @return Multicast address
 	 */
-	inline const MAC &mac() const { return _mac; }
+	inline const MAC& mac() const
+	{
+		return _mac;
+	}
 
 	/**
 	 * @return Additional distinguishing information
 	 */
-	inline uint32_t adi() const { return _adi; }
+	inline uint32_t adi() const
+	{
+		return _adi;
+	}
 
-	inline unsigned long hashCode() const { return (_mac.hashCode() ^ (unsigned long)_adi); }
+	inline unsigned long hashCode() const
+	{
+		return (_mac.hashCode() ^ (unsigned long)_adi);
+	}
 
-	inline bool operator==(const MulticastGroup &g) const { return ((_mac == g._mac)&&(_adi == g._adi)); }
+	inline bool operator==(const MulticastGroup& g) const
-	inline bool operator!=(const MulticastGroup &g) const { return ((_mac != g._mac)||(_adi != g._adi)); }
+	{
+		return ((_mac == g._mac) && (_adi == g._adi));
+	}
+	inline bool operator!=(const MulticastGroup& g) const
+	{
+		return ((_mac != g._mac) || (_adi != g._adi));
+	}
 	inline bool operator<(const MulticastGroup& g) const
 	{
 		if (_mac < g._mac) {
 			return true;
-		} else if (_mac == g._mac) {
+		}
+		else if (_mac == g._mac) {
 			return (_adi < g._adi);
 		}
 		return false;
 	}
-	inline bool operator>(const MulticastGroup &g) const { return (g < *this); }
+	inline bool operator>(const MulticastGroup& g) const
-	inline bool operator<=(const MulticastGroup &g) const { return !(g < *this); }
+	{
-	inline bool operator>=(const MulticastGroup &g) const { return !(*this < g); }
+		return (g < *this);
+	}
+	inline bool operator<=(const MulticastGroup& g) const
+	{
+		return ! (g < *this);
+	}
+	inline bool operator>=(const MulticastGroup& g) const
+	{
+		return ! (*this < g);
+	}
 
   private:
 	MAC _mac;

node/Multicaster.cpp

@@ -11,25 +11,24 @@
  */
 /****/
 
-#include <algorithm>
-
-#include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
 #include "Multicaster.hpp"
-#include "Topology.hpp"
+
-#include "Switch.hpp"
-#include "Packet.hpp"
-#include "Peer.hpp"
 #include "C25519.hpp"
 #include "CertificateOfMembership.hpp"
-#include "Node.hpp"
+#include "Constants.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
+#include "Packet.hpp"
+#include "Peer.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
+
+#include <algorithm>
 
 namespace ZeroTier {
 
-Multicaster::Multicaster(const RuntimeEnvironment *renv) :
+Multicaster::Multicaster(const RuntimeEnvironment* renv) : RR(renv), _groups(32)
-	RR(renv),
-	_groups(32)
 {
 }
 
@@ -71,7 +70,8 @@ unsigned int Multicaster::gather(const Address &queryingPeer,uint64_t nwid,const
 
 	if (! limit) {
 		return 0;
-	} else if (limit > 0xffff) {
+	}
+	else if (limit > 0xffff) {
 		limit = 0xffff;
 	}
 
@@ -146,16 +146,7 @@ std::vector<Address> Multicaster::getMembers(uint64_t nwid,const MulticastGroup
 	return ls;
 }
 
-void Multicaster::send(
+void Multicaster::send(void* tPtr, int64_t now, const SharedPtr<Network>& network, const Address& origin, const MulticastGroup& mg, const MAC& src, unsigned int etherType, const void* data, unsigned int len)
-	void *tPtr,
-	int64_t now,
-	const SharedPtr<Network> &network,
-	const Address &origin,
-	const MulticastGroup &mg,
-	const MAC &src,
-	unsigned int etherType,
-	const void *data,
-	unsigned int len)
 {
 	unsigned long idxbuf[4096];
 	unsigned long* indexes = idxbuf;
@@ -256,9 +247,6 @@ void Multicaster::send(
 			for (unsigned int i = 0; i < activeBridgeCount; ++i) {
 				if ((activeBridges[i] != RR->identity.address()) && (activeBridges[i] != origin)) {
 					out.sendOnly(RR, tPtr, activeBridges[i]);	// optimization: don't use dedup log if it's a one-pass send
-					if (++count >= limit) {
-						break;
-					}
 				}
 			}
 
@@ -270,7 +258,8 @@ void Multicaster::send(
 					++count;
 				}
 			}
-		} else {
+		}
+		else {
 			while (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
 				gs.txQueue.pop_front();
 			}
@@ -342,18 +331,7 @@ void Multicaster::send(
 			gs.txQueue.push_back(OutboundMulticast());
 			OutboundMulticast& out = gs.txQueue.back();
 
-			out.init(
+			out.init(RR, now, network->id(), network->config().disableCompression(), limit, gatherLimit, src, mg, etherType, data, len);
-				RR,
-				now,
-				network->id(),
-				network->config().disableCompression(),
-				limit,
-				gatherLimit,
-				src,
-				mg,
-				etherType,
-				data,
-				len);
 
 			if (origin) {
 				out.logAsSent(origin);
@@ -379,7 +357,9 @@ void Multicaster::send(
 				}
 			}
 		}
-	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
+	}
+	catch (...) {
+	}	// this is a sanity check to catch any failures and make sure indexes[] still gets deleted
 
 	// Free allocated memory buffer if any
 	if (indexes != idxbuf) {
@@ -397,7 +377,8 @@ void Multicaster::clean(int64_t now)
 		for (std::list<OutboundMulticast>::iterator tx(s->txQueue.begin()); tx != s->txQueue.end();) {
 			if ((tx->expired(now)) || (tx->atLimit())) {
 				s->txQueue.erase(tx++);
-			} else {
+			}
+			else {
 				++tx;
 			}
 		}
@@ -418,9 +399,11 @@ void Multicaster::clean(int64_t now)
 
 		if (count) {
 			s->members.resize(count);
-		} else if (s->txQueue.empty()) {
+		}
+		else if (s->txQueue.empty()) {
 			_groups.erase(*k);
-		} else {
+		}
+		else {
 			s->members.clear();
 		}
 	}
@@ -442,18 +425,21 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup
 			return;
 		}
 		gs.members.insert(m, MulticastGroupMember(member, now));
-	} else {
+	}
+	else {
 		gs.members.push_back(MulticastGroupMember(member, now));
 	}
 
 	for (std::list<OutboundMulticast>::iterator tx(gs.txQueue.begin()); tx != gs.txQueue.end();) {
 		if (tx->atLimit()) {
 			gs.txQueue.erase(tx++);
-		} else {
+		}
+		else {
 			tx->sendIfNew(RR, tPtr, member);
 			if (tx->atLimit()) {
 				gs.txQueue.erase(tx++);
-			} else {
+			}
+			else {
 				++tx;
 			}
 		}

node/Multicaster.hpp

@@ -14,22 +14,21 @@
 #ifndef ZT_MULTICASTER_HPP
 #define ZT_MULTICASTER_HPP
 
-#include <stdint.h>
+#include "Address.hpp"
-#include <string.h>
-
-#include <map>
-#include <vector>
-#include <list>
-
 #include "Constants.hpp"
 #include "Hashtable.hpp"
-#include "Address.hpp"
 #include "MAC.hpp"
 #include "MulticastGroup.hpp"
-#include "OutboundMulticast.hpp"
-#include "Utils.hpp"
 #include "Mutex.hpp"
+#include "OutboundMulticast.hpp"
 #include "SharedPtr.hpp"
+#include "Utils.hpp"
+
+#include <list>
+#include <map>
+#include <stdint.h>
+#include <string.h>
+#include <vector>
 
 namespace ZeroTier {
 
@@ -41,8 +40,7 @@ class Network;
 /**
  * Database of known multicast peers within a network
  */
-class Multicaster
+class Multicaster {
-{
   public:
 	Multicaster(const RuntimeEnvironment* renv);
 	~Multicaster();
@@ -126,16 +124,7 @@ public:
 	 * @param data Packet data
 	 * @param len Length of packet data
 	 */
-	void send(
+	void send(void* tPtr, int64_t now, const SharedPtr<Network>& network, const Address& origin, const MulticastGroup& mg, const MAC& src, unsigned int etherType, const void* data, unsigned int len);
-		void *tPtr,
-		int64_t now,
-		const SharedPtr<Network> &network,
-		const Address &origin,
-		const MulticastGroup &mg,
-		const MAC &src,
-		unsigned int etherType,
-		const void *data,
-		unsigned int len);
 
 	/**
 	 * Clean database
@@ -146,38 +135,72 @@ public:
 	void clean(int64_t now);
 
   private:
-	struct Key
+	struct Key {
+		Key() : nwid(0), mg()
 		{
-		Key() : nwid(0),mg() {}
+		}
-		Key(uint64_t n,const MulticastGroup &g) : nwid(n),mg(g) {}
+		Key(uint64_t n, const MulticastGroup& g) : nwid(n), mg(g)
+		{
+		}
 
 		uint64_t nwid;
 		MulticastGroup mg;
 
-		inline bool operator==(const Key &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); }
+		inline bool operator==(const Key& k) const
-		inline bool operator!=(const Key &k) const { return ((nwid != k.nwid)||(mg != k.mg)); }
+		{
-		inline unsigned long hashCode() const { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); }
+			return ((nwid == k.nwid) && (mg == k.mg));
+		}
+		inline bool operator!=(const Key& k) const
+		{
+			return ((nwid != k.nwid) || (mg != k.mg));
+		}
+		inline unsigned long hashCode() const
+		{
+			return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32)));
+		}
 	};
 
-	struct MulticastGroupMember
+	struct MulticastGroupMember {
+		MulticastGroupMember()
 		{
-		MulticastGroupMember() {}
+		}
-		MulticastGroupMember(const Address &a,uint64_t ts) : address(a),timestamp(ts) {}
+		MulticastGroupMember(const Address& a, uint64_t ts) : address(a), timestamp(ts)
+		{
+		}
 
-		inline bool operator<(const MulticastGroupMember &a) const { return (address < a.address); }
+		inline bool operator<(const MulticastGroupMember& a) const
-		inline bool operator==(const MulticastGroupMember &a) const { return (address == a.address); }
+		{
-		inline bool operator!=(const MulticastGroupMember &a) const { return (address != a.address); }
+			return (address < a.address);
-		inline bool operator<(const Address &a) const { return (address < a); }
+		}
-		inline bool operator==(const Address &a) const { return (address == a); }
+		inline bool operator==(const MulticastGroupMember& a) const
-		inline bool operator!=(const Address &a) const { return (address != a); }
+		{
+			return (address == a.address);
+		}
+		inline bool operator!=(const MulticastGroupMember& a) const
+		{
+			return (address != a.address);
+		}
+		inline bool operator<(const Address& a) const
+		{
+			return (address < a);
+		}
+		inline bool operator==(const Address& a) const
+		{
+			return (address == a);
+		}
+		inline bool operator!=(const Address& a) const
+		{
+			return (address != a);
+		}
 
 		Address address;
 		int64_t timestamp;	 // time of last notification
 	};
 
-	struct MulticastGroupStatus
+	struct MulticastGroupStatus {
+		MulticastGroupStatus() : lastExplicitGather(0)
 		{
-		MulticastGroupStatus() : lastExplicitGather(0) {}
+		}
 
 		int64_t lastExplicitGather;
 		std::list<OutboundMulticast> txQueue;		 // pending outbound multicasts

node/Mutex.hpp

@@ -18,15 +18,14 @@
 
 #ifdef __UNIX_LIKE__
 
+#include <pthread.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include <pthread.h>
 
 namespace ZeroTier {
 
 // libpthread based mutex lock
-class Mutex
+class Mutex {
-{
   public:
 	Mutex()
 	{
@@ -48,17 +47,14 @@ public:
 		pthread_mutex_unlock(&((const_cast<Mutex*>(this))->_mh));
 	}
 
-	class Lock
+	class Lock {
-	{
 	  public:
-		Lock(Mutex &m) :
+		Lock(Mutex& m) : _m(&m)
-			_m(&m)
 		{
 			m.lock();
 		}
 
-		Lock(const Mutex &m) :
+		Lock(const Mutex& m) : _m(const_cast<Mutex*>(&m))
-			_m(const_cast<Mutex *>(&m))
 		{
 			_m->lock();
 		}
@@ -73,8 +69,13 @@ public:
 	};
 
   private:
-	Mutex(const Mutex &) {}
+	Mutex(const Mutex&)
-	const Mutex &operator=(const Mutex &) { return *this; }
+	{
+	}
+	const Mutex& operator=(const Mutex&)
+	{
+		return *this;
+	}
 
 	pthread_mutex_t _mh;
 };
@@ -91,8 +92,7 @@ private:
 namespace ZeroTier {
 
 // Windows critical section based lock
-class Mutex
+class Mutex {
-{
   public:
 	Mutex()
 	{
@@ -124,17 +124,14 @@ public:
 		(const_cast<Mutex*>(this))->unlock();
 	}
 
-	class Lock
+	class Lock {
-	{
 	  public:
-		Lock(Mutex &m) :
+		Lock(Mutex& m) : _m(&m)
-			_m(&m)
 		{
 			m.lock();
 		}
 
-		Lock(const Mutex &m) :
+		Lock(const Mutex& m) : _m(const_cast<Mutex*>(&m))
-			_m(const_cast<Mutex *>(&m))
 		{
 			_m->lock();
 		}
@@ -149,8 +146,13 @@ public:
 	};
 
   private:
-	Mutex(const Mutex &) {}
+	Mutex(const Mutex&)
-	const Mutex &operator=(const Mutex &) { return *this; }
+	{
+	}
+	const Mutex& operator=(const Mutex&)
+	{
+		return *this;
+	}
 
 	CRITICAL_SECTION _cs;
 };

node/Network.cpp

@@ -11,30 +11,29 @@
  */
 /****/
 
-#include <stdio.h>
+#include "Network.hpp"
-#include <string.h>
-#include <stdlib.h>
-#include <math.h>
 
 #include "../include/ZeroTierDebug.h"
-
-#include "Constants.hpp"
 #include "../version.h"
-#include "Network.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "MAC.hpp"
 #include "Address.hpp"
-#include "InetAddress.hpp"
-#include "Switch.hpp"
 #include "Buffer.hpp"
-#include "Packet.hpp"
+#include "Constants.hpp"
+#include "InetAddress.hpp"
+#include "MAC.hpp"
+#include "Metrics.hpp"
 #include "NetworkController.hpp"
 #include "Node.hpp"
+#include "Packet.hpp"
 #include "Peer.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
 #include "Trace.hpp"
-#include "Metrics.hpp"
 
+#include <math.h>
 #include <set>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 namespace ZeroTier {
 
@@ -71,14 +70,7 @@ static inline bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLe
 	return false;	// overflow == invalid
 }
 
-enum _doZtFilterResult
+enum _doZtFilterResult { DOZTFILTER_NO_MATCH, DOZTFILTER_DROP, DOZTFILTER_REDIRECT, DOZTFILTER_ACCEPT, DOZTFILTER_SUPER_ACCEPT };
-{
-	DOZTFILTER_NO_MATCH,
-	DOZTFILTER_DROP,
-	DOZTFILTER_REDIRECT,
-	DOZTFILTER_ACCEPT,
-	DOZTFILTER_SUPER_ACCEPT
-};
 
 static _doZtFilterResult _doZtFilter(
 	const RuntimeEnvironment* RR,
@@ -137,7 +129,8 @@ static _doZtFilterResult _doZtFilter(
 #ifdef ZT_TRACE
 							// fprintf(stderr, "\tskip Drop");
 #endif
-							skipDrop = 0; continue;
+							skipDrop = 0;
+							continue;
 						}
 #ifdef ZT_TRACE
 						// fprintf(stderr, "\tDrop\n");
@@ -159,23 +152,29 @@ static _doZtFilterResult _doZtFilter(
 						const Address fwdAddr(rules[rn].v.fwd.address);
 						if (fwdAddr == ztSource) {
 							// Skip as no-op since source is target
-						} else if (fwdAddr == RR->identity.address()) {
+						}
+						else if (fwdAddr == RR->identity.address()) {
 							if (inbound) {
 								return DOZTFILTER_SUPER_ACCEPT;
-							} else {
 							}
-						} else if (fwdAddr == ztDest) {
+							else {
-						} else {
+							}
+						}
+						else if (fwdAddr == ztDest) {
+						}
+						else {
 							if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) {
 								ztDest = fwdAddr;
 								return DOZTFILTER_REDIRECT;
-							} else {
+							}
+							else {
 								cc = fwdAddr;
 								ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen;
 								ccWatch = (rt == ZT_NETWORK_RULE_ACTION_WATCH);
 							}
 						}
-					}	continue;
+					}
+						continue;
 
 					case ZT_NETWORK_RULE_ACTION_BREAK:
 						return DOZTFILTER_NO_MATCH;
@@ -184,7 +183,8 @@ static _doZtFilterResult _doZtFilter(
 					default:
 						continue;
 				}
-			} else {
+			}
+			else {
 				// If this is an incoming packet and we are a TEE or REDIRECT target, we should
 				// super-accept if we accept at all. This will cause us to accept redirected or
 				// tee'd packets in spite of MAC and ZT addressing checks.
@@ -247,28 +247,32 @@ static _doZtFilterResult _doZtFilter(
 			case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE:
 				if ((etherType == ZT_ETHERTYPE_IPV4) && (frameLen >= 20)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void*)&(rules[rn].v.ipv4.ip), 4, rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void*)(frameData + 12), 4, 0)));
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV4_DEST:
 				if ((etherType == ZT_ETHERTYPE_IPV4) && (frameLen >= 20)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void*)&(rules[rn].v.ipv4.ip), 4, rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void*)(frameData + 16), 4, 0)));
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE:
 				if ((etherType == ZT_ETHERTYPE_IPV6) && (frameLen >= 40)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void*)rules[rn].v.ipv6.ip, 16, rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void*)(frameData + 8), 16, 0)));
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV6_DEST:
 				if ((etherType == ZT_ETHERTYPE_IPV6) && (frameLen >= 40)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void*)rules[rn].v.ipv6.ip, 16, rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void*)(frameData + 24), 16, 0)));
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
@@ -276,24 +280,29 @@ static _doZtFilterResult _doZtFilter(
 				if ((etherType == ZT_ETHERTYPE_IPV4) && (frameLen >= 20)) {
 					const uint8_t tosMasked = frameData[1] & rules[rn].v.ipTos.mask;
 					thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0]) && (tosMasked <= rules[rn].v.ipTos.value[1]));
-				} else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) {
+				}
+				else if ((etherType == ZT_ETHERTYPE_IPV6) && (frameLen >= 40)) {
 					const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask;
 					thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0]) && (tosMasked <= rules[rn].v.ipTos.value[1]));
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL:
 				if ((etherType == ZT_ETHERTYPE_IPV4) && (frameLen >= 20)) {
 					thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]);
-				} else if (etherType == ZT_ETHERTYPE_IPV6) {
+				}
+				else if (etherType == ZT_ETHERTYPE_IPV6) {
 					unsigned int pos = 0, proto = 0;
 					if (_ipv6GetPayload(frameData, frameLen, pos, proto)) {
 						thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto);
-					} else {
+					}
+					else {
 						thisRuleMatches = hardNo;
 					}
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
@@ -308,38 +317,48 @@ static _doZtFilterResult _doZtFilter(
 							if (rules[rn].v.icmp.type == frameData[ihl]) {
 								if ((rules[rn].v.icmp.flags & 0x01) != 0) {
 									thisRuleMatches = (uint8_t)(frameData[ihl + 1] == rules[rn].v.icmp.code);
-								} else {
+								}
+								else {
 									thisRuleMatches = hardYes;
 								}
-							} else {
+							}
+							else {
 								thisRuleMatches = hardNo;
 							}
-						} else {
+						}
+						else {
 							thisRuleMatches = hardNo;
 						}
-					} else {
+					}
+					else {
 						thisRuleMatches = hardNo;
 					}
-				} else if (etherType == ZT_ETHERTYPE_IPV6) {
+				}
+				else if (etherType == ZT_ETHERTYPE_IPV6) {
 					unsigned int pos = 0, proto = 0;
 					if (_ipv6GetPayload(frameData, frameLen, pos, proto)) {
 						if ((proto == 0x3a) && (frameLen >= (pos + 2))) {
 							if (rules[rn].v.icmp.type == frameData[pos]) {
 								if ((rules[rn].v.icmp.flags & 0x01) != 0) {
 									thisRuleMatches = (uint8_t)(frameData[pos + 1] == rules[rn].v.icmp.code);
-								} else {
+								}
+								else {
 									thisRuleMatches = hardYes;
 								}
-							} else {
+							}
+							else {
 								thisRuleMatches = hardNo;
 							}
-						} else {
+						}
+						else {
 							thisRuleMatches = hardNo;
 						}
-					} else {
+					}
+					else {
 						thisRuleMatches = hardNo;
 					}
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
@@ -363,7 +382,8 @@ static _doZtFilterResult _doZtFilter(
 					}
 
 					thisRuleMatches = (p >= 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0]) && (p <= (int)rules[rn].v.port[1])) : (uint8_t)0;
-				} else if (etherType == ZT_ETHERTYPE_IPV6) {
+				}
+				else if (etherType == ZT_ETHERTYPE_IPV6) {
 					unsigned int pos = 0, proto = 0;
 					if (_ipv6GetPayload(frameData, frameLen, pos, proto)) {
 						int p = -1;
@@ -383,10 +403,12 @@ static _doZtFilterResult _doZtFilter(
 								break;
 						}
 						thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0]) && (p <= (int)rules[rn].v.port[1])) : (uint8_t)0;
-					} else {
+					}
+					else {
 						thisRuleMatches = hardNo;
 					}
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 				break;
@@ -403,7 +425,8 @@ static _doZtFilterResult _doZtFilter(
 					InetAddress src;
 					if ((etherType == ZT_ETHERTYPE_IPV4) && (frameLen >= 20)) {
 						src.set((const void*)(frameData + 12), 4, 0);
-					} else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) {
+					}
+					else if ((etherType == ZT_ETHERTYPE_IPV6) && (frameLen >= 40)) {
 						// IPv6 NDP requires special handling, since the src and dest IPs in the packet are empty or link-local.
 						if ((frameLen >= (40 + 8 + 16)) && (frameData[6] == 0x3a) && ((frameData[40] == 0x87) || (frameData[40] == 0x88))) {
 							if (frameData[40] == 0x87) {
@@ -411,15 +434,18 @@ static _doZtFilterResult _doZtFilter(
 								// hack by considering them authenticated. Otherwise you would pretty much have to do
 								// this manually in the rule set for IPv6 to work at all.
 								ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED;
-							} else {
+							}
+							else {
 								// Neighbor advertisements on the other hand can absolutely be authenticated.
 								src.set((const void*)(frameData + 40 + 8), 16, 0);
 							}
-						} else {
+						}
+						else {
 							// Other IPv6 packets can be handled normally
 							src.set((const void*)(frameData + 8), 16, 0);
 						}
-					} else if ((etherType == ZT_ETHERTYPE_ARP)&&(frameLen >= 28)) {
+					}
+					else if ((etherType == ZT_ETHERTYPE_ARP) && (frameLen >= 28)) {
 						src.set((const void*)(frameData + 14), 4, 0);
 					}
 					if (inbound) {
@@ -431,7 +457,8 @@ static _doZtFilterResult _doZtFilter(
 								ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED;
 							}
 						}
-					} else {
+					}
+					else {
 						for (unsigned int i = 0; i < nconf.certificateOfOwnershipCount; ++i) {
 							if ((src) && (nconf.certificatesOfOwnership[i].owns(src))) {
 								ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED;
@@ -447,7 +474,8 @@ static _doZtFilterResult _doZtFilter(
 					const unsigned int headerLen = 4 * (frameData[0] & 0xf);
 					cf |= (uint64_t)frameData[headerLen + 13];
 					cf |= (((uint64_t)(frameData[headerLen + 12] & 0x0f)) << 8);
-				} else if (etherType == ZT_ETHERTYPE_IPV6) {
+				}
+				else if (etherType == ZT_ETHERTYPE_IPV6) {
 					unsigned int pos = 0, proto = 0;
 					if (_ipv6GetPayload(frameData, frameLen, pos, proto)) {
 						if ((proto == 0x06) && (frameLen > (pos + 14))) {
@@ -486,24 +514,31 @@ static _doZtFilterResult _doZtFilter(
 						if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) {
 							const uint32_t diff = (ltv > rtv) ? (ltv - rtv) : (rtv - ltv);
 							thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value);
-						} else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) {
+						}
+						else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) {
 							thisRuleMatches = (uint8_t)((ltv & rtv) == rules[rn].v.tag.value);
-						} else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) {
+						}
+						else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) {
 							thisRuleMatches = (uint8_t)((ltv | rtv) == rules[rn].v.tag.value);
-						} else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) {
+						}
+						else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) {
 							thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value);
-						} else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) {
+						}
+						else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) {
 							thisRuleMatches = (uint8_t)((ltv == rules[rn].v.tag.value) && (rtv == rules[rn].v.tag.value));
-						} else { // sanity check, can't really happen
+						}
+						else {	 // sanity check, can't really happen
 							thisRuleMatches = hardNo;
 						}
-					} else {
+					}
+					else {
 						if ((inbound) && (! superAccept)) {
 							thisRuleMatches = hardNo;
 #ifdef ZT_TRACE
 							// fprintf(stderr, "\tinbound ");
 #endif
-						} else {
+						}
+						else {
 							// Outbound side is not strict since if we have to match both tags and
 							// we are sending a first packet to a recipient, we probably do not know
 							// about their tags yet. They will filter on inbound and we will filter
@@ -516,7 +551,8 @@ static _doZtFilterResult _doZtFilter(
 #endif
 						}
 					}
-				} else {
+				}
+				else {
 					thisRuleMatches = hardNo;
 				}
 			} break;
@@ -535,24 +571,29 @@ static _doZtFilterResult _doZtFilter(
 				if (superAccept) {
 					skipDrop = 1;
 					thisRuleMatches = hardYes;
-				} else if ( ((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER)&&(inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER)&&(!inbound)) ) {
+				}
+				else if (((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER) && (inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) && (! inbound))) {
 					const Tag* const remoteTag = ((membership) ? membership->getTag(nconf, rules[rn].v.tag.id) : (const Tag*)0);
 					if (remoteTag) {
 						thisRuleMatches = (uint8_t)(remoteTag->value() == rules[rn].v.tag.value);
-					} else {
+					}
+					else {
 						if (rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) {
 							// If we are checking the receiver and this is an outbound packet, we
 							// can't be strict since we may not yet know the receiver's tag.
 							skipDrop = 1;
 							thisRuleMatches = hardYes;
-						} else {
+						}
+						else {
 							thisRuleMatches = hardNo;
 						}
 					}
-				} else { // sender and outbound or receiver and inbound
+				}
+				else {	 // sender and outbound or receiver and inbound
 					if ((localTag != &(nconf.tags[nconf.tagCount])) && (localTag->id() == rules[rn].v.tag.id)) {
 						thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value);
-					} else {
+					}
+					else {
 						thisRuleMatches = hardNo;
 					}
 				}
@@ -572,7 +613,8 @@ static _doZtFilterResult _doZtFilter(
 						}
 					}
 					integer &= 0xffffffffffffffffULL >> (64 - bits);
-				} else {
+				}
+				else {
 					// Little-endian
 					unsigned int idx = rules[rn].v.intRange.idx;
 					const unsigned int eof = idx + bytes;
@@ -598,7 +640,8 @@ static _doZtFilterResult _doZtFilter(
 
 		if ((rules[rn].t & 0x40)) {
 			thisSetMatches |= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1));
-		} else {
+		}
+		else {
 			thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1));
 		}
 	}
@@ -610,23 +653,23 @@ static _doZtFilterResult _doZtFilter(
 
 const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL), 0);
 
-Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr,const NetworkConfig *nconf) :
+Network::Network(const RuntimeEnvironment* renv, void* tPtr, uint64_t nwid, void* uptr, const NetworkConfig* nconf)
-	RR(renv),
+	: RR(renv)
-	_uPtr(uptr),
+	, _uPtr(uptr)
-	_id(nwid),
+	, _id(nwid)
-	_nwidStr(OSUtils::networkIDStr(nwid)),
+	, _nwidStr(OSUtils::networkIDStr(nwid))
-	_lastAnnouncedMulticastGroupsUpstream(0),
+	, _lastAnnouncedMulticastGroupsUpstream(0)
-	_mac(renv->identity.address(),nwid),
+	, _mac(renv->identity.address(), nwid)
-	_portInitialized(false),
+	, _portInitialized(false)
-	_lastConfigUpdate(0),
+	, _lastConfigUpdate(0)
-	_destroyed(false),
+	, _destroyed(false)
-	_netconfFailure(NETCONF_FAILURE_NONE),
+	, _netconfFailure(NETCONF_FAILURE_NONE)
-	_portError(0),
+	, _portError(0)
-	_num_multicast_groups{Metrics::network_num_multicast_groups.Add({{"network_id", _nwidStr}})},
+	, _num_multicast_groups { Metrics::network_num_multicast_groups.Add({ { "network_id", _nwidStr } }) }
-	_incoming_packets_accepted{Metrics::network_packets.Add({{"direction","rx"},{"network_id", _nwidStr},{"accepted","yes"}})},
+	, _incoming_packets_accepted { Metrics::network_packets.Add({ { "direction", "rx" }, { "network_id", _nwidStr }, { "accepted", "yes" } }) }
-	_incoming_packets_dropped{Metrics::network_packets.Add({{"direction","rx"},{"network_id", _nwidStr},{"accepted","no"}})},
+	, _incoming_packets_dropped { Metrics::network_packets.Add({ { "direction", "rx" }, { "network_id", _nwidStr }, { "accepted", "no" } }) }
-	_outgoing_packets_accepted{Metrics::network_packets.Add({{"direction","tx"},{"network_id", _nwidStr},{"accepted","yes"}})},
+	, _outgoing_packets_accepted { Metrics::network_packets.Add({ { "direction", "tx" }, { "network_id", _nwidStr }, { "accepted", "yes" } }) }
-	_outgoing_packets_dropped{Metrics::network_packets.Add({{"direction","tx"},{"network_id", _nwidStr},{"accepted","no"}})}
+	, _outgoing_packets_dropped { Metrics::network_packets.Add({ { "direction", "tx" }, { "network_id", _nwidStr }, { "accepted", "no" } }) }
 {
 	for (int i = 0; i < ZT_NETWORK_MAX_INCOMING_UPDATES; ++i) {
 		_incomingConfigChunks[i].ts = 0;
@@ -635,7 +678,8 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 	if (nconf) {
 		this->setConfiguration(tPtr, *nconf, false);
 		_lastConfigUpdate = 0;	 // still want to re-request since it's likely outdated
-	} else {
+	}
+	else {
 		uint64_t tmp[2];
 		tmp[0] = nwid;
 		tmp[1] = 0;
@@ -652,10 +696,14 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 						_lastConfigUpdate = 0;	 // still want to re-request an update since it's likely outdated
 						got = true;
 					}
-				} catch ( ... ) {}
+				}
+				catch (...) {
+				}
 				delete nconf;
 			}
-		} catch ( ... ) {}
+		}
+		catch (...) {
+		}
 		delete dict;
 
 		if (! got) {
@@ -682,7 +730,8 @@ Network::~Network()
 	if (_destroyed) {
 		// This is done in Node::leave() so we can pass tPtr properly
 		// RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-	} else {
+	}
+	else {
 		RR->node->configureVirtualNetworkPort((void*)0, _id, &_uPtr, ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN, &ctmp);
 	}
 }
@@ -713,14 +762,32 @@ bool Network::filterOutgoingPacket(
 	Membership* const membership = (ztDest) ? _memberships.get(ztDest) : (Membership*)0;
 
 	switch (_doZtFilter(RR, rrl, _config, membership, false, ztSource, ztFinalDest, macSource, macDest, frameData, frameLen, etherType, vlanId, _config.rules, _config.ruleCount, cc, ccLength, ccWatch, qosBucket)) {
-
 		case DOZTFILTER_NO_MATCH: {
 			for (unsigned int c = 0; c < _config.capabilityCount; ++c) {
 				ztFinalDest = ztDest;	// sanity check, shouldn't be possible if there was no match
 				Address cc2;
 				unsigned int ccLength2 = 0;
 				bool ccWatch2 = false;
-				switch (_doZtFilter(RR,crrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2,qosBucket)) {
+				switch (_doZtFilter(
+					RR,
+					crrl,
+					_config,
+					membership,
+					false,
+					ztSource,
+					ztFinalDest,
+					macSource,
+					macDest,
+					frameData,
+					frameLen,
+					etherType,
+					vlanId,
+					_config.capabilities[c].rules(),
+					_config.capabilities[c].ruleCount(),
+					cc2,
+					ccLength2,
+					ccWatch2,
+					qosBucket)) {
 					case DOZTFILTER_NO_MATCH:
 					case DOZTFILTER_DROP:	// explicit DROP in a capability just terminates its evaluation and is an anti-pattern
 						break;
@@ -793,19 +860,69 @@ bool Network::filterOutgoingPacket(
 			RR->sw->send(tPtr, outp, true);
 
 			if (_config.remoteTraceTarget) {
-				RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0);
+				RR->t->networkFilter(
+					tPtr,
+					*this,
+					rrl,
+					(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog*)0,
+					(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability*)0,
+					ztSource,
+					ztDest,
+					macSource,
+					macDest,
+					frameData,
+					frameLen,
+					etherType,
+					vlanId,
+					noTee,
+					false,
+					0);
 			}
 			return false;	// DROP locally, since we redirected
-		} else {
+		}
+		else {
 			if (_config.remoteTraceTarget) {
-				RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,1);
+				RR->t->networkFilter(
+					tPtr,
+					*this,
+					rrl,
+					(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog*)0,
+					(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability*)0,
+					ztSource,
+					ztDest,
+					macSource,
+					macDest,
+					frameData,
+					frameLen,
+					etherType,
+					vlanId,
+					noTee,
+					false,
+					1);
 			}
 			return true;
 		}
-	} else {
+	}
+	else {
 		_outgoing_packets_dropped++;
 		if (_config.remoteTraceTarget) {
-			RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0);
+			RR->t->networkFilter(
+				tPtr,
+				*this,
+				rrl,
+				(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog*)0,
+				(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability*)0,
+				ztSource,
+				ztDest,
+				macSource,
+				macDest,
+				frameData,
+				frameLen,
+				etherType,
+				vlanId,
+				noTee,
+				false,
+				0);
 		}
 		return false;
 	}
@@ -837,7 +954,6 @@ int Network::filterIncomingPacket(
 	Membership& membership = _membership(sourcePeer->address());
 
 	switch (_doZtFilter(RR, rrl, _config, &membership, true, sourcePeer->address(), ztFinalDest, macSource, macDest, frameData, frameLen, etherType, vlanId, _config.rules, _config.ruleCount, cc, ccLength, ccWatch, qosBucket)) {
-
 		case DOZTFILTER_NO_MATCH: {
 			Membership::CapabilityIterator mci(membership, _config);
 			while ((c = mci.next())) {
@@ -920,7 +1036,8 @@ int Network::filterIncomingPacket(
 			}
 			return 0;	// DROP locally, since we redirected
 		}
-	} else {
+	}
+	else {
 		_incoming_packets_dropped++;
 	}
 
@@ -935,7 +1052,8 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr
 	Mutex::Lock _l(_lock);
 	if (std::binary_search(_myMulticastGroups.begin(), _myMulticastGroups.end(), mg)) {
 		return true;
-	} else if (includeBridgedGroups) {
+	}
+	else if (includeBridgedGroups) {
 		return _multicastGroupsBehindMe.contains(mg);
 	}
 	return false;
@@ -1017,7 +1135,8 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add
 					}
 
 					break;
-				} else if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) {
+				}
+				else if ((! c) || (_incomingConfigChunks[i].ts < c->ts)) {
 					c = &(_incomingConfigChunks[i]);
 				}
 			}
@@ -1044,7 +1163,8 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add
 					}
 				}
 			}
-		} else if ((source == controller())||(!source)) { // since old chunks aren't signed, only accept from controller itself (or via cluster backplane)
+		}
+		else if ((source == controller()) || (! source)) {	 // since old chunks aren't signed, only accept from controller itself (or via cluster backplane)
 			// Legacy support for OK(NETWORK_CONFIG_REQUEST) from older controllers
 			chunkId = packetId;
 			configUpdateId = chunkId;
@@ -1060,7 +1180,8 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add
 					c = &(_incomingConfigChunks[i]);
 				}
 			}
-		} else {
+		}
+		else {
 			// Single-chunk unsigned legacy configs are only allowed from the controller itself
 			return 0;
 		}
@@ -1089,7 +1210,8 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add
 					delete nc;
 					nc = (NetworkConfig*)0;
 				}
-			} catch ( ... ) {
+			}
+			catch (...) {
 				delete nc;
 				nc = (NetworkConfig*)0;
 			}
@@ -1100,7 +1222,8 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add
 		this->setConfiguration(tPtr, *nc, true);
 		delete nc;
 		return configUpdateId;
-	} else {
+	}
+	else {
 		return 0;
 	}
 
@@ -1149,12 +1272,16 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 					tmp[1] = 0;
 					RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_NETWORK_CONFIG, tmp, d->data(), d->sizeBytes());
 				}
-			} catch ( ... ) {}
+			}
+			catch (...) {
+			}
 			delete d;
 		}
 
 		return 2;	// OK and configuration has changed
-	} catch ( ... ) {} // ignore invalid configs
+	}
+	catch (...) {
+	}	// ignore invalid configs
 	return 0;
 }
 
@@ -1232,10 +1359,12 @@ void Network::requestConfiguration(void *tPtr)
 
 				this->setConfiguration(tPtr, *nconf, false);
 				delete nconf;
-			} else {
+			}
+			else {
 				this->setNotFound(tPtr);
 			}
-		} else if ((_id & 0xff) == 0x01) {
+		}
+		else if ((_id & 0xff) == 0x01) {
 			// ffAAaaaaaaaaaa01 -- where AA is the IPv4 /8 to use and aaaaaaaaaa is the anchor node for multicast gather and replication
 			const uint64_t myAddress = RR->identity.address().toInt();
 			const uint64_t networkHub = (_id >> 8) & 0xffffffffffULL;
@@ -1313,13 +1442,15 @@ void Network::requestConfiguration(void *tPtr)
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS, (uint64_t)ZT_MAX_NETWORK_TAGS);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS, (uint64_t)0);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV, (uint64_t)ZT_RULES_ENGINE_REVISION);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH, ZT_TARGET_NAME);
 
 	RR->t->networkConfigRequestSent(tPtr, *this, ctrl);
 
 	if (ctrl == RR->identity.address()) {
 		if (RR->localNetworkController) {
 			RR->localNetworkController->request(_id, InetAddress(), 0xffffffffffffffffULL, RR->identity, rmd);
-		} else {
+		}
+		else {
 			this->setNotFound(tPtr);
 		}
 		return;
@@ -1333,7 +1464,8 @@ void Network::requestConfiguration(void *tPtr)
 	if (_config) {
 		outp.append((uint64_t)_config.revision);
 		outp.append((uint64_t)_config.timestamp);
-	} else {
+	}
+	else {
 		outp.append((unsigned char)0, 16);
 	}
 	outp.compress();
@@ -1364,7 +1496,9 @@ bool Network::gate(void *tPtr,const SharedPtr<Peer> &peer)
 				return true;
 			}
 		}
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 	// printf("%.16llx %.10llx not allowed, COM ts %lld revocation %lld\n", _id, peer->address().toInt(), comTimestamp, comRevocationThreshold); fflush(stdout);
 
 	return false;
@@ -1404,7 +1538,8 @@ void Network::clean()
 		while (i.next(a, m)) {
 			if (! RR->topology->getPeerNoCache(*a)) {
 				_memberships.erase(*a);
-			} else {
+			}
+			else {
 				m->clean(now, _config);
 			}
 		}
@@ -1533,7 +1668,8 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	ec->mac = _mac.toInt();
 	if (_config) {
 		Utils::scopy(ec->name, sizeof(ec->name), _config.name);
-	} else {
+	}
+	else {
 		ec->name[0] = (char)0;
 	}
 	ec->status = _status();
@@ -1551,7 +1687,8 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 		if (i < _config.staticIpCount) {
 			memcpy(&(ec->assignedAddresses[i]), &(_config.staticIps[i]), sizeof(struct sockaddr_storage));
 			++ec->assignedAddressCount;
-		} else {
+		}
+		else {
 			memset(&(ec->assignedAddresses[i]), 0, sizeof(struct sockaddr_storage));
 		}
 	}
@@ -1561,7 +1698,8 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 		if (i < _config.routeCount) {
 			memcpy(&(ec->routes[i]), &(_config.routes[i]), sizeof(ZT_VirtualNetworkRoute));
 			++ec->routeCount;
-		} else {
+		}
+		else {
 			memset(&(ec->routes[i]), 0, sizeof(ZT_VirtualNetworkRoute));
 		}
 	}
@@ -1594,7 +1732,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu
 	std::vector<MulticastGroup> groups;
 	if (newMulticastGroup) {
 		groups.push_back(*newMulticastGroup);
-	} else {
+	}
+	else {
 		groups = _allMulticastGroups();
 	}
 
@@ -1713,7 +1852,8 @@ void Network::setAuthenticationRequired(void *tPtr, const char* issuerURL, const
 	_sendUpdateEvent(tPtr);
 }
 
-void Network::_sendUpdateEvent(void *tPtr) {
+void Network::_sendUpdateEvent(void* tPtr)
+{
 	ZT_VirtualNetworkConfig ctmp;
 	_externalConfig(&ctmp);
 	RR->node->configureVirtualNetworkPort(tPtr, _id, &_uPtr, (_portInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP, &ctmp);

node/Network.hpp

@@ -14,30 +14,28 @@
 #ifndef ZT_NETWORK_HPP
 #define ZT_NETWORK_HPP
 
-#include <stdint.h>
-
 #include "../include/ZeroTierOne.h"
-
-#include <string>
-#include <map>
-#include <vector>
-#include <algorithm>
-#include <stdexcept>
-
-#include "Constants.hpp"
-#include "Hashtable.hpp"
 #include "Address.hpp"
-#include "Mutex.hpp"
-#include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
-#include "MulticastGroup.hpp"
-#include "MAC.hpp"
-#include "Dictionary.hpp"
-#include "Multicaster.hpp"
-#include "Membership.hpp"
-#include "NetworkConfig.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Constants.hpp"
+#include "Dictionary.hpp"
+#include "Hashtable.hpp"
+#include "MAC.hpp"
+#include "Membership.hpp"
 #include "Metrics.hpp"
+#include "MulticastGroup.hpp"
+#include "Multicaster.hpp"
+#include "Mutex.hpp"
+#include "NetworkConfig.hpp"
+#include "SharedPtr.hpp"
+
+#include <algorithm>
+#include <map>
+#include <stdexcept>
+#include <stdint.h>
+#include <string>
+#include <vector>
 
 #define ZT_NETWORK_MAX_INCOMING_UPDATES 3
 #define ZT_NETWORK_MAX_UPDATE_CHUNKS	((ZT_NETWORKCONFIG_DICT_CAPACITY / 1024) + 1)
@@ -50,8 +48,7 @@ class Peer;
 /**
  * A virtual LAN
  */
-class Network
+class Network {
-{
 	friend class SharedPtr<Network>;
 
   public:
@@ -63,7 +60,10 @@ public:
 	/**
 	 * Compute primary controller device ID from network ID
 	 */
-	static inline Address controllerFor(uint64_t nwid) { return Address(nwid >> 24); }
+	static inline Address controllerFor(uint64_t nwid)
+	{
+		return Address(nwid >> 24);
+	}
 
 	/**
 	 * Construct a new network
@@ -81,14 +81,39 @@ public:
 
 	~Network();
 
-	inline uint64_t id() const { return _id; }
+	inline uint64_t id() const
-	inline Address controller() const { return Address(_id >> 24); }
+	{
-	inline bool multicastEnabled() const { return (_config.multicastLimit > 0); }
+		return _id;
-	inline bool hasConfig() const { return (_config); }
+	}
-	inline uint64_t lastConfigUpdate() const { return _lastConfigUpdate; }
+	inline Address controller() const
-	inline ZT_VirtualNetworkStatus status() const { Mutex::Lock _l(_lock); return _status(); }
+	{
-	inline const NetworkConfig &config() const { return _config; }
+		return Address(_id >> 24);
-	inline const MAC &mac() const { return _mac; }
+	}
+	inline bool multicastEnabled() const
+	{
+		return (_config.multicastLimit > 0);
+	}
+	inline bool hasConfig() const
+	{
+		return (_config);
+	}
+	inline uint64_t lastConfigUpdate() const
+	{
+		return _lastConfigUpdate;
+	}
+	inline ZT_VirtualNetworkStatus status() const
+	{
+		Mutex::Lock _l(_lock);
+		return _status();
+	}
+	inline const NetworkConfig& config() const
+	{
+		return _config;
+	}
+	inline const MAC& mac() const
+	{
+		return _mac;
+	}
 
 	/**
 	 * Apply filters to an outgoing packet
@@ -304,7 +329,10 @@ public:
 	/**
 	 * @return True if QoS is in effect for this network
 	 */
-	inline bool qosEnabled() { return false; }
+	inline bool qosEnabled()
+	{
+		return false;
+	}
 
 	/**
 	 * Set a bridge route
@@ -425,7 +453,10 @@ public:
 	/**
 	 * @return Externally usable pointer-to-pointer exported via the core API
 	 */
-	inline void **userPtr() { return &_uPtr; }
+	inline void** userPtr()
+	{
+		return &_uPtr;
+	}
 
   private:
 	ZT_VirtualNetworkStatus _status() const;
@@ -452,9 +483,11 @@ private:
 	NetworkConfig _config;
 	int64_t _lastConfigUpdate;
 
-	struct _IncomingConfigChunk
+	struct _IncomingConfigChunk {
+		_IncomingConfigChunk()
 		{
-		_IncomingConfigChunk() { memset(this,0,sizeof(_IncomingConfigChunk)); }
+			memset(this, 0, sizeof(_IncomingConfigChunk));
+		}
 		uint64_t ts;
 		uint64_t updateId;
 		uint64_t haveChunkIds[ZT_NETWORK_MAX_UPDATE_CHUNKS];
@@ -466,13 +499,7 @@ private:
 
 	bool _destroyed;
 
-	enum {
+	enum { NETCONF_FAILURE_NONE, NETCONF_FAILURE_ACCESS_DENIED, NETCONF_FAILURE_NOT_FOUND, NETCONF_FAILURE_INIT_FAILED, NETCONF_FAILURE_AUTHENTICATION_REQUIRED } _netconfFailure;
-		NETCONF_FAILURE_NONE,
-		NETCONF_FAILURE_ACCESS_DENIED,
-		NETCONF_FAILURE_NOT_FOUND,
-		NETCONF_FAILURE_INIT_FAILED,
-		NETCONF_FAILURE_AUTHENTICATION_REQUIRED
-	} _netconfFailure;
 	int _portError;	  // return value from port config callback
 	std::string _authenticationURL;
 

node/NetworkConfig.cpp

@@ -11,11 +11,10 @@
  */
 /****/
 
-#include <stdint.h>
+#include "NetworkConfig.hpp"
 
 #include <algorithm>
-
+#include <stdint.h>
-#include "NetworkConfig.hpp"
 
 namespace ZeroTier {
 
@@ -129,7 +128,8 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 				ZT_VirtualNetworkRuleType rt = (ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f);
 				if (rt == ZT_NETWORK_RULE_MATCH_ETHERTYPE) {
 					et = rules[i].v.etherType;
-				} else if (rt == ZT_NETWORK_RULE_ACTION_ACCEPT) {
+				}
+				else if (rt == ZT_NETWORK_RULE_ACTION_ACCEPT) {
 					if (((int)lastrt < 32) || (lastrt == ZT_NETWORK_RULE_MATCH_ETHERTYPE)) {
 						if (ets.length() > 0) {
 							ets.push_back(',');
@@ -280,7 +280,8 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 					return false;
 				}
 			}
-		} else if(this->ssoVersion == 1) {
+		}
+		else if (this->ssoVersion == 1) {
 			if (! d.add(ZT_NETWORKCONFIG_DICT_KEY_SSO_VERSION, this->ssoVersion)) {
 				return false;
 			}
@@ -309,7 +310,8 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 		}
 
 		delete tmp;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		delete tmp;
 		throw;
 	}
@@ -347,7 +349,8 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 		this->mtu = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MTU, ZT_DEFAULT_MTU);
 		if (this->mtu < 1280) {
 			this->mtu = 1280;	// minimum MTU allowed by IPv6 standard and others
-		} else if (this->mtu > ZT_MAX_MTU) {
+		}
+		else if (this->mtu > ZT_MAX_MTU) {
 			this->mtu = ZT_MAX_MTU;
 		}
 
@@ -405,7 +408,8 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 					}
 					this->rules[this->ruleCount++].t = (uint8_t)ZT_NETWORK_RULE_ACTION_ACCEPT;
 				}
-			} else {
+			}
+			else {
 				this->rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT;
 				this->ruleCount = 1;
 			}
@@ -420,7 +424,8 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 			delete tmp;
 			return false;
 #endif	 // ZT_SUPPORT_OLD_STYLE_NETCONF
-		} else {
+		}
+		else {
 			// Otherwise we can use the new fields
 			this->flags = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_FLAGS, 0);
 			this->type = (ZT_VirtualNetworkType)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TYPE, (uint64_t)ZT_NETWORK_TYPE_PRIVATE);
@@ -437,7 +442,9 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 						p += cap.deserialize(*tmp, p);
 						this->capabilities[this->capabilityCount++] = cap;
 					}
-				} catch ( ... ) {}
+				}
+				catch (...) {
+				}
 				std::sort(&(this->capabilities[0]), &(this->capabilities[this->capabilityCount]));
 			}
 
@@ -449,7 +456,9 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 						p += tag.deserialize(*tmp, p);
 						this->tags[this->tagCount++] = tag;
 					}
-				} catch ( ... ) {}
+				}
+				catch (...) {
+				}
 				std::sort(&(this->tags[0]), &(this->tags[this->tagCount]));
 			}
 
@@ -458,7 +467,8 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 				while (p < tmp->size()) {
 					if (certificateOfOwnershipCount < ZT_MAX_CERTIFICATES_OF_OWNERSHIP) {
 						p += certificatesOfOwnership[certificateOfOwnershipCount++].deserialize(*tmp, p);
-					} else {
+					}
+					else {
 						CertificateOfOwnership foo;
 						p += foo.deserialize(*tmp, p);
 					}
@@ -514,15 +524,18 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 				if (this->ssoEnabled) {
 					if (d.get(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL, (unsigned int)sizeof(this->authenticationURL)) > 0) {
 						this->authenticationURL[sizeof(this->authenticationURL) - 1] = 0;	// ensure null terminated
-					} else {
+					}
+					else {
 						this->authenticationURL[0] = 0;
 					}
 					this->authenticationExpiryTime = d.getI(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME, 0);
-				} else {
+				}
+				else {
 					this->authenticationURL[0] = 0;
 					this->authenticationExpiryTime = 0;
 				}
-			} else if (this->ssoVersion == 1) {
+			}
+			else if (this->ssoVersion == 1) {
 				// full flow
 				if (this->ssoEnabled) {
 					if (d.get(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL, (unsigned int)sizeof(this->authenticationURL)) > 0) {
@@ -545,11 +558,13 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 					}
 					if (d.get(ZT_NETWORKCONFIG_DICT_KEY_SSO_PROVIDER, this->ssoProvider, (unsigned int)(sizeof(this->ssoProvider))) > 0) {
 						this->ssoProvider[sizeof(this->ssoProvider) - 1] = 0;
-					} else {
+					}
+					else {
 						strncpy(this->ssoProvider, "default", sizeof(this->ssoProvider));
 						this->ssoProvider[sizeof(this->ssoProvider) - 1] = 0;
 					}
-				} else {
+				}
+				else {
 					this->authenticationURL[0] = 0;
 					this->authenticationExpiryTime = 0;
 					this->centralAuthURL[0] = 0;
@@ -568,7 +583,8 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 
 		delete tmp;
 		return true;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		delete tmp;
 		return false;
 	}

node/NetworkConfig.hpp

@@ -14,31 +14,29 @@
 #ifndef ZT_NETWORKCONFIG_HPP
 #define ZT_NETWORKCONFIG_HPP
 
-#include <stdint.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include <vector>
-#include <stdexcept>
-#include <algorithm>
-
 #include "../include/ZeroTierOne.h"
-
-#include "Constants.hpp"
-#include "Buffer.hpp"
-#include "DNS.hpp"
-#include "InetAddress.hpp"
-#include "MulticastGroup.hpp"
 #include "Address.hpp"
+#include "Buffer.hpp"
+#include "Capability.hpp"
 #include "CertificateOfMembership.hpp"
 #include "CertificateOfOwnership.hpp"
-#include "Capability.hpp"
+#include "Constants.hpp"
-#include "Tag.hpp"
+#include "DNS.hpp"
 #include "Dictionary.hpp"
 #include "Hashtable.hpp"
 #include "Identity.hpp"
-#include "Utils.hpp"
+#include "InetAddress.hpp"
+#include "MulticastGroup.hpp"
+#include "Tag.hpp"
 #include "Trace.hpp"
+#include "Utils.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <vector>
 
 /**
  * Default time delta for COMs, tags, and capabilities
@@ -93,7 +91,9 @@
 namespace ZeroTier {
 
 // Dictionary capacity needed for max size network config
-#define ZT_NETWORKCONFIG_DICT_CAPACITY (4096 + (sizeof(ZT_VirtualNetworkConfig)) + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
+#define ZT_NETWORKCONFIG_DICT_CAPACITY                                                                                                                                                                                                         \
+	(4096 + (sizeof(ZT_VirtualNetworkConfig)) + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS)                                              \
+	 + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
 
 // Dictionary capacity needed for max size network meta-data
 #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024
@@ -105,6 +105,8 @@ namespace ZeroTier {
 
 // Network config version
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v"
+// Network config version
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH "o"
 // Protocol version (see Packet.hpp)
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv"
 // Software vendor
@@ -246,40 +248,39 @@ namespace ZeroTier {
  * This is a memcpy()'able structure and is safe (in a crash sense) to modify
  * without locks.
  */
-class NetworkConfig
+class NetworkConfig {
-{
   public:
-	NetworkConfig() :
+	NetworkConfig()
-		networkId(0),
+		: networkId(0)
-		timestamp(0),
+		, timestamp(0)
-		credentialTimeMaxDelta(0),
+		, credentialTimeMaxDelta(0)
-		revision(0),
+		, revision(0)
-		issuedTo(),
+		, issuedTo()
-		remoteTraceTarget(),
+		, remoteTraceTarget()
-		flags(0),
+		, flags(0)
-		remoteTraceLevel(Trace::LEVEL_NORMAL),
+		, remoteTraceLevel(Trace::LEVEL_NORMAL)
-		mtu(0),
+		, mtu(0)
-		multicastLimit(0),
+		, multicastLimit(0)
-		specialistCount(0),
+		, specialistCount(0)
-		routeCount(0),
+		, routeCount(0)
-		staticIpCount(0),
+		, staticIpCount(0)
-		ruleCount(0),
+		, ruleCount(0)
-		capabilityCount(0),
+		, capabilityCount(0)
-		tagCount(0),
+		, tagCount(0)
-		certificateOfOwnershipCount(0),
+		, certificateOfOwnershipCount(0)
-		capabilities(),
+		, capabilities()
-		tags(),
+		, tags()
-		certificatesOfOwnership(),
+		, certificatesOfOwnership()
-		type(ZT_NETWORK_TYPE_PRIVATE),
+		, type(ZT_NETWORK_TYPE_PRIVATE)
-		dnsCount(0),
+		, dnsCount(0)
-		ssoEnabled(false),
+		, ssoEnabled(false)
-		authenticationURL(),
+		, authenticationURL()
-		authenticationExpiryTime(0),
+		, authenticationExpiryTime(0)
-		issuerURL(),
+		, issuerURL()
-		centralAuthURL(),
+		, centralAuthURL()
-		ssoNonce(),
+		, ssoNonce()
-		ssoState(),
+		, ssoState()
-		ssoClientID()
+		, ssoClientID()
 	{
 		name[0] = 0;
 		memset(specialists, 0, sizeof(uint64_t) * ZT_MAX_NETWORK_SPECIALISTS);
@@ -316,12 +317,18 @@ public:
 	/**
 	 * @return True if broadcast (ff:ff:ff:ff:ff:ff) address should work on this network
 	 */
-	inline bool enableBroadcast() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0); }
+	inline bool enableBroadcast() const
+	{
+		return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0);
+	}
 
 	/**
 	 * @return True if IPv6 NDP emulation should be allowed for certain "magic" IPv6 address patterns
 	 */
-	inline bool ndpEmulation() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); }
+	inline bool ndpEmulation() const
+	{
+		return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0);
+	}
 
 	/**
 	 * @return True if frames should not be compressed
@@ -342,12 +349,18 @@ public:
 	/**
 	 * @return Network type is public (no access control)
 	 */
-	inline bool isPublic() const { return (this->type == ZT_NETWORK_TYPE_PUBLIC); }
+	inline bool isPublic() const
+	{
+		return (this->type == ZT_NETWORK_TYPE_PUBLIC);
+	}
 
 	/**
 	 * @return Network type is private (certificate access control)
 	 */
-	inline bool isPrivate() const { return (this->type == ZT_NETWORK_TYPE_PRIVATE); }
+	inline bool isPrivate() const
+	{
+		return (this->type == ZT_NETWORK_TYPE_PRIVATE);
+	}
 
 	/**
 	 * @return ZeroTier addresses of devices on this network designated as active bridges
@@ -472,9 +485,18 @@ public:
 		return false;
 	}
 
-	inline operator bool() const { return (networkId != 0); }
+	inline operator bool() const
-	inline bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); }
+	{
-	inline bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); }
+		return (networkId != 0);
+	}
+	inline bool operator==(const NetworkConfig& nc) const
+	{
+		return (memcmp(this, &nc, sizeof(NetworkConfig)) == 0);
+	}
+	inline bool operator!=(const NetworkConfig& nc) const
+	{
+		return (! (*this == nc));
+	}
 
 	/**
 	 * Add a specialist or mask flags if already present

node/NetworkController.hpp

@@ -14,13 +14,13 @@
 #ifndef ZT_NETWORKCONFIGMASTER_HPP
 #define ZT_NETWORKCONFIGMASTER_HPP
 
-#include <stdint.h>
+#include "Address.hpp"
-
 #include "Constants.hpp"
 #include "Dictionary.hpp"
 #include "NetworkConfig.hpp"
 #include "Revocation.hpp"
-#include "Address.hpp"
+
+#include <stdint.h>
 
 namespace ZeroTier {
 
@@ -30,23 +30,14 @@ struct InetAddress;
 /**
  * Interface for network controller implementations
  */
-class NetworkController
+class NetworkController {
-{
   public:
-	enum ErrorCode
+	enum ErrorCode { NC_ERROR_NONE = 0, NC_ERROR_OBJECT_NOT_FOUND = 1, NC_ERROR_ACCESS_DENIED = 2, NC_ERROR_INTERNAL_SERVER_ERROR = 3, NC_ERROR_AUTHENTICATION_REQUIRED = 4 };
-	{
-		NC_ERROR_NONE = 0,
-		NC_ERROR_OBJECT_NOT_FOUND = 1,
-		NC_ERROR_ACCESS_DENIED = 2,
-		NC_ERROR_INTERNAL_SERVER_ERROR = 3,
-		NC_ERROR_AUTHENTICATION_REQUIRED = 4
-	};
 
 	/**
 	 * Interface for sender used to send pushes and replies
 	 */
-	class Sender
+	class Sender {
-	{
 	  public:
 		/**
 		 * Send a configuration to a remote peer
@@ -83,8 +74,12 @@ public:
 		virtual void ncSendError(uint64_t nwid, uint64_t requestPacketId, const Address& destination, NetworkController::ErrorCode errorCode, const void* errorData, unsigned int errorDataSize) = 0;
 	};
 
-	NetworkController() {}
+	NetworkController()
-	virtual ~NetworkController() {}
+	{
+	}
+	virtual ~NetworkController()
+	{
+	}
 
 	/**
 	 * Called when this is added to a Node to initialize and supply info
@@ -104,12 +99,7 @@ public:
 	 * @param metaData Meta-data bundled with request (if any)
 	 * @return Returns NETCONF_QUERY_OK if result 'nc' is valid, or an error code on error
 	 */
-	virtual void request(
+	virtual void request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData) = 0;
-		uint64_t nwid,
-		const InetAddress &fromAddr,
-		uint64_t requestPacketId,
-		const Identity &identity,
-		const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData) = 0;
 };
 
 }	// namespace ZeroTier

node/Node.cpp

@@ -11,30 +11,31 @@
  */
 /****/
 
-#include <stdio.h>
+#include "Node.hpp"
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <stdint.h>
 
 #include "../version.h"
-
-#include "Constants.hpp"
-#include "SharedPtr.hpp"
-#include "Node.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "NetworkController.hpp"
-#include "Switch.hpp"
-#include "Multicaster.hpp"
-#include "Topology.hpp"
-#include "Buffer.hpp"
-#include "Packet.hpp"
 #include "Address.hpp"
+#include "Buffer.hpp"
+#include "Constants.hpp"
 #include "Identity.hpp"
-#include "SelfAwareness.hpp"
-#include "Network.hpp"
-#include "Trace.hpp"
 #include "Metrics.hpp"
+#include "Multicaster.hpp"
+#include "Network.hpp"
+#include "NetworkController.hpp"
+#include "Packet.hpp"
+#include "PacketMultiplexer.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "SelfAwareness.hpp"
+#include "SharedPtr.hpp"
+#include "Switch.hpp"
+#include "Topology.hpp"
+#include "Trace.hpp"
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 // FIXME: remove this suppression and actually fix warnings
 #ifdef __GNUC__
@@ -47,17 +48,17 @@ namespace ZeroTier {
 /* Public Node interface (C++, exposed via CAPI bindings)                   */
 /****************************************************************************/
 
-Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now) :
+Node::Node(void* uptr, void* tptr, const struct ZT_Node_Callbacks* callbacks, int64_t now)
-	_RR(this),
+	: _RR(this)
-	RR(&_RR),
+	, RR(&_RR)
-	_uPtr(uptr),
+	, _uPtr(uptr)
-	_networks(8),
+	, _networks(8)
-	_now(now),
+	, _now(now)
-	_lastPingCheck(0),
+	, _lastPingCheck(0)
-	_lastGratuitousPingCheck(0),
+	, _lastGratuitousPingCheck(0)
-	_lastHousekeepingRun(0),
+	, _lastHousekeepingRun(0)
-	_lastMemoizedTraceSettings(0),
+	, _lastMemoizedTraceSettings(0)
-	_lowBandwidthMode(false)
+	, _lowBandwidthMode(false)
 {
 	if (callbacks->version != 0) {
 		throw ZT_EXCEPTION_INVALID_ARGUMENT;
@@ -84,7 +85,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		if (RR->identity.fromString(tmp)) {
 			RR->identity.toString(false, RR->publicIdentityStr);
 			RR->identity.toString(true, RR->secretIdentityStr);
-		} else {
+		}
+		else {
 			throw ZT_EXCEPTION_INVALID_IDENTITY;
 		}
 
@@ -101,7 +103,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		idtmp[1] = 0;
 		stateObjectPut(tptr, ZT_STATE_OBJECT_IDENTITY_SECRET, idtmp, RR->secretIdentityStr, (unsigned int)strlen(RR->secretIdentityStr));
 		stateObjectPut(tptr, ZT_STATE_OBJECT_IDENTITY_PUBLIC, idtmp, RR->publicIdentityStr, (unsigned int)strlen(RR->publicIdentityStr));
-	} else {
+	}
+	else {
 		idtmp[0] = RR->identity.address().toInt();
 		idtmp[1] = 0;
 		n = stateObjectGet(tptr, ZT_STATE_OBJECT_IDENTITY_PUBLIC, idtmp, tmp, sizeof(tmp) - 1);
@@ -119,9 +122,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		const unsigned long mcs = sizeof(Multicaster) + (((sizeof(Multicaster) & 0xf) != 0) ? (16 - (sizeof(Multicaster) & 0xf)) : 0);
 		const unsigned long topologys = sizeof(Topology) + (((sizeof(Topology) & 0xf) != 0) ? (16 - (sizeof(Topology) & 0xf)) : 0);
 		const unsigned long sas = sizeof(SelfAwareness) + (((sizeof(SelfAwareness) & 0xf) != 0) ? (16 - (sizeof(SelfAwareness) & 0xf)) : 0);
-		const unsigned long bc = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
+		const unsigned long bcs = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
+		const unsigned long pms = sizeof(PacketMultiplexer) + (((sizeof(PacketMultiplexer) & 0xf) != 0) ? (16 - (sizeof(PacketMultiplexer) & 0xf)) : 0);
 
-		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas + bc));
+		m = reinterpret_cast<char*>(::malloc(16 + ts + sws + mcs + topologys + sas + bcs + pms));
 		if (! m) {
 			throw std::bad_alloc();
 		}
@@ -141,7 +145,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		RR->sa = new (m) SelfAwareness(RR);
 		m += sas;
 		RR->bc = new (m) Bond(RR);
-	} catch ( ... ) {
+		m += bcs;
+		RR->pm = new (m) PacketMultiplexer(RR);
+	}
+	catch (...) {
 		if (RR->sa) {
 			RR->sa->~SelfAwareness();
 		}
@@ -160,6 +167,9 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		if (RR->bc) {
 			RR->bc->~Bond();
 		}
+		if (RR->pm) {
+			RR->pm->~PacketMultiplexer();
+		}
 		::free(m);
 		throw;
 	}
@@ -191,17 +201,13 @@ Node::~Node()
 	if (RR->bc) {
 		RR->bc->~Bond();
 	}
+	if (RR->pm) {
+		RR->pm->~PacketMultiplexer();
+	}
 	::free(RR->rtmem);
 }
 
-ZT_ResultCode Node::processWirePacket(
+ZT_ResultCode Node::processWirePacket(void* tptr, int64_t now, int64_t localSocket, const struct sockaddr_storage* remoteAddress, const void* packetData, unsigned int packetLength, volatile int64_t* nextBackgroundTaskDeadline)
-	void *tptr,
-	int64_t now,
-	int64_t localSocket,
-	const struct sockaddr_storage *remoteAddress,
-	const void *packetData,
-	unsigned int packetLength,
-	volatile int64_t *nextBackgroundTaskDeadline)
 {
 	_now = now;
 	RR->sw->onRemotePacket(tptr, localSocket, *(reinterpret_cast<const InetAddress*>(remoteAddress)), packetData, packetLength);
@@ -225,21 +231,26 @@ ZT_ResultCode Node::processVirtualNetworkFrame(
 	if (nw) {
 		RR->sw->onLocalEthernet(tptr, nw, MAC(sourceMac), MAC(destMac), etherType, vlanId, frameData, frameLength);
 		return ZT_RESULT_OK;
-	} else {
+	}
+	else {
 		return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 	}
 }
 
-// Closure used to ping upstream and active/online peers
+void Node::initMultithreading(unsigned int concurrency, bool cpuPinningEnabled)
-class _PingPeersThatNeedPing
 {
+	RR->pm->setUpPostDecodeReceiveThreads(concurrency, cpuPinningEnabled);
+}
+
+// Closure used to ping upstream and active/online peers
+class _PingPeersThatNeedPing {
   public:
-	_PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector<InetAddress> > &alwaysContact,int64_t now) :
+	_PingPeersThatNeedPing(const RuntimeEnvironment* renv, void* tPtr, Hashtable<Address, std::vector<InetAddress> >& alwaysContact, int64_t now)
-		RR(renv),
+		: RR(renv)
-		_tPtr(tPtr),
+		, _tPtr(tPtr)
-		_alwaysContact(alwaysContact),
+		, _alwaysContact(alwaysContact)
-		_now(now),
+		, _now(now)
-		_bestCurrentUpstream(RR->topology->getUpstreamPeer())
+		, _bestCurrentUpstream(RR->topology->getUpstreamPeer())
 	{
 	}
 
@@ -247,7 +258,6 @@ public:
 	{
 		const std::vector<InetAddress>* const alwaysContactEndpoints = _alwaysContact.get(p->address());
 		if (alwaysContactEndpoints) {
-
 			ZT_PeerRole role = RR->topology->role(p->address());
 
 			// Contact upstream peers as infrequently as possible
@@ -294,7 +304,8 @@ public:
 			}
 
 			_alwaysContact.erase(p->address());	  // after this we'll WHOIS all upstreams that remain
-		} else if (p->isActive(_now)) {
+		}
+		else if (p->isActive(_now)) {
 			p->doPingAndKeepalive(_tPtr, _now);
 		}
 	}
@@ -414,10 +425,12 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 			if (oldOnline != _online) {
 				postEvent(tptr, _online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 			}
-		} catch ( ... ) {
+		}
+		catch (...) {
 			return ZT_RESULT_FATAL_ERROR_INTERNAL;
 		}
-	} else {
+	}
+	else {
 		timeUntilNextPingCheck -= (unsigned long)timeSinceLastPingCheck;
 	}
 
@@ -432,14 +445,16 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 			RR->topology->doPeriodicTasks(tptr, now);
 			RR->sa->clean(now);
 			RR->mc->clean(now);
-		} catch ( ... ) {
+		}
+		catch (...) {
 			return ZT_RESULT_FATAL_ERROR_INTERNAL;
 		}
 	}
 
 	try {
 		*nextBackgroundTaskDeadline = now + (int64_t)std::max(std::min(bondCheckInterval, std::min(timeUntilNextPingCheck, RR->sw->doTimerTasks(tptr, now))), (unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 
@@ -498,7 +513,8 @@ ZT_ResultCode Node::multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multica
 	if (nw) {
 		nw->multicastSubscribe(tptr, MulticastGroup(MAC(multicastGroup), (uint32_t)(multicastAdi & 0xffffffff)));
 		return ZT_RESULT_OK;
-	} else {
+	}
+	else {
 		return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 	}
 }
@@ -509,7 +525,8 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u
 	if (nw) {
 		nw->multicastUnsubscribe(MulticastGroup(MAC(multicastGroup), (uint32_t)(multicastAdi & 0xffffffff)));
 		return ZT_RESULT_OK;
-	} else {
+	}
+	else {
 		return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 	}
 }
@@ -560,7 +577,8 @@ ZT_PeerList *Node::peers() const
 			p->versionMajor = pi->second->remoteVersionMajor();
 			p->versionMinor = pi->second->remoteVersionMinor();
 			p->versionRev = pi->second->remoteVersionRevision();
-		} else {
+		}
+		else {
 			p->versionMajor = -1;
 			p->versionMinor = -1;
 			p->versionRev = -1;
@@ -683,7 +701,9 @@ int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *d
 			RR->sw->send(tptr, outp, true);
 			return 1;
 		}
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 	return 0;
 }
 
@@ -768,7 +788,8 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de
 			return;
 		}
 		n->setConfiguration((void*)0, nc, true);
-	} else {
+	}
+	else {
 		Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>* dconf = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 		try {
 			if (nc.toDictionary(*dconf, sendLegacyFormatConfig)) {
@@ -808,7 +829,8 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de
 				}
 			}
 			delete dconf;
-		} catch ( ... ) {
+		}
+		catch (...) {
 			delete dconf;
 			throw;
 		}
@@ -823,7 +845,8 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev)
 			return;
 		}
 		n->addCredential((void*)0, RR->identity.address(), rev);
-	} else {
+	}
+	else {
 		Packet outp(destination, RR->identity.address(), Packet::VERB_NETWORK_CREDENTIALS);
 		outp.append((uint8_t)0x00);
 		outp.append((uint16_t)0);
@@ -858,7 +881,8 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			default:
 				break;
 		}
-	} else if (requestPacketId) {
+	}
+	else if (requestPacketId) {
 		Packet outp(destination, RR->identity.address(), Packet::VERB_ERROR);
 		outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
 		outp.append(requestPacketId);
@@ -904,11 +928,14 @@ enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct
 	try {
 		*node = reinterpret_cast<ZT_Node*>(new ZeroTier::Node(uptr, tptr, callbacks, now));
 		return ZT_RESULT_OK;
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch (std::runtime_error &exc) {
+	}
+	catch (std::runtime_error& exc) {
 		return ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -917,24 +944,21 @@ void ZT_Node_delete(ZT_Node *node)
 {
 	try {
 		delete (reinterpret_cast<ZeroTier::Node*>(node));
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
-enum ZT_ResultCode ZT_Node_processWirePacket(
+enum ZT_ResultCode
-	ZT_Node *node,
+ZT_Node_processWirePacket(ZT_Node* node, void* tptr, int64_t now, int64_t localSocket, const struct sockaddr_storage* remoteAddress, const void* packetData, unsigned int packetLength, volatile int64_t* nextBackgroundTaskDeadline)
-	void *tptr,
-	int64_t now,
-	int64_t localSocket,
-	const struct sockaddr_storage *remoteAddress,
-	const void *packetData,
-	unsigned int packetLength,
-	volatile int64_t *nextBackgroundTaskDeadline)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->processWirePacket(tptr, now, localSocket, remoteAddress, packetData, packetLength, nextBackgroundTaskDeadline);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_OK;   // "OK" since invalid packets are simply dropped, but the system is still up
 	}
 }
@@ -954,9 +978,11 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame(
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->processVirtualNetworkFrame(tptr, now, nwid, sourceMac, destMac, etherType, vlanId, frameData, frameLength, nextBackgroundTaskDeadline);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -965,9 +991,11 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->processBackgroundTasks(tptr, now, nextBackgroundTaskDeadline);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -976,9 +1004,11 @@ enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr,void *tpt
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->join(nwid, uptr, tptr);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -987,9 +1017,11 @@ enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr,void *t
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->leave(nwid, uptr, tptr);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -998,9 +1030,11 @@ enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tptr,uint64_t
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->multicastSubscribe(tptr, nwid, multicastGroup, multicastAdi);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -1009,9 +1043,11 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->multicastUnsubscribe(nwid, multicastGroup, multicastAdi);
-	} catch (std::bad_alloc &exc) {
+	}
+	catch (std::bad_alloc& exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -1020,7 +1056,8 @@ enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,u
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->orbit(tptr, moonWorldId, moonSeed);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -1029,7 +1066,8 @@ enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->deorbit(tptr, moonWorldId);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
@@ -1043,14 +1081,17 @@ void ZT_Node_status(ZT_Node *node,ZT_NodeStatus *status)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node*>(node)->status(status);
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
 ZT_PeerList* ZT_Node_peers(ZT_Node* node)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->peers();
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return (ZT_PeerList*)0;
 	}
 }
@@ -1059,7 +1100,8 @@ ZT_VirtualNetworkConfig *ZT_Node_networkConfig(ZT_Node *node,uint64_t nwid)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->networkConfig(nwid);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return (ZT_VirtualNetworkConfig*)0;
 	}
 }
@@ -1068,7 +1110,8 @@ ZT_VirtualNetworkList *ZT_Node_networks(ZT_Node *node)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->networks();
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return (ZT_VirtualNetworkList*)0;
 	}
 }
@@ -1077,14 +1120,17 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node*>(node)->freeQueryResult(qr);
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
 int ZT_Node_addLocalInterfaceAddress(ZT_Node* node, const struct sockaddr_storage* addr)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->addLocalInterfaceAddress(addr);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return 0;
 	}
 }
@@ -1093,14 +1139,17 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node*>(node)->clearLocalInterfaceAddresses();
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
 int ZT_Node_sendUserMessage(ZT_Node* node, void* tptr, uint64_t dest, uint64_t typeId, const void* data, unsigned int len)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->sendUserMessage(tptr, dest, typeId, data, len);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return 0;
 	}
 }
@@ -1109,14 +1158,17 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node*>(node)->setNetconfMaster(networkControllerInstance);
-	} catch ( ... ) {}
+	}
+	catch (...) {
+	}
 }
 
 enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node* node, const struct sockaddr_storage* pathNetwork, const ZT_PhysicalPathConfiguration* pathConfig)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node*>(node)->setPhysicalPathConfiguration(pathNetwork, pathConfig);
-	} catch ( ... ) {
+	}
+	catch (...) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }

node/Node.hpp

@@ -14,29 +14,26 @@
 #ifndef ZT_NODE_HPP
 #define ZT_NODE_HPP
 
+#include "../include/ZeroTierOne.h"
+#include "Bond.hpp"
+#include "Constants.hpp"
+#include "Hashtable.hpp"
+#include "InetAddress.hpp"
+#include "MAC.hpp"
+#include "Mutex.hpp"
+#include "Network.hpp"
+#include "NetworkController.hpp"
+#include "Path.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Salsa20.hpp"
+#include "SelfAwareness.hpp"
+
+#include <map>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
-#include <map>
 #include <vector>
 
-#include "Constants.hpp"
-
-#include "../include/ZeroTierOne.h"
-
-#include "RuntimeEnvironment.hpp"
-#include "InetAddress.hpp"
-#include "Mutex.hpp"
-#include "MAC.hpp"
-#include "Network.hpp"
-#include "Path.hpp"
-#include "Salsa20.hpp"
-#include "NetworkController.hpp"
-#include "Hashtable.hpp"
-#include "Bond.hpp"
-#include "SelfAwareness.hpp"
-
 // Bit mask for "expecting reply" hash
 #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255
 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31
@@ -50,28 +47,26 @@ class World;
  *
  * The pointer returned by ZT_Node_new() is an instance of this class.
  */
-class Node : public NetworkController::Sender
+class Node : public NetworkController::Sender {
-{
   public:
 	Node(void* uptr, void* tptr, const struct ZT_Node_Callbacks* callbacks, int64_t now);
 	virtual ~Node();
 
 	// Get rid of alignment warnings on 32-bit Windows and possibly improve performance
 #ifdef __WINDOWS__
-	void * operator new(size_t i) { return _mm_malloc(i,16); }
+	void* operator new(size_t i)
-	void operator delete(void* p) { _mm_free(p); }
+	{
+		return _mm_malloc(i, 16);
+	}
+	void operator delete(void* p)
+	{
+		_mm_free(p);
+	}
 #endif
 
 	// Public API Functions ----------------------------------------------------
 
-	ZT_ResultCode processWirePacket(
+	ZT_ResultCode processWirePacket(void* tptr, int64_t now, int64_t localSocket, const struct sockaddr_storage* remoteAddress, const void* packetData, unsigned int packetLength, volatile int64_t* nextBackgroundTaskDeadline);
-		void *tptr,
-		int64_t now,
-		int64_t localSocket,
-		const struct sockaddr_storage *remoteAddress,
-		const void *packetData,
-		unsigned int packetLength,
-		volatile int64_t *nextBackgroundTaskDeadline);
 	ZT_ResultCode processVirtualNetworkFrame(
 		void* tptr,
 		int64_t now,
@@ -103,35 +98,19 @@ public:
 
 	// Internal functions ------------------------------------------------------
 
-	inline int64_t now() const { return _now; }
+	inline int64_t now() const
+	{
+		return _now;
+	}
 
 	inline bool putPacket(void* tPtr, const int64_t localSocket, const InetAddress& addr, const void* data, unsigned int len, unsigned int ttl = 0)
 	{
-		return (_cb.wirePacketSendFunction(
+		return (_cb.wirePacketSendFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, localSocket, reinterpret_cast<const struct sockaddr_storage*>(&addr), data, len, ttl) == 0);
-			reinterpret_cast<ZT_Node *>(this),
-			_uPtr,
-			tPtr,
-			localSocket,
-			reinterpret_cast<const struct sockaddr_storage *>(&addr),
-			data,
-			len,
-			ttl) == 0);
 	}
 
 	inline void putFrame(void* tPtr, uint64_t nwid, void** nuptr, const MAC& source, const MAC& dest, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len)
 	{
-		_cb.virtualNetworkFrameFunction(
+		_cb.virtualNetworkFrameFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, nwid, nuptr, source.toInt(), dest.toInt(), etherType, vlanId, data, len);
-			reinterpret_cast<ZT_Node *>(this),
-			_uPtr,
-			tPtr,
-			nwid,
-			nuptr,
-			source.toInt(),
-			dest.toInt(),
-			etherType,
-			vlanId,
-			data,
-			len);
 	}
 
 	inline SharedPtr<Network> network(uint64_t nwid) const
@@ -169,18 +148,39 @@ public:
 		return _directPaths;
 	}
 
-	inline void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ev,md); }
+	inline void postEvent(void* tPtr, ZT_Event ev, const void* md = (const void*)0)
+	{
+		_cb.eventCallback(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, ev, md);
+	}
 
-	inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,nwid,nuptr,op,nc); }
+	inline int configureVirtualNetworkPort(void* tPtr, uint64_t nwid, void** nuptr, ZT_VirtualNetworkConfigOperation op, const ZT_VirtualNetworkConfig* nc)
+	{
+		return _cb.virtualNetworkConfigFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, nwid, nuptr, op, nc);
+	}
 
-	inline bool online() const { return _online; }
+	inline bool online() const
+	{
+		return _online;
+	}
 
-	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
+	inline int stateObjectGet(void* const tPtr, ZT_StateObjectType type, const uint64_t id[2], void* const data, const unsigned int maxlen)
-	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
+	{
-	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
+		return _cb.stateGetFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, type, id, data, maxlen);
+	}
+	inline void stateObjectPut(void* const tPtr, ZT_StateObjectType type, const uint64_t id[2], const void* const data, const unsigned int len)
+	{
+		_cb.statePutFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, type, id, data, (int)len);
+	}
+	inline void stateObjectDelete(void* const tPtr, ZT_StateObjectType type, const uint64_t id[2])
+	{
+		_cb.statePutFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, type, id, (const void*)0, -1);
+	}
 
 	bool shouldUsePathForZeroTierTraffic(void* tPtr, const Address& ztaddr, const int64_t localSocket, const InetAddress& remoteAddress);
-	inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast<struct sockaddr_storage *>(&addr)) != 0) : false ); }
+	inline bool externalPathLookup(void* tPtr, const Address& ztaddr, int family, InetAddress& addr)
+	{
+		return ((_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast<ZT_Node*>(this), _uPtr, tPtr, ztaddr.toInt(), family, reinterpret_cast<struct sockaddr_storage*>(&addr)) != 0) : false);
+	}
 
 	uint64_t prng();
 	ZT_ResultCode setPhysicalPathConfiguration(const struct sockaddr_storage* pathNetwork, const ZT_PhysicalPathConfiguration* pathConfig);
@@ -188,11 +188,20 @@ public:
 	World planet() const;
 	std::vector<World> moons() const;
 
-	inline const Identity &identity() const { return _RR.identity; }
+	inline const Identity& identity() const
+	{
+		return _RR.identity;
+	}
 
-	inline const std::vector<InetAddress> SurfaceAddresses() const { return _RR.sa->whoami(); }
+	inline const std::vector<InetAddress> SurfaceAddresses() const
+	{
+		return _RR.sa->whoami();
+	}
 
-	inline Bond *bondController() const { return _RR.bc; }
+	inline Bond* bondController() const
+	{
+		return _RR.bc;
+	}
 
 	/**
 	 * Register that we are expecting a reply to a packet ID
@@ -253,8 +262,14 @@ public:
 	virtual void ncSendRevocation(const Address& destination, const Revocation& rev);
 	virtual void ncSendError(uint64_t nwid, uint64_t requestPacketId, const Address& destination, NetworkController::ErrorCode errorCode, const void* errorData, unsigned int errorDataSize);
 
-	inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; }
+	inline const Address& remoteTraceTarget() const
-	inline Trace::Level remoteTraceLevel() const { return _remoteTraceLevel; }
+	{
+		return _remoteTraceTarget;
+	}
+	inline Trace::Level remoteTraceLevel() const
+	{
+		return _remoteTraceLevel;
+	}
 
 	inline bool localControllerHasAuthorized(const int64_t now, const uint64_t nwid, const Address& addr) const
 	{
@@ -283,7 +298,9 @@ public:
 		return _lowBandwidthMode;
 	}
 
-private:
+	void initMultithreading(unsigned int concurrency, bool cpuPinningEnabled);
+
+  public:
 	RuntimeEnvironment _RR;
 	RuntimeEnvironment* RR;
 	void* _uPtr;   // _uptr (lower case) is reserved in Visual Studio :P
@@ -301,13 +318,23 @@ private:
 
 	// Map that remembers if we have recently sent a network config to someone
 	// querying us as a controller.
-	struct _LocalControllerAuth
+	struct _LocalControllerAuth {
-	{
 		uint64_t nwid, address;
-		_LocalControllerAuth(const uint64_t nwid_,const Address &address_) : nwid(nwid_),address(address_.toInt()) {}
+		_LocalControllerAuth(const uint64_t nwid_, const Address& address_) : nwid(nwid_), address(address_.toInt())
-		inline unsigned long hashCode() const { return (unsigned long)(nwid ^ address); }
+		{
-		inline bool operator==(const _LocalControllerAuth &a) const { return ((a.nwid == nwid)&&(a.address == address)); }
+		}
-		inline bool operator!=(const _LocalControllerAuth &a) const { return ((a.nwid != nwid)||(a.address != address)); }
+		inline unsigned long hashCode() const
+		{
+			return (unsigned long)(nwid ^ address);
+		}
+		inline bool operator==(const _LocalControllerAuth& a) const
+		{
+			return ((a.nwid == nwid) && (a.address == address));
+		}
+		inline bool operator!=(const _LocalControllerAuth& a) const
+		{
+			return ((a.nwid != nwid) || (a.address != address));
+		}
 	};
 	Hashtable<_LocalControllerAuth, int64_t> _localControllerAuthorizations;
 	Mutex _localControllerAuthorizations_m;

node/OutboundMulticast.cpp

@@ -11,13 +11,14 @@
  */
 /****/
 
-#include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
 #include "OutboundMulticast.hpp"
-#include "Switch.hpp"
+
+#include "Constants.hpp"
 #include "Network.hpp"
 #include "Node.hpp"
 #include "Peer.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Switch.hpp"
 #include "Topology.hpp"
 
 namespace ZeroTier {
@@ -42,7 +43,8 @@ void OutboundMulticast::init(
 	if (src) {
 		_macSrc = src;
 		flags |= 0x04;
-	} else {
+	}
+	else {
 		_macSrc.fromAddress(RR->identity.address(), nwid);
 	}
 	_macDest = dest.mac();

node/OutboundMulticast.hpp

@@ -14,17 +14,16 @@
 #ifndef ZT_OUTBOUNDMULTICAST_HPP
 #define ZT_OUTBOUNDMULTICAST_HPP
 
-#include <stdint.h>
+#include "Address.hpp"
-
-#include <vector>
-#include <algorithm>
-
 #include "Constants.hpp"
 #include "MAC.hpp"
 #include "MulticastGroup.hpp"
-#include "Address.hpp"
 #include "Packet.hpp"
 
+#include <algorithm>
+#include <stdint.h>
+#include <vector>
+
 namespace ZeroTier {
 
 class CertificateOfMembership;
@@ -35,15 +34,16 @@ class RuntimeEnvironment;
  *
  * This object isn't guarded by a mutex; caller must synchronize access.
  */
-class OutboundMulticast
+class OutboundMulticast {
-{
   public:
 	/**
 	 * Create an uninitialized outbound multicast
 	 *
 	 * It must be initialized with init().
 	 */
-	OutboundMulticast() {}
+	OutboundMulticast()
+	{
+	}
 
 	/**
 	 * Initialize outbound multicast
@@ -77,18 +77,27 @@ public:
 	/**
 	 * @return Multicast creation time
 	 */
-	inline uint64_t timestamp() const { return _timestamp; }
+	inline uint64_t timestamp() const
+	{
+		return _timestamp;
+	}
 
 	/**
 	 * @param now Current time
 	 * @return True if this multicast is expired (has exceeded transmit timeout)
 	 */
-	inline bool expired(int64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); }
+	inline bool expired(int64_t now) const
+	{
+		return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT);
+	}
 
 	/**
 	 * @return True if this outbound multicast has been sent to enough peers
 	 */
-	inline bool atLimit() const { return (_alreadySentTo.size() >= _limit); }
+	inline bool atLimit() const
+	{
+		return (_alreadySentTo.size() >= _limit);
+	}
 
 	/**
 	 * Just send without checking log
@@ -135,7 +144,8 @@ public:
 		if (std::find(_alreadySentTo.begin(), _alreadySentTo.end(), toAddr) == _alreadySentTo.end()) {
 			sendAndLog(RR, tPtr, toAddr);
 			return true;
-		} else {
+		}
+		else {
 			return false;
 		}
 	}

node/Packet.cpp

@@ -11,14 +11,14 @@
  */
 /****/
 
-#include <stdint.h>
-#include <stddef.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-
 #include "Packet.hpp"
 
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
 #if defined(ZT_USE_X64_ASM_SALSA2012) && defined(ZT_ARCH_X64)
 #include "../ext/x64-salsa2012-asm/salsa2012.h"
 #endif
@@ -49,10 +49,11 @@ namespace ZeroTier {
 
 // ARM (32-bit) NEON crypto (must be detected)
 #ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012
-class _FastCryptoChecker
+class _FastCryptoChecker {
-{
   public:
-	_FastCryptoChecker() : canHas(zt_arm_has_neon()) {}
+	_FastCryptoChecker() : canHas(zt_arm_has_neon())
+	{
+	}
 	bool canHas;
 };
 static const _FastCryptoChecker _ZT_FAST_CRYPTO_CHECK;
@@ -63,7 +64,9 @@ static const _FastCryptoChecker _ZT_FAST_CRYPTO_CHECK;
 // No fast crypto available
 #ifndef ZT_HAS_FAST_CRYPTO
 #define ZT_HAS_FAST_CRYPTO() (false)
-#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) {}
+#define ZT_FAST_SINGLE_PASS_SALSA2012(b, l, n, k)                                                                                                                                                                                              \
+	{                                                                                                                                                                                                                                          \
+	}
 #endif
 
 /************************************************************************** */
@@ -224,35 +227,78 @@ typedef uintptr_t reg_t;
 
 static inline unsigned LZ4_isLittleEndian(void)
 {
-	const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental */
+	const union {
+		U32 u;
+		BYTE c[4];
+	} one = { 1 }; /* don't use static : performance detrimental */
 	return one.c[0];
 }
 
 #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 2)
-static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; }
+static U16 LZ4_read16(const void* memPtr)
-static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; }
+{
-static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; }
+	return *(const U16*)memPtr;
-static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+}
-static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+static U32 LZ4_read32(const void* memPtr)
+{
+	return *(const U32*)memPtr;
+}
+static reg_t LZ4_read_ARCH(const void* memPtr)
+{
+	return *(const reg_t*)memPtr;
+}
+static void LZ4_write16(void* memPtr, U16 value)
+{
+	*(U16*)memPtr = value;
+}
+static void LZ4_write32(void* memPtr, U32 value)
+{
+	*(U32*)memPtr = value;
+}
 #elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 1)
-typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
+typedef union {
-static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
+	U16 u16;
-static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+	U32 u32;
-static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
+	reg_t uArch;
-static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+} __attribute__((packed)) unalign;
-static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
+static U16 LZ4_read16(const void* ptr)
+{
+	return ((const unalign*)ptr)->u16;
+}
+static U32 LZ4_read32(const void* ptr)
+{
+	return ((const unalign*)ptr)->u32;
+}
+static reg_t LZ4_read_ARCH(const void* ptr)
+{
+	return ((const unalign*)ptr)->uArch;
+}
+static void LZ4_write16(void* memPtr, U16 value)
+{
+	((unalign*)memPtr)->u16 = value;
+}
+static void LZ4_write32(void* memPtr, U32 value)
+{
+	((unalign*)memPtr)->u32 = value;
+}
 #else  /* safe and portable access through memcpy() */
 static inline U16 LZ4_read16(const void* memPtr)
 {
-	U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
+	U16 val;
+	memcpy(&val, memPtr, sizeof(val));
+	return val;
 }
 static inline U32 LZ4_read32(const void* memPtr)
 {
-	U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
+	U32 val;
+	memcpy(&val, memPtr, sizeof(val));
+	return val;
 }
 static inline reg_t LZ4_read_ARCH(const void* memPtr)
 {
-	reg_t val; memcpy(&val, memPtr, sizeof(val)); return val;
+	reg_t val;
+	memcpy(&val, memPtr, sizeof(val));
+	return val;
 }
 static inline void LZ4_write16(void* memPtr, U16 value)
 {
@@ -268,7 +314,8 @@ static inline U16 LZ4_readLE16(const void* memPtr)
 {
 	if (LZ4_isLittleEndian()) {
 		return LZ4_read16(memPtr);
-	} else {
+	}
+	else {
 		const BYTE* p = (const BYTE*)memPtr;
 		return (U16)((U16)p[0] + (p[1] << 8));
 	}
@@ -278,7 +325,8 @@ static inline void LZ4_writeLE16(void* memPtr, U16 value)
 {
 	if (LZ4_isLittleEndian()) {
 		LZ4_write16(memPtr, value);
-	} else {
+	}
+	else {
 		BYTE* p = (BYTE*)memPtr;
 		p[0] = (BYTE)value;
 		p[1] = (BYTE)(value >> 8);
@@ -321,7 +369,10 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #define RUN_BITS (8 - ML_BITS)
 #define RUN_MASK ((1U << RUN_BITS) - 1)
 
-#define LZ4_STATIC_ASSERT(c)	{ enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c)                                                                                                                                                                                                                   \
+	{                                                                                                                                                                                                                                          \
+		enum { LZ4_static_assert = 1 / (int)(! ! (c)) };                                                                                                                                                                                       \
+	} /* use only *after* variable declarations */
 
 static inline unsigned LZ4_NbCommonBytes(reg_t val)
 {
@@ -334,10 +385,12 @@ static inline unsigned LZ4_NbCommonBytes (reg_t val)
 #elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 3))) && ! defined(LZ4_FORCE_SW_BITCOUNT)
 			return (__builtin_ctzll((U64)val) >> 3);
 #else
-			static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+			static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7,
+													 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
 			return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
 #endif
-		} else /* 32 bits */ {
+		}
+		else /* 32 bits */ {
 #if defined(_MSC_VER) && ! defined(LZ4_FORCE_SW_BITCOUNT)
 			unsigned long r;
 			_BitScanForward(&r, (U32)val);
@@ -349,7 +402,8 @@ static inline unsigned LZ4_NbCommonBytes (reg_t val)
 			return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
 #endif
 		}
-	} else   /* Big Endian CPU */ {
+	}
+	else /* Big Endian CPU */ {
 		if (sizeof(val) == 8) {
 #if defined(_MSC_VER) && defined(_WIN64) && ! defined(LZ4_FORCE_SW_BITCOUNT)
 			unsigned long r = 0;
@@ -361,20 +415,23 @@ static inline unsigned LZ4_NbCommonBytes (reg_t val)
 			unsigned r;
 			if (! (val >> 32)) {
 				r = 4;
-			} else {
+			}
+			else {
 				r = 0;
 				val >>= 32;
 			}
 			if (! (val >> 16)) {
 				r += 2;
 				val >>= 8;
-			} else {
+			}
+			else {
 				val >>= 24;
 			}
 			r += (! val);
 			return r;
 #endif
-		} else /* 32 bits */ {
+		}
+		else /* 32 bits */ {
 #if defined(_MSC_VER) && ! defined(LZ4_FORCE_SW_BITCOUNT)
 			unsigned long r = 0;
 			_BitScanReverse(&r, (unsigned long)val);
@@ -386,7 +443,8 @@ static inline unsigned LZ4_NbCommonBytes (reg_t val)
 			if (! (val >> 16)) {
 				r = 2;
 				val >>= 8;
-			} else {
+			}
+			else {
 				r = 0;
 				val >>= 24;
 			}
@@ -406,17 +464,20 @@ static inline unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE
 		reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
 		if (! diff) {
 			pIn += STEPSIZE;
-			pMatch+=STEPSIZE; continue;
+			pMatch += STEPSIZE;
+			continue;
 		}
 		pIn += LZ4_NbCommonBytes(diff);
 		return (unsigned)(pIn - pStart);
 	}
 
 	if ((STEPSIZE == 8) && (pIn < (pInLimit - 3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) {
-		pIn+=4; pMatch+=4;
+		pIn += 4;
+		pMatch += 4;
 	}
 	if ((pIn < (pInLimit - 1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) {
-		pIn+=2; pMatch+=2;
+		pIn += 2;
+		pMatch += 2;
 	}
 	if ((pIn < pInLimit) && (*pMatch == *pIn)) {
 		pIn++;
@@ -436,13 +497,17 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
 typedef enum { full = 0, partial = 1 } earlyEnd_directive;
 
-static inline int LZ4_compressBound(int isize)  { return LZ4_COMPRESSBOUND(isize); }
+static inline int LZ4_compressBound(int isize)
+{
+	return LZ4_COMPRESSBOUND(isize);
+}
 
 static inline U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
 {
 	if (tableType == byU16) {
 		return ((sequence * 2654435761U) >> ((MINMATCH * 8) - (LZ4_HASHLOG + 1)));
-	} else {
+	}
+	else {
 		return ((sequence * 2654435761U) >> ((MINMATCH * 8) - LZ4_HASHLOG));
 	}
 }
@@ -454,7 +519,8 @@ static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
 	const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG + 1 : LZ4_HASHLOG;
 	if (LZ4_isLittleEndian()) {
 		return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
-	} else {
+	}
+	else {
 		return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 	}
 }
@@ -471,15 +537,18 @@ static inline void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase,
 {
 	switch (tableType) {
 		case byPtr: {
-		const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p;
+			const BYTE** hashTable = (const BYTE**)tableBase;
+			hashTable[h] = p;
 			return;
 		}
 		case byU32: {
-		U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase);
+			U32* hashTable = (U32*)tableBase;
+			hashTable[h] = (U32)(p - srcBase);
 			return;
 		}
 		case byU16: {
-		U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase);
+			U16* hashTable = (U16*)tableBase;
+			hashTable[h] = (U16)(p - srcBase);
 			return;
 		}
 	}
@@ -599,7 +668,8 @@ FORCE_INLINE int LZ4_compress_generic(
 					if (match < (const BYTE*)source) {
 						refDelta = dictDelta;
 						lowLimit = dictionary;
-					} else {
+					}
+					else {
 						refDelta = 0;
 						lowLimit = (const BYTE*)source;
 					}
@@ -607,9 +677,7 @@ FORCE_INLINE int LZ4_compress_generic(
 				forwardH = LZ4_hashPosition(forwardIp, tableType);
 				LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-			} while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+			} while (((dictIssue == dictSmall) ? (match < lowRefLimit) : 0) || ((tableType == byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match + refDelta) != LZ4_read32(ip)));
-				|| ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-				|| (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
 		}
 
 		/* Catch up */
@@ -633,7 +701,8 @@ FORCE_INLINE int LZ4_compress_generic(
 					*op++ = 255;
 				}
 				*op++ = (BYTE)len;
-			} else {
+			}
+			else {
 				*token = (BYTE)(litLength << ML_BITS);
 			}
 
@@ -665,7 +734,8 @@ _next_match:
 					matchCode += more;
 					ip += more;
 				}
-			} else {
+			}
+			else {
 				matchCode = LZ4_count(ip + MINMATCH, match + MINMATCH, matchlimit);
 				ip += MINMATCH + matchCode;
 			}
@@ -685,7 +755,8 @@ _next_match:
 				}
 				op += matchCode / 255;
 				*op++ = (BYTE)(matchCode % 255);
-			} else {
+			}
+			else {
 				*token += (BYTE)(matchCode);
 			}
 		}
@@ -706,15 +777,14 @@ _next_match:
 			if (match < (const BYTE*)source) {
 				refDelta = dictDelta;
 				lowLimit = dictionary;
-			} else {
+			}
+			else {
 				refDelta = 0;
 				lowLimit = (const BYTE*)source;
 			}
 		}
 		LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-		if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
+		if (((dictIssue == dictSmall) ? (match >= lowRefLimit) : 1) && (match + MAX_DISTANCE >= ip) && (LZ4_read32(match + refDelta) == LZ4_read32(ip))) {
-			&& (match+MAX_DISTANCE>=ip)
-			&& (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) {
 			token = op++;
 			*token = 0;
 			goto _next_match;
@@ -739,7 +809,8 @@ _last_literals:
 				*op++ = 255;
 			}
 			*op++ = (BYTE)accumulator;
-		} else {
+		}
+		else {
 			*op++ = (BYTE)(lastRun << ML_BITS);
 		}
 		memcpy(op, anchor, lastRun);
@@ -759,13 +830,16 @@ static inline int LZ4_compress_fast_extState(void* state, const char* source, ch
 	if (maxOutputSize >= LZ4_compressBound(inputSize)) {
 		if (inputSize < LZ4_64Klimit) {
 			return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
-		} else {
+		}
+		else {
 			return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*) == 8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
 		}
-	} else {
+	}
+	else {
 		if (inputSize < LZ4_64Klimit) {
 			return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
-		} else {
+		}
+		else {
 			return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*) == 8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
 		}
 	}
@@ -825,7 +899,6 @@ FORCE_INLINE int LZ4_decompress_generic(
 	const int safeDecode = (endOnInput == endOnInputSize);
 	const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
 
-
 	/* Special cases */
 	if ((partialDecoding) && (oexit > oend - MFLIMIT)) {
 		oexit = oend - MFLIMIT; /* targetOutputSize too high => decode everything */
@@ -861,8 +934,7 @@ FORCE_INLINE int LZ4_decompress_generic(
 
 		/* copy literals */
 		cpy = op + length;
-		if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
+		if (((endOnInput) && ((cpy > (partialDecoding ? oexit : oend - MFLIMIT)) || (ip + length > iend - (2 + 1 + LASTLITERALS)))) || ((! endOnInput) && (cpy > oend - WILDCOPYLENGTH))) {
-			|| ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) {
 			if (partialDecoding) {
 				if (cpy > oend) {
 					goto _output_error; /* Error : write attempt beyond end of output buffer */
@@ -870,7 +942,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 				if ((endOnInput) && (ip + length > iend)) {
 					goto _output_error; /* Error : read attempt beyond end of input buffer */
 				}
-			} else {
+			}
+			else {
 				if ((! endOnInput) && (cpy != oend)) {
 					goto _output_error; /* Error : block decoding must stop exactly there */
 				}
@@ -923,7 +996,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 				/* match can be copied as a single segment from external dictionary */
 				memmove(op, dictEnd - (lowPrefix - match), length);
 				op += length;
-			} else {
+			}
+			else {
 				/* match encompass external dictionary and current block */
 				size_t const copySize = (size_t)(lowPrefix - match);
 				size_t const restSize = length - copySize;
@@ -935,7 +1009,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 					while (op < endOfMatch) {
 						*op++ = *copyFrom++;
 					}
-				} else {
+				}
+				else {
 					memcpy(op, lowPrefix, restSize);
 					op += restSize;
 				}
@@ -954,7 +1029,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 			match += dec32table[offset];
 			memcpy(op + 4, match, 4);
 			match -= dec64;
-		} else {
+		}
+		else {
 			LZ4_copy8(op, match);
 			match += 8;
 		}
@@ -973,7 +1049,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 			while (op < cpy) {
 				*op++ = *match++;
 			}
-		} else {
+		}
+		else {
 			LZ4_copy8(op, match);
 			if (length > 16) {
 				LZ4_wildCopy(op + 8, match + 8, cpy);
@@ -985,7 +1062,8 @@ FORCE_INLINE int LZ4_decompress_generic(
 	/* end of decoding */
 	if (endOnInput) {
 		return (int)(((char*)op) - dest); /* Nb of output bytes decoded */
-	} else {
+	}
+	else {
 		return (int)(((const char*)ip) - source); /* Nb of input bytes read */
 	}
 	/* Overflow error detected */
@@ -1030,7 +1108,8 @@ void Packet::armor(const void *key,bool encryptPayload,const AES aesKeys[2])
 		*reinterpret_cast<uint64_t*>(data + ZT_PACKET_IDX_IV) = tag[0];
 		*reinterpret_cast<uint64_t*>(data + ZT_PACKET_IDX_MAC) = tag[1];
 #endif
-	} else {
+	}
+	else {
 		setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE);
 
 		uint8_t mangledKey[32];
@@ -1048,7 +1127,8 @@ void Packet::armor(const void *key,bool encryptPayload,const AES aesKeys[2])
 #else
 			(*reinterpret_cast<uint64_t*>(data + ZT_PACKET_IDX_MAC)) = mac[0];
 #endif
-		} else {
+		}
+		else {
 			Salsa20 s20(mangledKey, data + ZT_PACKET_IDX_IV);
 
 			uint64_t macKey[4];
@@ -1094,7 +1174,8 @@ bool Packet::dearmor(const void *key,const AES aesKeys[2])
 			dec.update(payload, payloadLen);
 			return dec.finish();
 		}
-	} else if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) {
+	}
+	else if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE) || (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) {
 		uint8_t mangledKey[32];
 		_salsa20MangleKey((const unsigned char*)key, mangledKey);
 		if (ZT_HAS_FAST_CRYPTO()) {
@@ -1114,7 +1195,8 @@ bool Packet::dearmor(const void *key,const AES aesKeys[2])
 			if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) {
 				Salsa20::memxor(data + ZT_PACKET_IDX_VERB, reinterpret_cast<const uint8_t*>(keyStream + 8), payloadLen);
 			}
-		} else {
+		}
+		else {
 			Salsa20 s20(mangledKey, data + ZT_PACKET_IDX_IV);
 			uint64_t macKey[4];
 			s20.crypt12(ZERO_KEY, macKey, sizeof(macKey));
@@ -1183,7 +1265,8 @@ bool Packet::uncompress()
 			if ((ucl > 0) && (ucl <= (int)(capacity() - ZT_PACKET_IDX_PAYLOAD))) {
 				setSize((unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD);
 				memcpy(data + ZT_PACKET_IDX_PAYLOAD, buf, ucl);
-			} else {
+			}
+			else {
 				return false;
 			}
 		}

node/Packet.hpp

@@ -14,21 +14,19 @@
 #ifndef ZT_N_PACKET_HPP
 #define ZT_N_PACKET_HPP
 
-#include <stdint.h>
+#include "AES.hpp"
-#include <string.h>
-#include <stdio.h>
-
-#include <string>
-#include <iostream>
-
-#include "Constants.hpp"
-
 #include "Address.hpp"
+#include "Buffer.hpp"
+#include "Constants.hpp"
 #include "Poly1305.hpp"
 #include "Salsa20.hpp"
-#include "AES.hpp"
 #include "Utils.hpp"
-#include "Buffer.hpp"
+
+#include <iostream>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <string>
 
 /**
  * Protocol version -- incremented only for major changes
@@ -388,8 +386,7 @@ namespace ZeroTier {
  * For unencrypted packets, MAC is computed on plaintext. Only HELLO is ever
  * sent in the clear, as it's the "here is my public key" message.
  */
-class Packet : public Buffer<ZT_PROTO_MAX_PACKET_LENGTH>
+class Packet : public Buffer<ZT_PROTO_MAX_PACKET_LENGTH> {
-{
   public:
 	/**
 	 * A packet fragment
@@ -417,22 +414,17 @@ public:
 	 * receipt to authenticate and decrypt; there is no per-fragment MAC. (But if
 	 * fragments are corrupt, the MAC will fail for the whole assembled packet.)
 	 */
-	class Fragment : public Buffer<ZT_PROTO_MAX_PACKET_LENGTH>
+	class Fragment : public Buffer<ZT_PROTO_MAX_PACKET_LENGTH> {
-	{
 	  public:
-		Fragment() :
+		Fragment() : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>()
-			Buffer<ZT_PROTO_MAX_PACKET_LENGTH>()
 		{
 		}
 
-		template<unsigned int C2>
+		template <unsigned int C2> Fragment(const Buffer<C2>& b) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b)
-		Fragment(const Buffer<C2> &b) :
-			Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b)
 		{
 		}
 
-		Fragment(const void *data,unsigned int len) :
+		Fragment(const void* data, unsigned int len) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(data, len)
-			Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(data,len)
 		{
 		}
 
@@ -481,32 +473,50 @@ public:
 		 *
 		 * @return Destination ZT address
 		 */
-		inline Address destination() const { return Address(field(ZT_PACKET_FRAGMENT_IDX_DEST,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
+		inline Address destination() const
+		{
+			return Address(field(ZT_PACKET_FRAGMENT_IDX_DEST, ZT_ADDRESS_LENGTH), ZT_ADDRESS_LENGTH);
+		}
 
 		/**
 		 * @return True if fragment is of a valid length
 		 */
-		inline bool lengthValid() const { return (size() >= ZT_PACKET_FRAGMENT_IDX_PAYLOAD); }
+		inline bool lengthValid() const
+		{
+			return (size() >= ZT_PACKET_FRAGMENT_IDX_PAYLOAD);
+		}
 
 		/**
 		 * @return ID of packet this is a fragment of
 		 */
-		inline uint64_t packetId() const { return at<uint64_t>(ZT_PACKET_FRAGMENT_IDX_PACKET_ID); }
+		inline uint64_t packetId() const
+		{
+			return at<uint64_t>(ZT_PACKET_FRAGMENT_IDX_PACKET_ID);
+		}
 
 		/**
 		 * @return Total number of fragments in packet
 		 */
-		inline unsigned int totalFragments() const { return (((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) >> 4) & 0xf); }
+		inline unsigned int totalFragments() const
+		{
+			return (((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) >> 4) & 0xf);
+		}
 
 		/**
 		 * @return Fragment number of this fragment
 		 */
-		inline unsigned int fragmentNumber() const { return ((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) & 0xf); }
+		inline unsigned int fragmentNumber() const
+		{
+			return ((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) & 0xf);
+		}
 
 		/**
 		 * @return Fragment ZT hop count
 		 */
-		inline unsigned int hops() const { return (unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_HOPS]); }
+		inline unsigned int hops() const
+		{
+			return (unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_HOPS]);
+		}
 
 		/**
 		 * Increment this packet's hop count
@@ -519,7 +529,10 @@ public:
 		/**
 		 * @return Length of payload in bytes
 		 */
-		inline unsigned int payloadLength() const { return ((size() > ZT_PACKET_FRAGMENT_IDX_PAYLOAD) ? (size() - ZT_PACKET_FRAGMENT_IDX_PAYLOAD) : 0); }
+		inline unsigned int payloadLength() const
+		{
+			return ((size() > ZT_PACKET_FRAGMENT_IDX_PAYLOAD) ? (size() - ZT_PACKET_FRAGMENT_IDX_PAYLOAD) : 0);
+		}
 
 		/**
 		 * @return Raw packet payload
@@ -1056,8 +1069,7 @@ public:
 	/**
 	 * Error codes for VERB_ERROR
 	 */
-	enum ErrorCode
+	enum ErrorCode {
-	{
 		/* No error, not actually used in transit */
 		ERROR_NONE = 0x00,
 
@@ -1089,14 +1101,11 @@ public:
 		ERROR_NETWORK_AUTHENTICATION_REQUIRED = 0x09
 	};
 
-	template<unsigned int C2>
+	template <unsigned int C2> Packet(const Buffer<C2>& b) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b)
-	Packet(const Buffer<C2> &b) :
-		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b)
 	{
 	}
 
-	Packet(const void *data,unsigned int len) :
+	Packet(const void* data, unsigned int len) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(data, len)
-		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(data,len)
 	{
 	}
 
@@ -1107,8 +1116,7 @@ public:
 	 * Use the header access methods (setDestination() and friends) to fill out
 	 * the header. Payload should be appended; initial size is header size.
 	 */
-	Packet() :
+	Packet() : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
-		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
 	{
 		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV, 8), 8);
 		(*this)[ZT_PACKET_IDX_FLAGS] = 0;	// zero flags, cipher ID, and hops
@@ -1123,8 +1131,7 @@ public:
 	 * @param prototype Prototype packet
 	 * @param dest Destination ZeroTier address for new packet
 	 */
-	Packet(const Packet &prototype,const Address &dest) :
+	Packet(const Packet& prototype, const Address& dest) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(prototype)
-		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(prototype)
 	{
 		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV, 8), 8);
 		setDestination(dest);
@@ -1137,8 +1144,7 @@ public:
 	 * @param source Source ZT address
 	 * @param v Verb
 	 */
-	Packet(const Address &dest,const Address &source,const Verb v) :
+	Packet(const Address& dest, const Address& source, const Verb v) : Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
-		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
 	{
 		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV, 8), 8);
 		setDestination(dest);
@@ -1171,45 +1177,66 @@ public:
 	 * technically different but otherwise identical copies of the same
 	 * packet.
 	 */
-	inline void newInitializationVector() { Utils::getSecureRandom(field(ZT_PACKET_IDX_IV,8),8); }
+	inline void newInitializationVector()
+	{
+		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV, 8), 8);
+	}
 
 	/**
 	 * Set this packet's destination
 	 *
 	 * @param dest ZeroTier address of destination
 	 */
-	inline void setDestination(const Address &dest) { dest.copyTo(field(ZT_PACKET_IDX_DEST,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
+	inline void setDestination(const Address& dest)
+	{
+		dest.copyTo(field(ZT_PACKET_IDX_DEST, ZT_ADDRESS_LENGTH), ZT_ADDRESS_LENGTH);
+	}
 
 	/**
 	 * Set this packet's source
 	 *
 	 * @param source ZeroTier address of source
 	 */
-	inline void setSource(const Address &source) { source.copyTo(field(ZT_PACKET_IDX_SOURCE,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
+	inline void setSource(const Address& source)
+	{
+		source.copyTo(field(ZT_PACKET_IDX_SOURCE, ZT_ADDRESS_LENGTH), ZT_ADDRESS_LENGTH);
+	}
 
 	/**
 	 * Get this packet's destination
 	 *
 	 * @return Destination ZT address
 	 */
-	inline Address destination() const { return Address(field(ZT_PACKET_IDX_DEST,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
+	inline Address destination() const
+	{
+		return Address(field(ZT_PACKET_IDX_DEST, ZT_ADDRESS_LENGTH), ZT_ADDRESS_LENGTH);
+	}
 
 	/**
 	 * Get this packet's source
 	 *
 	 * @return Source ZT address
 	 */
-	inline Address source() const { return Address(field(ZT_PACKET_IDX_SOURCE,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
+	inline Address source() const
+	{
+		return Address(field(ZT_PACKET_IDX_SOURCE, ZT_ADDRESS_LENGTH), ZT_ADDRESS_LENGTH);
+	}
 
 	/**
 	 * @return True if packet is of valid length
 	 */
-	inline bool lengthValid() const { return (size() >= ZT_PROTO_MIN_PACKET_LENGTH); }
+	inline bool lengthValid() const
+	{
+		return (size() >= ZT_PROTO_MIN_PACKET_LENGTH);
+	}
 
 	/**
 	 * @return True if packet is fragmented (expect fragments)
 	 */
-	inline bool fragmented() const { return (((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0); }
+	inline bool fragmented() const
+	{
+		return (((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0);
+	}
 
 	/**
 	 * Set this packet's fragmented flag
@@ -1220,7 +1247,8 @@ public:
 	{
 		if (f) {
 			(*this)[ZT_PACKET_IDX_FLAGS] |= (char)ZT_PROTO_FLAG_FRAGMENTED;
-		} else {
+		}
+		else {
 			(*this)[ZT_PACKET_IDX_FLAGS] &= (char)(~ZT_PROTO_FLAG_FRAGMENTED);
 		}
 	}
@@ -1228,12 +1256,18 @@ public:
 	/**
 	 * @return True if compressed (result only valid if unencrypted)
 	 */
-	inline bool compressed() const { return (((unsigned char)(*this)[ZT_PACKET_IDX_VERB] & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0); }
+	inline bool compressed() const
+	{
+		return (((unsigned char)(*this)[ZT_PACKET_IDX_VERB] & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0);
+	}
 
 	/**
 	 * @return ZeroTier forwarding hops (0 to 7)
 	 */
-	inline unsigned int hops() const { return ((unsigned int)(*this)[ZT_PACKET_IDX_FLAGS] & 0x07); }
+	inline unsigned int hops() const
+	{
+		return ((unsigned int)(*this)[ZT_PACKET_IDX_FLAGS] & 0x07);
+	}
 
 	/**
 	 * Increment this packet's hop count
@@ -1270,7 +1304,8 @@ public:
 		// Set DEPRECATED "encrypted" flag -- used by pre-1.0.3 peers
 		if (c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) {
 			b |= ZT_PROTO_FLAG_ENCRYPTED;
-		} else {
+		}
+		else {
 			b &= (~ZT_PROTO_FLAG_ENCRYPTED);
 		}
 	}
@@ -1280,7 +1315,10 @@ public:
 	 *
 	 * @return Trusted path ID (from MAC field)
 	 */
-	inline uint64_t trustedPathId() const { return at<uint64_t>(ZT_PACKET_IDX_MAC); }
+	inline uint64_t trustedPathId() const
+	{
+		return at<uint64_t>(ZT_PACKET_IDX_MAC);
+	}
 
 	/**
 	 * Set this packet's trusted path ID and set the cipher spec to trusted path
@@ -1304,7 +1342,10 @@ public:
 	 *
 	 * @return Packet ID
 	 */
-	inline uint64_t packetId() const { return at<uint64_t>(ZT_PACKET_IDX_IV); }
+	inline uint64_t packetId() const
+	{
+		return at<uint64_t>(ZT_PACKET_IDX_IV);
+	}
 
 	/**
 	 * Set packet verb
@@ -1314,22 +1355,34 @@ public:
 	 *
 	 * @param v New packet verb
 	 */
-	inline void setVerb(Verb v) { (*this)[ZT_PACKET_IDX_VERB] = (char)v; }
+	inline void setVerb(Verb v)
+	{
+		(*this)[ZT_PACKET_IDX_VERB] = (char)v;
+	}
 
 	/**
 	 * @return Packet verb (not including flag bits)
 	 */
-	inline Verb verb() const { return (Verb)((*this)[ZT_PACKET_IDX_VERB] & 0x1f); }
+	inline Verb verb() const
+	{
+		return (Verb)((*this)[ZT_PACKET_IDX_VERB] & 0x1f);
+	}
 
 	/**
 	 * @return Length of packet payload
 	 */
-	inline unsigned int payloadLength() const { return ((size() < ZT_PROTO_MIN_PACKET_LENGTH) ? 0 : (size() - ZT_PROTO_MIN_PACKET_LENGTH)); }
+	inline unsigned int payloadLength() const
+	{
+		return ((size() < ZT_PROTO_MIN_PACKET_LENGTH) ? 0 : (size() - ZT_PROTO_MIN_PACKET_LENGTH));
+	}
 
 	/**
 	 * @return Raw packet payload
 	 */
-	inline const unsigned char *payload() const { return field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); }
+	inline const unsigned char* payload() const
+	{
+		return field(ZT_PACKET_IDX_PAYLOAD, size() - ZT_PACKET_IDX_PAYLOAD);
+	}
 
 	/**
 	 * Armor packet for transport

node/PacketMultiplexer.cpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c)2013-2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "PacketMultiplexer.hpp"
+
+#include "Constants.hpp"
+#include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+namespace ZeroTier {
+
+PacketMultiplexer::PacketMultiplexer(const RuntimeEnvironment* renv)
+{
+	RR = renv;
+};
+
+void PacketMultiplexer::putFrame(void* tPtr, uint64_t nwid, void** nuptr, const MAC& source, const MAC& dest, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len, unsigned int flowId)
+{
+#if defined(__APPLE__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__WINDOWS__)
+	RR->node->putFrame(tPtr, nwid, nuptr, source, dest, etherType, vlanId, (const void*)data, len);
+	return;
+#endif
+
+	if (! _enabled) {
+		RR->node->putFrame(tPtr, nwid, nuptr, source, dest, etherType, vlanId, (const void*)data, len);
+		return;
+	}
+
+	PacketRecord* packet;
+	_rxPacketVector_m.lock();
+	if (_rxPacketVector.empty()) {
+		packet = new PacketRecord;
+	}
+	else {
+		packet = _rxPacketVector.back();
+		_rxPacketVector.pop_back();
+	}
+	_rxPacketVector_m.unlock();
+
+	packet->tPtr = tPtr;
+	packet->nwid = nwid;
+	packet->nuptr = nuptr;
+	packet->source = source.toInt();
+	packet->dest = dest.toInt();
+	packet->etherType = etherType;
+	packet->vlanId = vlanId;
+	packet->len = len;
+	packet->flowId = flowId;
+	memcpy(packet->data, data, len);
+
+	int bucket = flowId % _concurrency;
+	_rxPacketQueues[bucket]->postLimit(packet, 2048);
+}
+
+void PacketMultiplexer::setUpPostDecodeReceiveThreads(unsigned int concurrency, bool cpuPinningEnabled)
+{
+#if defined(__APPLE__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__WINDOWS__)
+	return;
+#endif
+	_enabled = true;
+	_concurrency = concurrency;
+	bool _enablePinning = cpuPinningEnabled;
+
+	for (unsigned int i = 0; i < _concurrency; ++i) {
+		fprintf(stderr, "Reserved queue for thread %d\n", i);
+		_rxPacketQueues.push_back(new BlockingQueue<PacketRecord*>());
+	}
+
+	// Each thread picks from its own queue to feed into the core
+	for (unsigned int i = 0; i < _concurrency; ++i) {
+		_rxThreads.push_back(std::thread([this, i, _enablePinning]() {
+			fprintf(stderr, "Created post-decode packet ingestion thread %d\n", i);
+
+			PacketRecord* packet = nullptr;
+			for (;;) {
+				if (! _rxPacketQueues[i]->get(packet)) {
+					break;
+				}
+				if (! packet) {
+					break;
+				}
+
+				// fprintf(stderr, "popped packet from queue %d\n", i);
+
+				MAC sourceMac = MAC(packet->source);
+				MAC destMac = MAC(packet->dest);
+
+				RR->node->putFrame(packet->tPtr, packet->nwid, packet->nuptr, sourceMac, destMac, packet->etherType, 0, (const void*)packet->data, packet->len);
+				{
+					Mutex::Lock l(_rxPacketVector_m);
+					_rxPacketVector.push_back(packet);
+				}
+				/*
+				if (ZT_ResultCode_isFatal(err)) {
+					char tmp[256];
+					OSUtils::ztsnprintf(tmp, sizeof(tmp), "error processing packet: %d", (int)err);
+					Mutex::Lock _l(_termReason_m);
+					_termReason = ONE_UNRECOVERABLE_ERROR;
+					_fatalErrorMessage = tmp;
+					this->terminate();
+					break;
+				}
+				*/
+			}
+		}));
+	}
+}
+
+}	// namespace ZeroTier

node/PacketMultiplexer.hpp

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c)2013-2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_PACKET_MULTIPLEXER_HPP
+#define ZT_PACKET_MULTIPLEXER_HPP
+
+#include "../osdep/BlockingQueue.hpp"
+#include "MAC.hpp"
+#include "Mutex.hpp"
+#include "RuntimeEnvironment.hpp"
+
+#include <thread>
+#include <vector>
+
+namespace ZeroTier {
+
+struct PacketRecord {
+	void* tPtr;
+	uint64_t nwid;
+	void** nuptr;
+	uint64_t source;
+	uint64_t dest;
+	unsigned int etherType;
+	unsigned int vlanId;
+	uint8_t data[ZT_MAX_MTU];
+	unsigned int len;
+	unsigned int flowId;
+};
+
+class PacketMultiplexer {
+  public:
+	const RuntimeEnvironment* RR;
+
+	PacketMultiplexer(const RuntimeEnvironment* renv);
+
+	void setUpPostDecodeReceiveThreads(unsigned int concurrency, bool cpuPinningEnabled);
+
+	void putFrame(void* tPtr, uint64_t nwid, void** nuptr, const MAC& source, const MAC& dest, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len, unsigned int flowId);
+
+	std::vector<BlockingQueue<PacketRecord*>*> _rxPacketQueues;
+
+	unsigned int _concurrency;
+	// pool
+	std::vector<PacketRecord*> _rxPacketVector;
+	std::vector<std::thread> _rxPacketThreads;
+	Mutex _rxPacketVector_m, _rxPacketThreads_m;
+
+	std::vector<std::thread> _rxThreads;
+	unsigned int _rxThreadCount;
+	bool _enabled;
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_PACKET_MULTIPLEXER_HPP

node/Path.cpp

@@ -12,8 +12,9 @@
 /****/
 
 #include "Path.hpp"
-#include "RuntimeEnvironment.hpp"
+
 #include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
 
 namespace ZeroTier {
 

node/Path.hpp

@@ -14,20 +14,19 @@
 #ifndef ZT_PATH_HPP
 #define ZT_PATH_HPP
 
-#include <stdint.h>
+#include "AtomicCounter.hpp"
-#include <string.h>
-#include <stdlib.h>
-
-#include <stdexcept>
-#include <algorithm>
-
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "SharedPtr.hpp"
-#include "AtomicCounter.hpp"
-#include "Utils.hpp"
 #include "Packet.hpp"
 #include "RingBuffer.hpp"
+#include "SharedPtr.hpp"
+#include "Utils.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
 
 /**
  * Maximum return value of preferenceRank()
@@ -41,8 +40,7 @@ class RuntimeEnvironment;
 /**
  * A path across the physical network
  */
-class Path
+class Path {
-{
 	friend class SharedPtr<Path>;
 	friend class Bond;
 
@@ -50,10 +48,11 @@ public:
 	/**
 	 * Efficient unique key for paths in a Hashtable
 	 */
-	class HashKey
+	class HashKey {
-	{
 	  public:
-		HashKey() {}
+		HashKey()
+		{
+		}
 
 		HashKey(const int64_t l, const InetAddress& r)
 		{
@@ -61,69 +60,82 @@ public:
 				_k[0] = (uint64_t)reinterpret_cast<const struct sockaddr_in*>(&r)->sin_addr.s_addr;
 				_k[1] = (uint64_t)reinterpret_cast<const struct sockaddr_in*>(&r)->sin_port;
 				_k[2] = (uint64_t)l;
-			} else if (r.ss_family == AF_INET6) {
+			}
+			else if (r.ss_family == AF_INET6) {
 				memcpy(_k, reinterpret_cast<const struct sockaddr_in6*>(&r)->sin6_addr.s6_addr, 16);
 				_k[2] = ((uint64_t)reinterpret_cast<const struct sockaddr_in6*>(&r)->sin6_port << 32) ^ (uint64_t)l;
-			} else {
+			}
+			else {
 				memcpy(_k, &r, std::min(sizeof(_k), sizeof(InetAddress)));
 				_k[2] += (uint64_t)l;
 			}
 		}
 
-		inline unsigned long hashCode() const { return (unsigned long)(_k[0] + _k[1] + _k[2]); }
+		inline unsigned long hashCode() const
+		{
+			return (unsigned long)(_k[0] + _k[1] + _k[2]);
+		}
 
-		inline bool operator==(const HashKey &k) const { return ( (_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]) ); }
+		inline bool operator==(const HashKey& k) const
-		inline bool operator!=(const HashKey &k) const { return (!(*this == k)); }
+		{
+			return ((_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]));
+		}
+		inline bool operator!=(const HashKey& k) const
+		{
+			return (! (*this == k));
+		}
 
 	  private:
 		uint64_t _k[3];
 	};
 
-	Path() :
+	Path()
-		_lastOut(0),
+		: _lastOut(0)
-		_lastIn(0),
+		, _lastIn(0)
-		_lastTrustEstablishedPacketReceived(0),
+		, _lastTrustEstablishedPacketReceived(0)
-		_lastEchoRequestReceived(0),
+		, _lastEchoRequestReceived(0)
-		_localPort(0),
+		, _localPort(0)
-		_localSocket(-1),
+		, _localSocket(-1)
-		_latencyMean(0.0),
+		, _latencyMean(0.0)
-		_latencyVariance(0.0),
+		, _latencyVariance(0.0)
-		_packetLossRatio(0.0),
+		, _packetLossRatio(0.0)
-		_packetErrorRatio(0.0),
+		, _packetErrorRatio(0.0)
-		_assignedFlowCount(0),
+		, _assignedFlowCount(0)
-		_valid(true),
+		, _valid(true)
-		_eligible(false),
+		, _eligible(false)
-		_bonded(false),
+		, _bonded(false)
-		_mtu(0),
+		, _mtu(0)
-		_givenLinkSpeed(0),
+		, _givenLinkSpeed(0)
-		_relativeQuality(0),
+		, _relativeQuality(0)
-		_latency(0xffff),
+		, _latency(0xffff)
-		_addr(),
+		, _addr()
-		_ipScope(InetAddress::IP_SCOPE_NONE)
+		, _ipScope(InetAddress::IP_SCOPE_NONE)
-		{}
+	{
+	}
 
-	Path(const int64_t localSocket,const InetAddress &addr) :
+	Path(const int64_t localSocket, const InetAddress& addr)
-		_lastOut(0),
+		: _lastOut(0)
-		_lastIn(0),
+		, _lastIn(0)
-		_lastTrustEstablishedPacketReceived(0),
+		, _lastTrustEstablishedPacketReceived(0)
-		_lastEchoRequestReceived(0),
+		, _lastEchoRequestReceived(0)
-		_localPort(0),
+		, _localPort(0)
-		_localSocket(localSocket),
+		, _localSocket(localSocket)
-		_latencyMean(0.0),
+		, _latencyMean(0.0)
-		_latencyVariance(0.0),
+		, _latencyVariance(0.0)
-		_packetLossRatio(0.0),
+		, _packetLossRatio(0.0)
-		_packetErrorRatio(0.0),
+		, _packetErrorRatio(0.0)
-		_assignedFlowCount(0),
+		, _assignedFlowCount(0)
-		_valid(true),
+		, _valid(true)
-		_eligible(false),
+		, _eligible(false)
-		_bonded(false),
+		, _bonded(false)
-		_mtu(0),
+		, _mtu(0)
-		_givenLinkSpeed(0),
+		, _givenLinkSpeed(0)
-		_relativeQuality(0),
+		, _relativeQuality(0)
-		_latency(0xffff),
+		, _latency(0xffff)
-		_addr(addr),
+		, _addr(addr)
-		_ipScope(addr.ipScope())
+		, _ipScope(addr.ipScope())
-	{}
+	{
+	}
 
 	/**
 	 * Called when a packet is received from this remote path, regardless of content
@@ -138,7 +150,10 @@ public:
 	/**
 	 * Set time last trusted packet was received (done in Peer::received())
 	 */
-	inline void trustedPacketReceived(const uint64_t t) { _lastTrustEstablishedPacketReceived = t; }
+	inline void trustedPacketReceived(const uint64_t t)
+	{
+		_lastTrustEstablishedPacketReceived = t;
+	}
 
 	/**
 	 * Send a packet via this path (last out time is also updated)
@@ -157,7 +172,10 @@ public:
 	 *
 	 * @param t Time of send
 	 */
-	inline void sent(const int64_t t) { _lastOut = t; }
+	inline void sent(const int64_t t)
+	{
+		_lastOut = t;
+	}
 
 	/**
 	 * Update path latency with a new measurement
@@ -169,7 +187,8 @@ public:
 		unsigned int pl = _latency;
 		if (pl < 0xffff) {
 			_latency = (pl + l) / 2;
-		} else {
+		}
+		else {
 			_latency = l;
 		}
 	}
@@ -177,27 +196,42 @@ public:
 	/**
 	 * @return Local socket as specified by external code
 	 */
-	inline int64_t localSocket() const { return _localSocket; }
+	inline int64_t localSocket() const
+	{
+		return _localSocket;
+	}
 
 	/**
 	 * @return Local port corresponding to the localSocket
 	 */
-	inline int64_t localPort() const { return _localPort; }
+	inline int64_t localPort() const
+	{
+		return _localPort;
+	}
 
 	/**
 	 * @return Physical address
 	 */
-	inline const InetAddress &address() const { return _addr; }
+	inline const InetAddress& address() const
+	{
+		return _addr;
+	}
 
 	/**
 	 * @return IP scope -- faster shortcut for address().ipScope()
 	 */
-	inline InetAddress::IpScope ipScope() const { return _ipScope; }
+	inline InetAddress::IpScope ipScope() const
+	{
+		return _ipScope;
+	}
 
 	/**
 	 * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
 	 */
-	inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }
+	inline bool trustEstablished(const int64_t now) const
+	{
+		return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION);
+	}
 
 	/**
 	 * @return Preference rank, higher == better
@@ -253,7 +287,10 @@ public:
 	/**
 	 * @return Latency or 0xffff if unknown
 	 */
-	inline unsigned int latency() const { return _latency; }
+	inline unsigned int latency() const
+	{
+		return _latency;
+	}
 
 	/**
 	 * @return Path quality -- lower is better
@@ -268,34 +305,50 @@ public:
 	/**
 	 * @return True if this path is alive (receiving heartbeats)
 	 */
-	inline bool alive(const int64_t now) const {
+	inline bool alive(const int64_t now) const
+	{
 		return (now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000);
 	}
 
 	/**
 	 * @return True if this path needs a heartbeat
 	 */
-	inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); }
+	inline bool needsHeartbeat(const int64_t now) const
+	{
+		return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD);
+	}
 
 	/**
 	 * @return Last time we sent something
 	 */
-	inline int64_t lastOut() const { return _lastOut; }
+	inline int64_t lastOut() const
+	{
+		return _lastOut;
+	}
 
 	/**
 	 * @return Last time we received anything
 	 */
-	inline int64_t lastIn() const { return _lastIn; }
+	inline int64_t lastIn() const
+	{
+		return _lastIn;
+	}
 
 	/**
 	 * @return the age of the path in terms of receiving packets
 	 */
-	inline int64_t age(int64_t now) { return (now - _lastIn); }
+	inline int64_t age(int64_t now)
+	{
+		return (now - _lastIn);
+	}
 
 	/**
 	 * @return Time last trust-established packet was received
 	 */
-	inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
+	inline int64_t lastTrustEstablishedPacketReceived() const
+	{
+		return _lastTrustEstablishedPacketReceived;
+	}
 
 	/**
 	 * Rate limit gate for inbound ECHO requests
@@ -312,68 +365,101 @@ public:
 	/**
 	 * @return Mean latency as reported by the bonding layer
 	 */
-	inline float latencyMean() const { return _latencyMean; }
+	inline float latencyMean() const
+	{
+		return _latencyMean;
+	}
 
 	/**
 	 * @return Latency variance as reported by the bonding layer
 	 */
-	inline float latencyVariance() const { return _latencyVariance; }
+	inline float latencyVariance() const
+	{
+		return _latencyVariance;
+	}
 
 	/**
 	 * @return Packet Loss Ratio as reported by the bonding layer
 	 */
-	inline float packetLossRatio() const { return _packetLossRatio; }
+	inline float packetLossRatio() const
+	{
+		return _packetLossRatio;
+	}
 
 	/**
 	 * @return Packet Error Ratio as reported by the bonding layer
 	 */
-	inline float packetErrorRatio() const { return _packetErrorRatio; }
+	inline float packetErrorRatio() const
+	{
+		return _packetErrorRatio;
+	}
 
 	/**
 	 * @return Number of flows assigned to this path
 	 */
-	inline unsigned int assignedFlowCount() const { return _assignedFlowCount; }
+	inline unsigned int assignedFlowCount() const
+	{
+		return _assignedFlowCount;
+	}
 
 	/**
 	 * @return Whether this path is valid as reported by the bonding layer. The bonding layer
 	 * actually checks with Phy to see if the interface is still up
 	 */
-	inline bool valid() const { return _valid; }
+	inline bool valid() const
+	{
+		return _valid;
+	}
 
 	/**
 	 * @return Whether this path is eligible for use in a bond as reported by the bonding layer
 	 */
-	inline bool eligible() const { return _eligible; }
+	inline bool eligible() const
+	{
+		return _eligible;
+	}
 
 	/**
 	 * @return Whether this path is bonded as reported by the bonding layer
 	 */
-	inline bool bonded() const { return _bonded; }
+	inline bool bonded() const
+	{
+		return _bonded;
+	}
 
 	/**
 	 * @return Whether the user-specified MTU for this path (determined by MTU for parent link)
 	 */
-	inline uint16_t mtu() const { return _mtu; }
+	inline uint16_t mtu() const
+	{
+		return _mtu;
+	}
 
 	/**
 	 * @return Given link capacity as reported by the bonding layer
 	 */
-	inline uint32_t givenLinkSpeed() const { return _givenLinkSpeed; }
+	inline uint32_t givenLinkSpeed() const
+	{
+		return _givenLinkSpeed;
+	}
 
 	/**
 	 * @return Path's quality as reported by the bonding layer
 	 */
-	inline float relativeQuality() const { return _relativeQuality; }
+	inline float relativeQuality() const
+	{
+		return _relativeQuality;
+	}
 
 	/**
 	 * @return Physical interface name that this path lives on
 	 */
-	char *ifname() {
+	char* ifname()
+	{
 		return _ifname;
 	}
 
   private:
-
 	char _ifname[ZT_MAX_PHYSIFNAME] = {};
 
 	volatile int64_t _lastOut;

node/Peer.cpp

@@ -11,18 +11,19 @@
  */
 /****/
 
+#include "Peer.hpp"
+
 #include "../version.h"
 #include "Constants.hpp"
-#include "Peer.hpp"
-#include "Switch.hpp"
-#include "Network.hpp"
-#include "SelfAwareness.hpp"
-#include "Packet.hpp"
-#include "Trace.hpp"
 #include "InetAddress.hpp"
-#include "RingBuffer.hpp"
-#include "Utils.hpp"
 #include "Metrics.hpp"
+#include "Network.hpp"
+#include "Packet.hpp"
+#include "RingBuffer.hpp"
+#include "SelfAwareness.hpp"
+#include "Switch.hpp"
+#include "Trace.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 
@@ -128,7 +129,8 @@ void Peer::received(
 							break;
 						}
 					}
-				} else {
+				}
+				else {
 					break;
 				}
 			}
@@ -157,7 +159,8 @@ void Peer::received(
 								}
 							}
 						}
-					} else {
+					}
+					else {
 						replacePath = i;
 						break;
 					}
@@ -175,17 +178,20 @@ void Peer::received(
 						_bond->nominatePathToBond(_paths[replacePath].p, now);
 					}
 				}
-			} else {
+			}
+			else {
 				Mutex::Lock ltl(_lastTriedPath_m);
 
 				bool triedTooRecently = false;
 				for (std::list<std::pair<Path*, int64_t> >::iterator i(_lastTriedPath.begin()); i != _lastTriedPath.end();) {
 					if ((now - i->second) > 1000) {
 						_lastTriedPath.erase(i++);
-					} else if (i->first == path.ptr()) {
+					}
+					else if (i->first == path.ptr()) {
 						++i;
 						triedTooRecently = true;
-					} else {
+					}
+					else {
 						++i;
 					}
 				}
@@ -278,7 +284,8 @@ SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32
 					bestPath = i;
 				}
 			}
-		} else {
+		}
+		else {
 			break;
 		}
 	}
@@ -329,7 +336,8 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o
 					}
 					break;
 			}
-		} else {
+		}
+		else {
 			break;
 		}
 	}
@@ -354,7 +362,8 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o
 					}
 					break;
 			}
-		} else {
+		}
+		else {
 			break;
 		}
 	}
@@ -387,14 +396,16 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o
 				if (other->_paths[theirs].p->address().ss_family == AF_INET6) {
 					outp.append((uint8_t)16);
 					outp.append(other->_paths[theirs].p->address().rawIpData(), 16);
-				} else {
+				}
+				else {
 					outp.append((uint8_t)4);
 					outp.append(other->_paths[theirs].p->address().rawIpData(), 4);
 				}
 				outp.armor(_key, true, aesKeysIfSupported());
 				Metrics::pkt_rendezvous_out++;
 				_paths[mine].p->send(RR, tPtr, outp.data(), outp.size(), now);
-			} else {
+			}
+			else {
 				Packet outp(other->_id.address(), RR->identity.address(), Packet::VERB_RENDEZVOUS);
 				outp.append((uint8_t)0);
 				_id.address().appendTo(outp);
@@ -402,7 +413,8 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o
 				if (_paths[mine].p->address().ss_family == AF_INET6) {
 					outp.append((uint8_t)16);
 					outp.append(_paths[mine].p->address().rawIpData(), 16);
-				} else {
+				}
+				else {
 					outp.append((uint8_t)4);
 					outp.append(_paths[mine].p->address().rawIpData(), 4);
 				}
@@ -454,7 +466,8 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA
 		outp.armor(_key, false, nullptr);	// false == don't encrypt full payload, but add MAC
 		RR->node->expectReplyTo(outp.packetId());
 		RR->node->putPacket(tPtr, RR->node->lowBandwidthModeEnabled() ? localSocket : -1, atAddress, outp.data(), outp.size());
-	} else {
+	}
+	else {
 		RR->node->expectReplyTo(outp.packetId());
 		RR->sw->send(tPtr, outp, false);   // false == don't encrypt full payload, but add MAC
 	}
@@ -468,7 +481,8 @@ void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAdd
 		Metrics::pkt_echo_out++;
 		RR->node->expectReplyTo(outp.packetId());
 		RR->node->putPacket(tPtr, localSocket, atAddress, outp.data(), outp.size());
-	} else {
+	}
+	else {
 		sendHELLO(tPtr, localSocket, atAddress, now);
 	}
 }
@@ -549,7 +563,8 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
 		for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
 			if (_paths[i].p) {
 				maxPriority = std::max(_paths[i].priority, maxPriority);
-			} else {
+			}
+			else {
 				break;
 			}
 		}
@@ -564,7 +579,8 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
 						_paths[i].p->sent(now);
 						sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2;
 					}
-				} else {
+				}
+				else {
 					_paths[i] = _PeerPath();
 					deletionOccurred = true;
 				}
@@ -620,7 +636,8 @@ void Peer::clusterRedirect(void *tPtr,const SharedPtr<Path> &originatingPath,con
 					newPriority = _paths[i].priority;
 					break;
 				}
-			} else {
+			}
+			else {
 				break;
 			}
 		}
@@ -664,14 +681,14 @@ void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddres
 				_paths[i].p->sent(now);
 				_paths[i].lr = 0;	// path will not be used unless it speaks again
 			}
-		} else {
+		}
+		else {
 			break;
 		}
 	}
 }
 
-void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
+void Peer::recordOutgoingPacket(const SharedPtr<Path>& path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
-	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
 #ifndef ZT_NO_PEER_METRICS
 	_outgoing_packet++;
@@ -691,8 +708,7 @@ void Peer::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
 	}
 }
 
-void Peer::recordIncomingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
+void Peer::recordIncomingPacket(const SharedPtr<Path>& path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
-	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
 	if (_localMultipathSupported && _bond) {
 		_bond->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);