Plumb through external interface stuff

go/pkg/zerotier/localconfig.go

@@ -33,7 +33,15 @@ type LocalConfigPhysicalPathConfiguration struct {
 // LocalConfigVirtualAddressConfiguration contains settings for virtual addresses
 type LocalConfigVirtualAddressConfiguration struct {
 	// Try is a list of IPs/ports to try for this peer in addition to anything learned from roots or direct path push
-	Try []*InetAddress `json:",omitempty"`
+	Try []InetAddress `json:",omitempty"`
+}
+
+// ExternalAddress is an externally visible address
+type ExternalAddress struct {
+	InetAddress
+
+	// Permanent indicates that this address should be incorporated into this node's Locator
+	Permanent bool `json:"permanent"`
 }
 
 // LocalConfigSettings contains node settings
@@ -66,19 +74,19 @@ type LocalConfigSettings struct {
 	InterfacePrefixBlacklist []string `json:"interfacePrefixBlacklist,omitempty"`
 
 	// ExplicitAddresses are explicit IP/port addresses to advertise to other nodes, such as externally mapped ports on a router
-	ExplicitAddresses []*InetAddress `json:"explicitAddresses,omitempty"`
+	ExplicitAddresses []ExternalAddress `json:"explicitAddresses,omitempty"`
 }
 
 // LocalConfig is the local.conf file and stores local settings for the node.
 type LocalConfig struct {
 	// Physical path configurations by CIDR IP/bits
-	Physical map[string]*LocalConfigPhysicalPathConfiguration `json:"physical,omitempty"`
+	Physical map[string]LocalConfigPhysicalPathConfiguration `json:"physical,omitempty"`
 
 	// Virtual node specific configurations by 10-digit hex ZeroTier address
-	Virtual map[Address]*LocalConfigVirtualAddressConfiguration `json:"virtual,omitempty"`
+	Virtual map[Address]LocalConfigVirtualAddressConfiguration `json:"virtual,omitempty"`
 
 	// Network local configurations by 16-digit hex ZeroTier network ID
-	Network map[NetworkID]*NetworkLocalSettings `json:"network,omitempty"`
+	Network map[NetworkID]NetworkLocalSettings `json:"network,omitempty"`
 
 	// LocalConfigSettings contains other local settings for this node
 	Settings LocalConfigSettings `json:"settings,omitempty"`
@@ -87,9 +95,9 @@ type LocalConfig struct {
 // Read this local config from a file, initializing to defaults if the file does not exist
 func (lc *LocalConfig) Read(p string, saveDefaultsIfNotExist bool) error {
 	if lc.Physical == nil {
-		lc.Physical = make(map[string]*LocalConfigPhysicalPathConfiguration)
+		lc.Physical = make(map[string]LocalConfigPhysicalPathConfiguration)
-		lc.Virtual = make(map[Address]*LocalConfigVirtualAddressConfiguration)
+		lc.Virtual = make(map[Address]LocalConfigVirtualAddressConfiguration)
-		lc.Network = make(map[NetworkID]*NetworkLocalSettings)
+		lc.Network = make(map[NetworkID]NetworkLocalSettings)
 		lc.Settings.PrimaryPort = 9993
 		lc.Settings.SecondaryPort = 16384 + (rand.Int() % 16384)
 		lc.Settings.TertiaryPort = 32768 + (rand.Int() % 16384)

go/pkg/zerotier/node.go

@@ -31,6 +31,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"reflect"
 	"sort"
 	"strings"
 	"sync"
@@ -312,6 +313,8 @@ func NewNode(basePath string) (*Node, error) {
 	n.runLock.Lock() // used to block Close() until below gorountine exits
 	go func() {
 		lastMaintenanceRun := int64(0)
+		var previousExplicitExternalAddresses []ExternalAddress
+		var portsA [3]int
 		for atomic.LoadUint32(&n.running) != 0 {
 			time.Sleep(1 * time.Second)
 
@@ -346,42 +349,40 @@ func NewNode(basePath string) (*Node, error) {
 					n.networksLock.RUnlock()
 				}
 
+				interfaceAddressesChanged := false
+				ports := portsA[:0]
+				if n.localConfig.Settings.PrimaryPort > 0 && n.localConfig.Settings.PrimaryPort < 65536 {
+					ports = append(ports, n.localConfig.Settings.PrimaryPort)
+				}
+				if n.localConfig.Settings.SecondaryPort > 0 && n.localConfig.Settings.SecondaryPort < 65536 {
+					ports = append(ports, n.localConfig.Settings.SecondaryPort)
+				}
+				if n.localConfig.Settings.TertiaryPort > 0 && n.localConfig.Settings.TertiaryPort < 65536 {
+					ports = append(ports, n.localConfig.Settings.TertiaryPort)
+				}
+
 				// Open or close locally bound UDP ports for each local interface address.
 				// This opens ports if they are not already open and then closes ports if
 				// they are open but no longer seem to exist.
 				n.interfaceAddressesLock.Lock()
 				for astr, ipn := range interfaceAddresses {
 					if _, alreadyKnown := n.interfaceAddresses[astr]; !alreadyKnown {
+						interfaceAddressesChanged = true
 						ipCstr := C.CString(ipn.String())
-						if n.localConfig.Settings.PrimaryPort > 0 && n.localConfig.Settings.PrimaryPort < 65536 {
+						for _, p := range ports {
-							n.log.Printf("UDP binding to port %d on interface %s", n.localConfig.Settings.PrimaryPort, astr)
+							n.log.Printf("UDP binding to port %d on interface %s", p, astr)
-							C.ZT_GoNode_phyStartListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.PrimaryPort))
+							C.ZT_GoNode_phyStartListen(n.gn, nil, ipCstr, C.int(p))
-						}
-						if n.localConfig.Settings.SecondaryPort > 0 && n.localConfig.Settings.SecondaryPort < 65536 {
-							n.log.Printf("UDP binding to port %d on interface %s", n.localConfig.Settings.SecondaryPort, astr)
-							C.ZT_GoNode_phyStartListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.SecondaryPort))
-						}
-						if n.localConfig.Settings.TertiaryPort > 0 && n.localConfig.Settings.TertiaryPort < 65536 {
-							n.log.Printf("UDP binding to port %d on interface %s", n.localConfig.Settings.TertiaryPort, astr)
-							C.ZT_GoNode_phyStartListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.TertiaryPort))
 						}
 						C.free(unsafe.Pointer(ipCstr))
 					}
 				}
 				for astr, ipn := range n.interfaceAddresses {
 					if _, stillPresent := interfaceAddresses[astr]; !stillPresent {
+						interfaceAddressesChanged = true
 						ipCstr := C.CString(ipn.String())
-						if n.localConfig.Settings.PrimaryPort > 0 && n.localConfig.Settings.PrimaryPort < 65536 {
+						for _, p := range ports {
-							n.log.Printf("UDP closing socket bound to port %d on interface %s", n.localConfig.Settings.PrimaryPort, astr)
+							n.log.Printf("UDP closing socket bound to port %d on interface %s", p, astr)
-							C.ZT_GoNode_phyStopListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.PrimaryPort))
+							C.ZT_GoNode_phyStopListen(n.gn, nil, ipCstr, C.int(p))
-						}
-						if n.localConfig.Settings.SecondaryPort > 0 && n.localConfig.Settings.SecondaryPort < 65536 {
-							n.log.Printf("UDP closing socket bound to port %d on interface %s", n.localConfig.Settings.SecondaryPort, astr)
-							C.ZT_GoNode_phyStopListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.SecondaryPort))
-						}
-						if n.localConfig.Settings.TertiaryPort > 0 && n.localConfig.Settings.TertiaryPort < 65536 {
-							n.log.Printf("UDP closing socket bound to port %d on interface %s", n.localConfig.Settings.TertiaryPort, astr)
-							C.ZT_GoNode_phyStopListen(n.gn, nil, ipCstr, C.int(n.localConfig.Settings.TertiaryPort))
 						}
 						C.free(unsafe.Pointer(ipCstr))
 					}
@@ -389,6 +390,41 @@ func NewNode(basePath string) (*Node, error) {
 				n.interfaceAddresses = interfaceAddresses
 				n.interfaceAddressesLock.Unlock()
 
+				// Update node's understanding of our interface addressaes if they've changed
+				if interfaceAddressesChanged || reflect.DeepEqual(n.localConfig.Settings.ExplicitAddresses, previousExplicitExternalAddresses) {
+					externalAddresses := make(map[[3]uint64]*ExternalAddress)
+					for _, ip := range interfaceAddresses {
+						for _, p := range ports {
+							a := &ExternalAddress{
+								InetAddress: InetAddress{
+									IP:   ip,
+									Port: p,
+								},
+								Permanent: false,
+							}
+							externalAddresses[a.key()] = a
+						}
+					}
+					for _, a := range n.localConfig.Settings.ExplicitAddresses {
+						externalAddresses[a.key()] = &a
+					}
+					if len(externalAddresses) > 0 {
+						cAddrs := make([]C.ZT_InterfaceAddress, len(externalAddresses))
+						cAddrCount := 0
+						for _, a := range externalAddresses {
+							makeSockaddrStorage(a.IP, a.Port, &(cAddrs[cAddrCount].address))
+							cAddrs[cAddrCount].permanent = 0
+							if a.Permanent {
+								cAddrs[cAddrCount].permanent = 1
+							}
+							cAddrCount++
+						}
+						C.ZT_Node_setInterfaceAddresses(unsafe.Pointer(n.zn), &cAddrs[0], C.uint(cAddrCount))
+					} else {
+						C.ZT_Node_setInterfaceAddresses(unsafe.Pointer(n.zn), nil, 0)
+					}
+				}
+
 				// Trim log if it's gone over its size limit
 				if n.localConfig.Settings.LogSizeMax > 0 && n.logW != nil {
 					_ = n.logW.trim(n.localConfig.Settings.LogSizeMax*1024, 0.5, true)
@@ -462,7 +498,7 @@ func (n *Node) SetLocalConfig(lc *LocalConfig) (restartRequired bool, err error)
 	for nid, nc := range lc.Network {
 		nw := n.networks[nid]
 		if nw != nil {
-			nw.SetLocalSettings(nc)
+			nw.SetLocalSettings(&nc)
 		}
 	}
 
@@ -722,7 +758,7 @@ func (n *Node) pathLookup(ztAddress Address) (net.IP, int) {
 	n.localConfigLock.RLock()
 	defer n.localConfigLock.RUnlock()
 	virt := n.localConfig.Virtual[ztAddress]
-	if virt != nil && len(virt.Try) > 0 {
+	if len(virt.Try) > 0 {
 		idx := rand.Int() % len(virt.Try)
 		return virt.Try[idx].IP, virt.Try[idx].Port
 	}

include/ZeroTierCore.h

@@ -1123,10 +1123,27 @@ typedef struct
 	unsigned long networkCount;
 } ZT_VirtualNetworkList;
 
+/**
+ * Address where this node could be reached via an external interface
+ */
+typedef struct
+{
+	/**
+	 * IP and port as would be reachable by external nodes
+	 */
+	struct sockaddr_storage address;
+
+	/**
+	 * If nonzero this address is static and can be incorporated into this node's Locator
+	 */
+	int permanent;
+} ZT_InterfaceAddress;
+
 /**
  * Physical path configuration
  */
-typedef struct {
+typedef struct
+{
 	/**
 	 * If non-zero set this physical network path to be trusted to disable encryption and authentication
 	 */
@@ -1312,6 +1329,15 @@ enum ZT_StateObjectType
 	 */
 	ZT_STATE_OBJECT_IDENTITY_SECRET = 2,
 
+	/**
+	 * This node's locator
+	 *
+	 * Object ID: 0
+	 * Canonical path: <HOME>/locator
+	 * Persistence: optional
+	 */
+	ZT_STATE_OBJECT_LOCATOR = 3,
+
 	/**
 	 * Peer and related state
 	 *
@@ -1335,7 +1361,7 @@ enum ZT_StateObjectType
 	 *
 	 * Object ID: 0
 	 * Canonical path: <HOME>/roots
-	 * Persitence: required if root settings should persist
+	 * Persistence: required if root settings should persist
 	 */
 	ZT_STATE_OBJECT_ROOTS = 7
 };
@@ -1958,33 +1984,13 @@ ZT_SDK_API void ZT_Node_setNetworkUserPtr(ZT_Node *node,uint64_t nwid,void *ptr)
 ZT_SDK_API void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
 
 /**
- * Add a local interface address
+ * Set external interface addresses where this node could be reached
  *
- * This is used to make ZeroTier aware of those local interface addresses
+ * @param node Node instance
- * that you wish to use for ZeroTier communication. This is optional, and if
+ * @param addrs Addresses
- * it is not used ZeroTier will rely upon upstream peers (and roots) to
+ * @param addrCount Number of items in addrs[]
- * perform empirical address discovery and NAT traversal. But the use of this
- * method is recommended as it improves peer discovery when both peers are
- * on the same LAN.
- *
- * It is the responsibility of the caller to take care that these are never
- * ZeroTier interface addresses, whether these are assigned by ZeroTier or
- * are otherwise assigned to an interface managed by this ZeroTier instance.
- * This can cause recursion or other undesirable behavior.
- *
- * This returns a boolean indicating whether or not the address was
- * accepted. ZeroTier will only communicate over certain address types
- * and (for IP) address classes.
- *
- * @param addr Local interface address
- * @return Boolean: non-zero if address was accepted and added
  */
-ZT_SDK_API int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr);
+ZT_SDK_API void ZT_Node_setInterfaceAddresses(ZT_Node *node,const ZT_InterfaceAddress *addrs,unsigned int addrCount);
-
-/**
- * Clear local interface addresses
- */
-ZT_SDK_API void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node);
 
 /**
  * Send a VERB_USER_MESSAGE to another ZeroTier node

node/Node.cpp

@@ -594,22 +594,23 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
+void Node::setInterfaceAddresses(const ZT_InterfaceAddress *addrs,unsigned int addrCount)
-{
-	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
-		Mutex::Lock _l(_localInterfaceAddresses_m);
-		if (std::find(_localInterfaceAddresses.begin(),_localInterfaceAddresses.end(),*(reinterpret_cast<const InetAddress *>(addr))) == _localInterfaceAddresses.end()) {
-			_localInterfaceAddresses.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
-			return 1;
-		}
-	}
-	return 0;
-}
-
-void Node::clearLocalInterfaceAddresses()
 {
 	Mutex::Lock _l(_localInterfaceAddresses_m);
 	_localInterfaceAddresses.clear();
+	for(unsigned int i=0;i<addrCount;++i) {
+		if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(&addrs[i].address)))) {
+			bool dupe = false;
+			for(unsigned int j=0;j<i;++j) {
+				if (*(reinterpret_cast<const InetAddress *>(&addrs[j].address)) == *(reinterpret_cast<const InetAddress *>(&addrs[i].address))) {
+					dupe = true;
+					break;
+				}
+			}
+			if (!dupe)
+				_localInterfaceAddresses.push_back(addrs[i]);
+		}
+	}
 }
 
 int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len)
@@ -993,19 +994,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
+void ZT_Node_setInterfaceAddresses(ZT_Node *node,const ZT_InterfaceAddress *addrs,unsigned int addrCount)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
+		reinterpret_cast<ZeroTier::Node *>(node)->setInterfaceAddresses(addrs,addrCount);
-	} catch ( ... ) {
-		return 0;
-	}
-}
-
-void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
-{
-	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
 	} catch ( ... ) {}
 }
 

node/Node.hpp

@@ -167,12 +167,14 @@ public:
 		return nw;
 	}
 
-	ZT_ALWAYS_INLINE std::vector<InetAddress> directPaths() const
+	ZT_ALWAYS_INLINE std::vector<ZT_InterfaceAddress> directPaths() const
 	{
 		Mutex::Lock _l(_localInterfaceAddresses_m);
 		return _localInterfaceAddresses;
 	}
 
+	void setInterfaceAddresses(const ZT_InterfaceAddress *addrs,unsigned int addrCount);
+
 	ZT_ALWAYS_INLINE void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ev,md); }
 	ZT_ALWAYS_INLINE void configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { _cb.virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,nwid,nuptr,op,nc); }
 	ZT_ALWAYS_INLINE bool online() const { return _online; }
@@ -284,15 +286,12 @@ private:
 	Hashtable< _LocalControllerAuth,int64_t > _localControllerAuthorizations;
 	Mutex _localControllerAuthorizations_m;
 
-	// Curreently joined networks
 	Hashtable< uint64_t,SharedPtr<Network> > _networks;
 	Mutex _networks_m;
 
-	// Local interface addresses as reported by the code harnessing this Node
+	std::vector<ZT_InterfaceAddress> _localInterfaceAddresses;
-	std::vector<InetAddress> _localInterfaceAddresses;
 	Mutex _localInterfaceAddresses_m;
 
-	// Lock to ensure processBackgroundTasks never gets run concurrently
 	Mutex _backgroundTasksLock;
 
 	uint8_t _multipathMode;

node/Peer.cpp

@@ -170,20 +170,30 @@ void Peer::received(
 	const int64_t sinceLastPush = now - _lastDirectPathPushSent;
 	if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
 		_lastDirectPathPushSent = now;
-		std::vector<InetAddress> pathsToPush(RR->node->directPaths());
+		std::vector<ZT_InterfaceAddress> pathsToPush(RR->node->directPaths());
 		if (pathsToPush.size() > 0) {
-			std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
+			std::vector<ZT_InterfaceAddress>::const_iterator p(pathsToPush.begin());
 			while (p != pathsToPush.end()) {
 				ScopedPtr<Packet> outp(new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS));
 				outp->addSize(2); // leave room for count
 				unsigned int count = 0;
 				while ((p != pathsToPush.end())&&((outp->size() + 24) < 1200)) {
 					uint8_t addressType = 4;
-					switch(p->ss_family) {
+					uint8_t addressLength = 6;
+					unsigned int ipLength = 4;
+					const void *rawIpData;
+					const void *rawIpPort;
+					switch(p->address.ss_family) {
 						case AF_INET:
+							rawIpData = &(reinterpret_cast<const struct sockaddr_in *>(&(p->address))->sin_addr.s_addr);
+							rawIpPort = &(reinterpret_cast<const struct sockaddr_in *>(&(p->address))->sin_port);
 							break;
 						case AF_INET6:
+							rawIpData = reinterpret_cast<const struct sockaddr_in6 *>(&(p->address))->sin6_addr.s6_addr;
+							rawIpPort = &(reinterpret_cast<const struct sockaddr_in6 *>(&(p->address))->sin6_port);
 							addressType = 6;
+							addressLength = 18;
+							ipLength = 16;
 							break;
 						default: // we currently only push IP addresses
 							++p;
@@ -193,9 +203,9 @@ void Peer::received(
 					outp->append((uint8_t)0); // no flags
 					outp->append((uint16_t)0); // no extensions
 					outp->append(addressType);
-					outp->append((uint8_t)((addressType == 4) ? 6 : 18));
+					outp->append(addressLength);
-					outp->append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+					outp->append(rawIpData,ipLength);
-					outp->append((uint16_t)p->port());
+					outp->append(rawIpPort,2);
 
 					++count;
 					++p;