From 3c9ea2b6674819612cd88a2e141e0f8f633b06cd Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Mon, 5 Dec 2022 13:21:05 -0800 Subject: [PATCH 01/16] Add low-bandwidth mode --- node/Multicaster.cpp | 3 ++- node/Node.cpp | 20 ++++++++++++++++---- node/Node.hpp | 11 +++++++++++ node/Peer.cpp | 15 ++++++++++----- node/Peer.hpp | 2 ++ service/OneService.cpp | 1 + 6 files changed, 42 insertions(+), 10 deletions(-) diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 4856b88ee..d93b2ae10 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -268,7 +268,8 @@ void Multicaster::send( const unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1; - if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) { + int timerScale = RR->node->lowBandwidthModeEnabled() ? 3 : 1; + if ((gs.members.empty())||((now - gs.lastExplicitGather) >= (ZT_MULTICAST_EXPLICIT_GATHER_DELAY * timerScale))) { gs.lastExplicitGather = now; Address explicitGatherPeers[16]; diff --git a/node/Node.cpp b/node/Node.cpp index 019a8afca..7bdf331dc 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -202,6 +202,14 @@ public: { const std::vector *const alwaysContactEndpoints = _alwaysContact.get(p->address()); if (alwaysContactEndpoints) { + + // Contact upstream peers as infrequently as possible + ZT_PeerRole role = RR->topology->role(p->address()); + int roleBasedTimerScale = (role == ZT_PEER_ROLE_LEAF) ? 2 : 16; + if ((RR->node->now() - p->lastSentFullHello()) <= (ZT_PATH_HEARTBEAT_PERIOD * roleBasedTimerScale)) { + return; + } + const unsigned int sent = p->doPingAndKeepalive(_tPtr,_now); bool contacted = (sent != 0); @@ -262,7 +270,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 } } - unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL; + unsigned long timeUntilNextPingCheck = _lowBandwidthMode ? (ZT_PING_CHECK_INVERVAL * 5) : ZT_PING_CHECK_INVERVAL; const int64_t timeSinceLastPingCheck = now - _lastPingCheck; if (timeSinceLastPingCheck >= timeUntilNextPingCheck) { try { @@ -309,6 +317,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 // Get peers we should stay connected to according to network configs // Also get networks and whether they need config so we only have to do one pass over networks + int timerScale = _lowBandwidthMode ? 64 : 1; std::vector< std::pair< SharedPtr,bool > > networkConfigNeeded; { Mutex::Lock l(_networks_m); @@ -317,7 +326,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 SharedPtr *network = (SharedPtr *)0; while (i.next(nwid,network)) { (*network)->config().alwaysContactAddresses(alwaysContact); - networkConfigNeeded.push_back( std::pair< SharedPtr,bool >(*network,(((now - (*network)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!(*network)->hasConfig()))) ); + networkConfigNeeded.push_back( std::pair< SharedPtr,bool >(*network,(((now - (*network)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY * timerScale)||(!(*network)->hasConfig()))) ); } } @@ -336,9 +345,12 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 // Refresh network config or broadcast network updates to members as needed for(std::vector< std::pair< SharedPtr,bool > >::const_iterator n(networkConfigNeeded.begin());n!=networkConfigNeeded.end();++n) { - if (n->second) + if (n->second) { n->first->requestConfiguration(tptr); - n->first->sendUpdatesToMembers(tptr); + } + if (! _lowBandwidthMode) { + n->first->sendUpdatesToMembers(tptr); + } } // Update online status, post status change as event diff --git a/node/Node.hpp b/node/Node.hpp index 834f50cc9..0ddd5cb7b 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -269,6 +269,16 @@ public: _stats.inVerbBytes[v] += (uint64_t)bytes; } + inline void setLowBandwidthMode(bool isEnabled) + { + _lowBandwidthMode = isEnabled; + } + + inline bool lowBandwidthModeEnabled() + { + return _lowBandwidthMode; + } + private: RuntimeEnvironment _RR; RuntimeEnvironment *RR; @@ -316,6 +326,7 @@ private: int64_t _lastMemoizedTraceSettings; volatile int64_t _prngState[2]; bool _online; + bool _lowBandwidthMode; }; } // namespace ZeroTier diff --git a/node/Peer.cpp b/node/Peer.cpp index 99fa8d277..c1dc124c6 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -219,11 +219,15 @@ void Peer::received( // is done less frequently. if (this->trustEstablished(now)) { const int64_t sinceLastPush = now - _lastDirectPathPushSent; - if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) { + bool lowBandwidth = RR->node->lowBandwidthModeEnabled(); + int timerScale = lowBandwidth ? 16 : 1; + if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH * timerScale : ZT_DIRECT_PATH_PUSH_INTERVAL)) { _lastDirectPathPushSent = now; std::vector pathsToPush(RR->node->directPaths()); - std::vector ma = RR->sa->whoami(); - pathsToPush.insert(pathsToPush.end(), ma.begin(), ma.end()); + if (! lowBandwidth) { + std::vector ma = RR->sa->whoami(); + pathsToPush.insert(pathsToPush.end(), ma.begin(), ma.end()); + } if (!pathsToPush.empty()) { std::vector::const_iterator p(pathsToPush.begin()); while (p != pathsToPush.end()) { @@ -453,7 +457,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA if (atAddress) { outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC RR->node->expectReplyTo(outp.packetId()); - RR->node->putPacket(tPtr,-1,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,RR->node->lowBandwidthModeEnabled() ? localSocket : -1,atAddress,outp.data(),outp.size()); } else { RR->node->expectReplyTo(outp.packetId()); RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC @@ -477,8 +481,9 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now) if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { _lastTriedMemorizedPath = now; InetAddress mp; - if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp)) + if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp)) { attemptToContactAt(tPtr,-1,mp,now,true); + } } } diff --git a/node/Peer.hpp b/node/Peer.hpp index 0192143e3..668bbbee9 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -302,6 +302,8 @@ public: */ inline int64_t isActive(int64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline int64_t lastSentFullHello() { return _lastSentFullHello; } + /** * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown / no paths */ diff --git a/service/OneService.cpp b/service/OneService.cpp index daba4d99b..21717e3a5 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2122,6 +2122,7 @@ public: fprintf(stderr,"WARNING: using manually-specified secondary and/or tertiary ports. This can cause NAT issues." ZT_EOL_S); } _portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true); + _node->setLowBandwidthMode(OSUtils::jsonBool(settings["lowBandwidthMode"],false)); #ifndef ZT_SDK const std::string up(OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT)); From 7587ef51362e2e4fab774965d0513ca95a7d26ba Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Mon, 12 Dec 2022 09:44:31 +0100 Subject: [PATCH 02/16] basic builds on github (#1815) Initial Github Actions build --- .github/workflows/build.yml | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..06f416606 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,40 @@ +on: [ push, pull_request ] + +jobs: + build_ubuntu: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: make + run: make + + - name: selftest + run: make selftest + + # build_macos: + # runs-on: macos-latest + # steps: + # - name: checkout + # uses: actions/checkout@v3 + + # - name: make + # run: make + + # - name: selftest + # run: make selftest + + # build_windows: + # runs-on: windows-latest + # steps: + # - name: checkout + # uses: actions/checkout@v3 + + # - name: setup msbuild + # uses: microsoft/setup-msbuild@v1.1.3 + + # - name: msbuild + # run: | + # msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild + From 117d7194af58ccf64f7cc11ffcb11a730ebece93 Mon Sep 17 00:00:00 2001 From: Travis LaDuke Date: Tue, 13 Dec 2022 02:52:21 -0800 Subject: [PATCH 03/16] Improve default route on macOS (#1680) re: issue #1088 and probably: https://discuss.zerotier.com/t/default-route-issue-osx-monterey-m1/6974 // current zerotier. // allow default adds two sets of routes. ``` netstat -rnfinet | grep "/1\|default" | sort 0/1 10.2.0.2 UGScg feth4823 0/1 192.168.82.1 UGScIg en7 128.0/1 10.2.0.2 UGSc feth4823 128.0/1 192.168.82.1 UGScI en7 default 192.168.82.1 UGScg en7 ``` Then, something chaotic happens eventually, and networking stops working. // after patch ``` netstat -rnfinet | grep "/1\|default" | sort 0/1 10.2.0.2 UGScg feth4823 128.0/1 10.2.0.2 UGSc feth4823 default 192.168.82.1 UGScg en7 ``` After the change, I can still: - use default route - route to other subnets I tested on high sierra through monterey and on freebsd13.1 --- osdep/ManagedRoute.cpp | 70 ------------------------------------------ 1 file changed, 70 deletions(-) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 325f4c803..64342e2c2 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -439,76 +439,6 @@ bool ManagedRoute::sync() #ifdef __BSD__ // ------------------------------------------------------------ - if (_device[0]) { - bool haveDevice = false; - struct ifaddrs *ifa = (struct ifaddrs *)0; - if (!getifaddrs(&ifa)) { - struct ifaddrs *p = ifa; - while (p) { - if ((p->ifa_name)&&(!strcmp(_device, p->ifa_name))) { - haveDevice = true; - break; - } - p = p->ifa_next; - } - freeifaddrs(ifa); - } - if (!haveDevice) - return false; - } - - // Find lowest metric system route that this route should override (if any) - InetAddress newSystemVia; - char newSystemDevice[128]; - newSystemDevice[0] = (char)0; - int systemMetric = 9999999; - std::vector<_RTE> rtes(_getRTEs(_target,false)); - for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) { - if (r->via) { - if ( ((!newSystemVia)||(r->metric < systemMetric)) && (strcmp(r->device,_device) != 0) ) { - newSystemVia = r->via; - Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device); - systemMetric = r->metric; - } - } - } - - // Get device corresponding to route if we don't have that already - if ((newSystemVia)&&(!newSystemDevice[0])) { - rtes = _getRTEs(newSystemVia,true); - for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) { - if ( (r->device[0]) && (strcmp(r->device,_device) != 0) ) { - Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device); - break; - } - } - } - if (!newSystemDevice[0]) - newSystemVia.zero(); - - // Shadow system route if it exists, also delete any obsolete shadows - // and replace them with the new state. sync() is called periodically to - // allow us to do that if underlying connectivity changes. - if ((_systemVia != newSystemVia)||(strcmp(_systemDevice,newSystemDevice) != 0)) { - if (_systemVia) { - _routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0); - if (rightt) - _routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0); - } - - _systemVia = newSystemVia; - Utils::scopy(_systemDevice,sizeof(_systemDevice),newSystemDevice); - - if (_systemVia) { - _routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0); - //_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0); - if (rightt) { - _routeCmd("add",rightt,_systemVia,_systemDevice,(const char *)0); - //_routeCmd("change",rightt,_systemVia,_systemDevice,(const char *)0); - } - } - } - //if (!_applied.count(leftt)) { _applied[leftt] = !_via; //_routeCmd("delete",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device); From 0210ba9c13bae2c46408129ae6e56934fe17d63c Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Tue, 13 Dec 2022 16:16:17 +0100 Subject: [PATCH 04/16] enabling mac and windows github builds --- .github/workflows/build.yml | 39 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06f416606..54aa2a058 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,28 +13,27 @@ jobs: - name: selftest run: make selftest - # build_macos: - # runs-on: macos-latest - # steps: - # - name: checkout - # uses: actions/checkout@v3 + build_macos: + runs-on: macos-latest + steps: + - name: checkout + uses: actions/checkout@v3 - # - name: make - # run: make + - name: make + run: make - # - name: selftest - # run: make selftest + - name: selftest + run: make selftest - # build_windows: - # runs-on: windows-latest - # steps: - # - name: checkout - # uses: actions/checkout@v3 + build_windows: + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v3 - # - name: setup msbuild - # uses: microsoft/setup-msbuild@v1.1.3 + - name: setup msbuild + uses: microsoft/setup-msbuild@v1.1.3 - # - name: msbuild - # run: | - # msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild - + - name: msbuild + run: | + msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild From d2de0292ccd4b22227b653624de0bb893d850dfe Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Tue, 13 Dec 2022 16:49:08 +0100 Subject: [PATCH 05/16] mac and windows github builds (#1817) --- .github/workflows/build.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54aa2a058..d60c4709d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,36 +1,44 @@ -on: [ push, pull_request ] - +on: [ push ] + jobs: build_ubuntu: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 - - - name: make - run: make - - - name: selftest - run: make selftest + - name: checkout + uses: actions/checkout@v3 + + - name: make + run: make + + - name: selftest + run: make selftest build_macos: runs-on: macos-latest steps: - name: checkout uses: actions/checkout@v3 - + + - name: Install Rust Toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-apple-darwin + override: true + components: rustfmt, clippy + - name: make run: make - + - name: selftest run: make selftest - + build_windows: runs-on: windows-latest steps: - name: checkout uses: actions/checkout@v3 - + - name: setup msbuild uses: microsoft/setup-msbuild@v1.1.3 From e0e91e83976d91fc5c3ad60136bf5fad420fdeff Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Tue, 13 Dec 2022 17:20:52 +0100 Subject: [PATCH 06/16] disabling windows builds --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d60c4709d..db836c6ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,15 +33,15 @@ jobs: - name: selftest run: make selftest - build_windows: - runs-on: windows-latest - steps: - - name: checkout - uses: actions/checkout@v3 + # build_windows: + # runs-on: windows-latest + # steps: + # - name: checkout + # uses: actions/checkout@v3 - - name: setup msbuild - uses: microsoft/setup-msbuild@v1.1.3 + # - name: setup msbuild + # uses: microsoft/setup-msbuild@v1.1.3 - - name: msbuild - run: | - msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild + # - name: msbuild + # run: | + # msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild From 10170b41c30d4bffde3a691562cf41c8afc823cd Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Tue, 13 Dec 2022 18:32:07 +0100 Subject: [PATCH 07/16] Revert "Improve default route on macOS (#1680)" This reverts commit 117d7194af58ccf64f7cc11ffcb11a730ebece93. --- osdep/ManagedRoute.cpp | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 64342e2c2..325f4c803 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -439,6 +439,76 @@ bool ManagedRoute::sync() #ifdef __BSD__ // ------------------------------------------------------------ + if (_device[0]) { + bool haveDevice = false; + struct ifaddrs *ifa = (struct ifaddrs *)0; + if (!getifaddrs(&ifa)) { + struct ifaddrs *p = ifa; + while (p) { + if ((p->ifa_name)&&(!strcmp(_device, p->ifa_name))) { + haveDevice = true; + break; + } + p = p->ifa_next; + } + freeifaddrs(ifa); + } + if (!haveDevice) + return false; + } + + // Find lowest metric system route that this route should override (if any) + InetAddress newSystemVia; + char newSystemDevice[128]; + newSystemDevice[0] = (char)0; + int systemMetric = 9999999; + std::vector<_RTE> rtes(_getRTEs(_target,false)); + for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) { + if (r->via) { + if ( ((!newSystemVia)||(r->metric < systemMetric)) && (strcmp(r->device,_device) != 0) ) { + newSystemVia = r->via; + Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device); + systemMetric = r->metric; + } + } + } + + // Get device corresponding to route if we don't have that already + if ((newSystemVia)&&(!newSystemDevice[0])) { + rtes = _getRTEs(newSystemVia,true); + for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) { + if ( (r->device[0]) && (strcmp(r->device,_device) != 0) ) { + Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device); + break; + } + } + } + if (!newSystemDevice[0]) + newSystemVia.zero(); + + // Shadow system route if it exists, also delete any obsolete shadows + // and replace them with the new state. sync() is called periodically to + // allow us to do that if underlying connectivity changes. + if ((_systemVia != newSystemVia)||(strcmp(_systemDevice,newSystemDevice) != 0)) { + if (_systemVia) { + _routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0); + if (rightt) + _routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0); + } + + _systemVia = newSystemVia; + Utils::scopy(_systemDevice,sizeof(_systemDevice),newSystemDevice); + + if (_systemVia) { + _routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0); + //_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0); + if (rightt) { + _routeCmd("add",rightt,_systemVia,_systemDevice,(const char *)0); + //_routeCmd("change",rightt,_systemVia,_systemDevice,(const char *)0); + } + } + } + //if (!_applied.count(leftt)) { _applied[leftt] = !_via; //_routeCmd("delete",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device); From 3e41163bbea2a7ff79213c51462b7bf4b2adb4f6 Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Wed, 14 Dec 2022 11:52:30 +0100 Subject: [PATCH 08/16] fixing windows github builds (#1818) --- .github/workflows/build.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index db836c6ca..8d9dc461d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,6 @@ jobs: toolchain: stable target: aarch64-apple-darwin override: true - components: rustfmt, clippy - name: make run: make @@ -33,15 +32,20 @@ jobs: - name: selftest run: make selftest - # build_windows: - # runs-on: windows-latest - # steps: - # - name: checkout - # uses: actions/checkout@v3 + build_windows: + runs-on: windows-latest + steps: + - name: gitconfig + run: | + git config --global core.autocrlf false + git config --global core.eol lf - # - name: setup msbuild - # uses: microsoft/setup-msbuild@v1.1.3 + - name: checkout + uses: actions/checkout@v3 - # - name: msbuild - # run: | - # msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild + - name: setup msbuild + uses: microsoft/setup-msbuild@v1.1.3 + + - name: msbuild + run: | + msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild From a558bd93121fc7be31e67a683aa75b44ebb5e16f Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Wed, 14 Dec 2022 19:10:08 +0100 Subject: [PATCH 09/16] cache cago on github actions (#1819) --- .github/workflows/build.yml | 97 +++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d9dc461d..1f76d5b67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,33 +4,73 @@ jobs: build_ubuntu: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 - - - name: make - run: make - - - name: selftest - run: make selftest - - build_macos: - runs-on: macos-latest - steps: + - name: gitconfig + run: | + git config --global core.autocrlf false + git config --global core.eol lf - name: checkout uses: actions/checkout@v3 - - - name: Install Rust Toolchain + - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: stable target: aarch64-apple-darwin override: true + components: rustfmt, clippy + - name: Set up cargo cache + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- - name: make run: make - - name: selftest - run: make selftest + run: | + make selftest + ./zerotier-selftest + + build_macos: + runs-on: macos-latest + steps: + - name: gitconfig + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: checkout + uses: actions/checkout@v3 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-apple-darwin + override: true + components: rustfmt, clippy + - name: Set up cargo cache + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- + - name: make + run: make + - name: selftest + run: | + make selftest + ./zerotier-selftest build_windows: runs-on: windows-latest @@ -39,13 +79,30 @@ jobs: run: | git config --global core.autocrlf false git config --global core.eol lf - - name: checkout uses: actions/checkout@v3 - + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-apple-darwin + override: true + components: rustfmt, clippy + - name: Set up cargo cache + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- - name: setup msbuild uses: microsoft/setup-msbuild@v1.1.3 - - name: msbuild run: | - msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild + msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release /property:Platform=x64 /t:ZeroTierOne:Rebuild + From 4959d8079bec7f9cf3dcad8224bf0b4db955bd1b Mon Sep 17 00:00:00 2001 From: travis laduke Date: Mon, 19 Dec 2022 13:26:29 -0800 Subject: [PATCH 10/16] enable ICMP in windows firewall --- osdep/WinFWHelper.cpp | 172 ++++++++++++++++++ osdep/WinFWHelper.hpp | 31 ++++ service/OneService.cpp | 19 +- windows/ZeroTierOne/ZeroTierOne.vcxproj | 2 + .../ZeroTierOne/ZeroTierOne.vcxproj.filters | 6 + 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 osdep/WinFWHelper.cpp create mode 100644 osdep/WinFWHelper.hpp diff --git a/osdep/WinFWHelper.cpp b/osdep/WinFWHelper.cpp new file mode 100644 index 000000000..40f38977e --- /dev/null +++ b/osdep/WinFWHelper.cpp @@ -0,0 +1,172 @@ +#include "WinFWHelper.hpp" + + +namespace ZeroTier { + + + +void ZeroTier::WinFWHelper::newICMPRule(const InetAddress& ip, uint64_t nwid) +{ + char nwString[32] = { 0 }; + char ipbuf[64]; + + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + ip.toString(ipbuf); + + if (ip.isV4()) { + WinFWHelper::newICMPv4Rule(ipbuf, nwid); + } + else { + WinFWHelper::newICMPv6Rule(ipbuf, nwid); + } +} + +void ZeroTier::WinFWHelper::removeICMPRule(const InetAddress& ip, uint64_t nwid) +{ + char nwString[32] = { 0 }; + char ipbuf[64]; + + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + ip.toString(ipbuf); + + if (ip.isV4()) { + WinFWHelper::removeICMPv4Rule(ipbuf, nwid); + } + else { + WinFWHelper::removeICMPv6Rule(ipbuf, nwid); + } +} + + +void WinFWHelper::newICMPv4Rule(std::string address, uint64_t nwid) +{ + // allows icmp, scoped to a specific ip address and interface name + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "New-NetFirewallRule -DisplayName zerotier-icmpv4-)" + nwString2 + address + + R"( -InterfaceAlias 'ZeroTier One `[)" + nwString2 + R"(`]')" + + " -Protocol ICMPv4 -Action Allow" + + " -LocalAddress " + address + "\"\r\n"; + + _run(cmd); +} + +void WinFWHelper::newICMPv6Rule(std::string address, uint64_t nwid) +{ + // allows icmp, scoped to a specific ip address and interface name + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "New-NetFirewallRule -DisplayName zerotier-icmpv6-)" + nwString2 + address + + R"( -InterfaceAlias 'ZeroTier One `[)" + nwString2 + R"(`]')" + + " -Protocol ICMPv6 -Action Allow" + + " -LocalAddress " + address + "\"\r\n"; + + _run(cmd); +} + +void WinFWHelper::removeICMPv4Rule(std::string addr, uint64_t nwid) +{ + // removes 1 icmp firewall rule + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "Remove-NetFirewallRule -DisplayName zerotier-icmpv4-)" + nwString2 + addr + + "\"\r\n"; + + _run(cmd); +} + +void WinFWHelper::removeICMPv6Rule(std::string addr, uint64_t nwid) +{ + // removes 1 icmp firewall rule + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "Remove-NetFirewallRule -DisplayName zerotier-icmpv6-)" + nwString2 + addr + + "\"\r\n"; + + _run(cmd); +} + +void WinFWHelper::removeICMPv4Rules(uint64_t nwid) +{ + // removes all icmp firewall rules for this network id + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "Remove-NetFirewallRule -DisplayName zerotier-icmpv4-)" + nwString2 + "*\" \r\n"; + + _run(cmd); +} + +void WinFWHelper::removeICMPv6Rules(uint64_t nwid) +{ + // removes all icmp firewall rules for this network id + + char nwString[32] = { 0 }; + sprintf(nwString, "%.16llx", nwid); + std::string nwString2 = { nwString }; + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "Remove-NetFirewallRule -DisplayName zerotier-icmpv6-)" + nwString2 + "*\" \r\n"; + + _run(cmd); +} + +void WinFWHelper::removeICMPRules() +{ + // removes all icmp firewall rules for all networks + + std::string cmd = R"(C:\Windows\System32\WindowsPowershell\v1.0\powershell.exe "Remove-NetFirewallRule -DisplayName zerotier-icmp*)" + std::string("\r\n"); + + _run(cmd); +} + +void WinFWHelper::removeICMPRules(uint64_t nwid) +{ + // removes all icmp firewall rules for this network + WinFWHelper::removeICMPv4Rules(nwid); + WinFWHelper::removeICMPv6Rules(nwid); +} + + + +void WinFWHelper::_run(std::string cmd) +{ + + #ifdef ZT_DEBUG + fprintf(stderr, cmd.c_str()); + #endif + + STARTUPINFOA startupInfo; + PROCESS_INFORMATION processInfo; + startupInfo.cb = sizeof(startupInfo); + memset(&startupInfo, 0, sizeof(STARTUPINFOA)); + memset(&processInfo, 0, sizeof(PROCESS_INFORMATION)); + + if (CreateProcessA(NULL, (LPSTR)cmd.c_str(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { + WaitForSingleObject(processInfo.hProcess, INFINITE); + + CloseHandle(processInfo.hProcess); + CloseHandle(processInfo.hThread); + } +} + + + +} // namespace ZeroTier \ No newline at end of file diff --git a/osdep/WinFWHelper.hpp b/osdep/WinFWHelper.hpp new file mode 100644 index 000000000..a8d9e27aa --- /dev/null +++ b/osdep/WinFWHelper.hpp @@ -0,0 +1,31 @@ +#ifndef WIN_FW_HELPER_H_ +#define WIN_FW_HELPER_H_ + +#include "../node/InetAddress.hpp" + +#include +#include + +namespace ZeroTier { + +class WinFWHelper { + public: + static void newICMPRule(const InetAddress& ip, uint64_t nwid); + static void removeICMPRule(const InetAddress& ip, uint64_t nwid); + static void removeICMPRules(uint64_t nwid); + static void removeICMPRules(); + + + private: + static void _run(std::string cmd); + static void newICMPv4Rule(std::string address, uint64_t nwid); + static void newICMPv6Rule(std::string address, uint64_t nwid); + static void removeICMPv4Rule(std::string address, uint64_t nwid); + static void removeICMPv6Rule(std::string address, uint64_t nwid); + static void removeICMPv4Rules(uint64_t nwid); + static void removeICMPv6Rules(uint64_t nwid); +}; + +} // namespace ZeroTier + +#endif \ No newline at end of file diff --git a/service/OneService.cpp b/service/OneService.cpp index 5984b8b86..3dd344e0e 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -78,6 +78,7 @@ #include "../osdep/MacDNSHelper.hpp" #elif defined(__WINDOWS__) #include "../osdep/WinDNSHelper.hpp" +#include "../osdep/WinFWHelper.hpp" #endif #ifdef ZT_USE_SYSTEM_HTTP_PARSER @@ -847,6 +848,9 @@ public: virtual ~OneServiceImpl() { +#ifdef __WINDOWS__ + WinFWHelper::removeICMPRules(); +#endif _binder.closeAll(_phy); _phy.close(_localControlSocket4); _phy.close(_localControlSocket6); @@ -855,6 +859,8 @@ public: curl_global_cleanup(); #endif + + #ifdef ZT_USE_MINIUPNPC delete _portMapper; #endif @@ -899,6 +905,7 @@ public: _node = new Node(this,(void *)0,&cb,OSUtils::now()); } + // local.conf readLocalSettings(); applyLocalConfig(); @@ -2262,6 +2269,10 @@ public: if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) { if (!n.tap()->removeIp(*ip)) fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString(ipbuf)); + + #ifdef __WINDOWS__ + WinFWHelper::removeICMPRule(*ip, n.config().nwid); + #endif } } @@ -2269,6 +2280,10 @@ public: if (std::find(n.managedIps().begin(),n.managedIps().end(),*ip) == n.managedIps().end()) { if (!n.tap()->addIp(*ip)) fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString(ipbuf)); + + #ifdef __WINDOWS__ + WinFWHelper::newICMPRule(*ip, n.config().nwid); + #endif } } @@ -2749,8 +2764,10 @@ public: n.tap().reset(); _nets.erase(nwid); #if defined(__WINDOWS__) && !defined(ZT_SDK) - if ((op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY)&&(winInstanceId.length() > 0)) + if ((op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) && (winInstanceId.length() > 0)) { WindowsEthernetTap::deletePersistentTapDevice(winInstanceId.c_str()); + WinFWHelper::removeICMPRules(nwid); + } #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 8b2712d3f..36e50ffac 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -124,6 +124,7 @@ + true true @@ -241,6 +242,7 @@ + diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index ebdd33267..ecf664f82 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -288,6 +288,9 @@ Source Files\node + + Source Files\osdep + @@ -551,6 +554,9 @@ Header Files\osdep + + Header Files\osdep + From 91bae4b1a8708aae0c21aca8266fb15147e9de22 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 22 Dec 2022 10:08:02 -0800 Subject: [PATCH 11/16] Add missing default initialization of _lowBandwidthMode --- node/Node.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/Node.cpp b/node/Node.cpp index 7bdf331dc..46f3a4add 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -50,7 +50,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64 _lastPingCheck(0), _lastGratuitousPingCheck(0), _lastHousekeepingRun(0), - _lastMemoizedTraceSettings(0) + _lastMemoizedTraceSettings(0), + _lowBandwidthMode(false) { if (callbacks->version != 0) throw ZT_EXCEPTION_INVALID_ARGUMENT; From a6742b7f82c875572eee696de796fac906dccb57 Mon Sep 17 00:00:00 2001 From: travis laduke Date: Fri, 23 Dec 2022 16:09:53 -0800 Subject: [PATCH 12/16] Prevent shadowing VM routes as default route (macOS) If you have a VM host like parallels, sometimes you get these link-local default routes: ``` netstat -nrfinet | grep "default\|\/1" 0/1 10.2.0.12 UGScg feth4823 default 192.168.82.1 UGScg en1 0/1 192.168.82.1 UGScIg en1 default link#22 UCSIg bridge101 ! 128.0/1 10.2.0.12 UGSc feth4823 128.0/1 192.168.82.1 UGScI en1 ``` (the link#22 one) The _getRTEs function inclused these routes in the list it makes as like: device: bridge101, target: 0.0.0.0/0 If it happens to be first in the list, bridge101 gets selected as the default route. Then Full Tunnel Mode doesn't work. The other routes in the list are like: device: en1 target: 192.168.1.0/24 via: metric: 0 ifscope: 0 device: en1 target: 192.168.1.1/32 via: metric: 0 ifscope: 0 We only need the device name from this, so either one will work. --- osdep/ManagedRoute.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 325f4c803..a8f996839 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -477,7 +477,7 @@ bool ManagedRoute::sync() if ((newSystemVia)&&(!newSystemDevice[0])) { rtes = _getRTEs(newSystemVia,true); for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) { - if ( (r->device[0]) && (strcmp(r->device,_device) != 0) ) { + if ( (r->device[0]) && (strcmp(r->device,_device) != 0) && r->target.netmaskBits() != 0) { Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device); break; } From 27c26a77a25c201221eff694083651a7fe8ff56b Mon Sep 17 00:00:00 2001 From: Sean OMeara Date: Sat, 7 Jan 2023 21:28:12 +0100 Subject: [PATCH 13/16] running build workflow on pull_requests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f76d5b67..ddd6dd448 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -on: [ push ] +on: [ push, pull_request ] jobs: build_ubuntu: From d31f238be02899858f1bdac0efbccfa4da5bac1d Mon Sep 17 00:00:00 2001 From: Brenton Bostick Date: Wed, 11 Jan 2023 13:42:30 -0500 Subject: [PATCH 14/16] fix typos (#1843) --- include/ZeroTierOne.h | 2 +- java/jni/ZT_jniutils.cpp | 3 +-- java/src/com/zerotier/sdk/ResultCode.java | 2 +- node/MAC.hpp | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 23f97b388..a12b3291b 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -384,7 +384,7 @@ enum ZT_ResultCode */ ZT_RESULT_OK_IGNORED = 1, - // Fatal errors (>100, <1000) + // Fatal errors (>=100, <1000) /** * Ran out of memory diff --git a/java/jni/ZT_jniutils.cpp b/java/jni/ZT_jniutils.cpp index bfb969abb..c479f87e2 100644 --- a/java/jni/ZT_jniutils.cpp +++ b/java/jni/ZT_jniutils.cpp @@ -43,7 +43,7 @@ jobject createResultObject(JNIEnv *env, ZT_ResultCode code) resultClass = lookup.findClass("com/zerotier/sdk/ResultCode"); if(resultClass == NULL) { - LOGE("Couldnt find ResultCode class"); + LOGE("Couldn't find ResultCode class"); return NULL; // exception thrown } @@ -1032,4 +1032,3 @@ jobject newVirtualNetworkDNS(JNIEnv *env, const ZT_VirtualNetworkDNS &dns) #ifdef __cplusplus } #endif - diff --git a/java/src/com/zerotier/sdk/ResultCode.java b/java/src/com/zerotier/sdk/ResultCode.java index 66f575613..09e7d3b13 100644 --- a/java/src/com/zerotier/sdk/ResultCode.java +++ b/java/src/com/zerotier/sdk/ResultCode.java @@ -41,7 +41,7 @@ public enum ResultCode { */ RESULT_OK(0), - // Fatal errors (> 0, < 1000) + // Fatal errors (>=100, <1000) /** * Ran out of memory */ diff --git a/node/MAC.hpp b/node/MAC.hpp index 76ee4bf31..de72a1aed 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -149,7 +149,7 @@ public: /** * Get the ZeroTier address for this MAC on this network (assuming no bridging of course, basic unicast) * - * This just XORs the next-lest-significant 5 bytes of the network ID again to unmask. + * This just XORs the next-least-significant 5 bytes of the network ID again to unmask. * * @param nwid Network ID */ From eccc31a4b9a447f3c922abb63478cd2b3609f5dd Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Tue, 10 Jan 2023 16:01:07 -0800 Subject: [PATCH 15/16] Add forced TCP relay mode This patch implements a "TUNNELED" status indicator and "forceTcpRelay" setting for custom relays via local.conf. For example: { "settings": { "tcpFallbackRelay": "6.79.53.215/443", "forceTcpRelay":true } } --- one.cpp | 20 +++++++++++++++++--- service/OneService.cpp | 34 +++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/one.cpp b/one.cpp index 62710547c..3b158cc14 100644 --- a/one.cpp +++ b/one.cpp @@ -449,12 +449,16 @@ static int cli(int argc,char **argv) if (json) { printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { - printf("200 peers\n " ZT_EOL_S); + bool anyTunneled = false; + printf("200 peers\n " ZT_EOL_S); if (j.is_array()) { for(unsigned long k=0;k &bond) +static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer, SharedPtr &bond, bool isTunneled) { char tmp[256]; @@ -542,6 +542,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer, SharedPtr pj["latency"] = peer->latency; pj["role"] = prole; pj["isBonded"] = peer->isBonded; + pj["tunneled"] = isTunneled; if (bond && peer->isBonded) { pj["bondingPolicyCode"] = peer->bondingPolicy; pj["bondingPolicyStr"] = Bond::getPolicyStrByCode(peer->bondingPolicy); @@ -714,6 +715,7 @@ public: PhySocket *_localControlSocket6; bool _updateAutoApply; bool _allowTcpFallbackRelay; + bool _forceTcpRelay; bool _allowSecondaryPort; unsigned int _primaryPort; @@ -812,6 +814,7 @@ public: ,_localControlSocket4((PhySocket *)0) ,_localControlSocket6((PhySocket *)0) ,_updateAutoApply(false) + ,_forceTcpRelay(false) ,_primaryPort(port) ,_udpPortPickerCounter(0) ,_lastDirectReceiveFromGlobal(0) @@ -1093,7 +1096,10 @@ public: if (_ports[i]) p[pc++] = _ports[i]; } - _binder.refresh(_phy,p,pc,explicitBind,*this); + if (!_forceTcpRelay) { + // Only bother binding UDP ports if we aren't forcing TCP-relay mode + _binder.refresh(_phy,p,pc,explicitBind,*this); + } { Mutex::Lock _l(_nets_m); for(std::map::iterator n(_nets.begin());n!=_nets.end();++n) { @@ -1111,8 +1117,9 @@ public: } // Close TCP fallback tunnel if we have direct UDP - if ((_tcpFallbackTunnel)&&((now - _lastDirectReceiveFromGlobal) < (ZT_TCP_FALLBACK_AFTER / 2))) + if (!_forceTcpRelay && (_tcpFallbackTunnel) && ((now - _lastDirectReceiveFromGlobal) < (ZT_TCP_FALLBACK_AFTER / 2))) { _phy.close(_tcpFallbackTunnel->sock); + } // Sync multicast group memberships if ((now - lastTapMulticastGroupCheck) >= ZT_TAP_CHECK_MULTICAST_INTERVAL) { @@ -1488,7 +1495,7 @@ public: if (ps[1] == "show") { SharedPtr bond = _node->bondController()->getBondByPeerId(wantp); if (bond) { - _peerToJson(res,&(pl->peers[i]),bond); + _peerToJson(res,&(pl->peers[i]),bond,(_tcpFallbackTunnel != (TcpConnection *)0)); scode = 200; } else { scode = 400; @@ -1530,6 +1537,7 @@ public: } json &settings = res["config"]["settings"]; settings["allowTcpFallbackRelay"] = OSUtils::jsonBool(settings["allowTcpFallbackRelay"],_allowTcpFallbackRelay); + settings["forceTcpRelay"] = OSUtils::jsonBool(settings["forceTcpRelay"],_forceTcpRelay); settings["primaryPort"] = OSUtils::jsonInt(settings["primaryPort"],(uint64_t)_primaryPort) & 0xffff; settings["secondaryPort"] = OSUtils::jsonInt(settings["secondaryPort"],(uint64_t)_secondaryPort) & 0xffff; settings["tertiaryPort"] = OSUtils::jsonInt(settings["tertiaryPort"],(uint64_t)_tertiaryPort) & 0xffff; @@ -1634,7 +1642,7 @@ public: const uint64_t id = pl->peers[i].address; bond = _node->bondController()->getBondByPeerId(id); } - _peerToJson(pj,&(pl->peers[i]),bond); + _peerToJson(pj,&(pl->peers[i]),bond,(_tcpFallbackTunnel != (TcpConnection *)0)); res.push_back(pj); } @@ -1649,7 +1657,7 @@ public: if (pl->peers[i].isBonded) { bond = _node->bondController()->getBondByPeerId(wantp); } - _peerToJson(res,&(pl->peers[i]),bond); + _peerToJson(res,&(pl->peers[i]),bond,(_tcpFallbackTunnel != (TcpConnection *)0)); scode = 200; break; } @@ -2118,6 +2126,8 @@ public: // bondingPolicy cannot be used with allowTcpFallbackRelay _allowTcpFallbackRelay = OSUtils::jsonBool(settings["allowTcpFallbackRelay"],true); + _forceTcpRelay = OSUtils::jsonBool(settings["forceTcpRelay"],false); + #ifdef ZT_TCP_FALLBACK_RELAY _fallbackRelayAddress = InetAddress(OSUtils::jsonString(settings["tcpFallbackRelay"], ZT_TCP_FALLBACK_RELAY).c_str()); #endif @@ -2404,6 +2414,9 @@ public: inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) { + if (_forceTcpRelay) { + return; + } const uint64_t now = OSUtils::now(); if ((len >= 16)&&(reinterpret_cast(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) _lastDirectReceiveFromGlobal = now; @@ -3112,7 +3125,7 @@ public: // IP address in ZT_TCP_FALLBACK_AFTER milliseconds. If we do start getting // valid direct traffic we'll stop using it and close the socket after a while. const int64_t now = OSUtils::now(); - if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) { + if (_forceTcpRelay || (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER))) { if (_tcpFallbackTunnel) { bool flushNow = false; { @@ -3138,7 +3151,7 @@ public: void *tmpptr = (void *)_tcpFallbackTunnel; phyOnTcpWritable(_tcpFallbackTunnel->sock,&tmpptr); } - } else if (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INVERVAL / 2))) { + } else if (_forceTcpRelay || (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INVERVAL / 2)))) { const InetAddress addr(_fallbackRelayAddress); TcpConnection *tc = new TcpConnection(); { @@ -3159,12 +3172,15 @@ public: } } } + if (_forceTcpRelay) { + // Shortcut here so that we don't emit any UDP packets + return 0; + } #endif // ZT_TCP_FALLBACK_RELAY // Even when relaying we still send via UDP. This way if UDP starts // working we can instantly "fail forward" to it and stop using TCP // proxy fallback, which is slow. - if ((localSocket != -1)&&(localSocket != 0)&&(_binder.isUdpSocketValid((PhySocket *)((uintptr_t)localSocket)))) { if ((ttl)&&(addr->ss_family == AF_INET)) _phy.setIp4UdpTtl((PhySocket *)((uintptr_t)localSocket),ttl); const bool r = _phy.udpSend((PhySocket *)((uintptr_t)localSocket),(const struct sockaddr *)addr,data,len); From 39f3f5b2d95fe1e393954623b0943189441df187 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Wed, 11 Jan 2023 20:12:15 -0800 Subject: [PATCH 16/16] User-configurable physical MTU for individual links This patch allows users to specify the physical layer MTU for individual links when in multipath mode. For example: { "settings": { "defaultBondingPolicy": "custom-balance-xor", "policies": { "custom-balance-xor": { "basePolicy": "balance-xor", "failoverInterval": 5000, "links": { "weird_5g_link": { "mtu": 1300 }, "enp5s0": { "mtu": 1400 } } } } } } --- node/Bond.cpp | 3 ++- node/Bond.hpp | 16 +++++++++++++++- node/Path.hpp | 8 ++++++++ node/Switch.cpp | 11 ++++++++--- node/Switch.hpp | 2 +- service/OneService.cpp | 3 ++- 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/node/Bond.cpp b/node/Bond.cpp index dc17e6bee..8edd7e297 100644 --- a/node/Bond.cpp +++ b/node/Bond.cpp @@ -184,7 +184,7 @@ SharedPtr Bond::getLinkBySocket(const std::string& policyAlias, uint64_t l auto search = _interfaceToLinkMap[policyAlias].find(ifnameStr); if (search == _interfaceToLinkMap[policyAlias].end()) { if (createIfNeeded) { - SharedPtr s = new Link(ifnameStr, 0, 0, true, ZT_BOND_SLAVE_MODE_PRIMARY, ""); + SharedPtr s = new Link(ifnameStr, 0, 0, 0, true, ZT_BOND_SLAVE_MODE_PRIMARY, ""); _interfaceToLinkMap[policyAlias].insert(std::pair >(ifnameStr, s)); return s; } @@ -1253,6 +1253,7 @@ void Bond::estimatePathQuality(int64_t now) if (link) { int linkSpeed = link->capacity(); _paths[i].p->_givenLinkSpeed = linkSpeed; + _paths[i].p->_mtu = link->mtu(); maxObservedLinkCap = linkSpeed > maxObservedLinkCap ? linkSpeed : maxObservedLinkCap; } } diff --git a/node/Bond.hpp b/node/Bond.hpp index 3419c8cfe..389b970e9 100644 --- a/node/Bond.hpp +++ b/node/Bond.hpp @@ -122,9 +122,10 @@ class Link { * @param mode * @param failoverToLinkStr */ - Link(std::string ifnameStr, uint8_t ipvPref, uint32_t capacity, bool enabled, uint8_t mode, std::string failoverToLinkStr) + Link(std::string ifnameStr, uint8_t ipvPref, uint16_t mtu, uint32_t capacity, bool enabled, uint8_t mode, std::string failoverToLinkStr) : _ifnameStr(ifnameStr) , _ipvPref(ipvPref) + , _mtu(mtu) , _capacity(capacity) , _relativeCapacity(0.0) , _enabled(enabled) @@ -226,6 +227,14 @@ class Link { return _ipvPref; } + /** + * @return The MTU for this link (as specified by the user.) + */ + inline uint16_t mtu() + { + return _mtu; + } + /** * @return The mode (e.g. primary/spare) for this link (as specified by the user.) */ @@ -260,6 +269,11 @@ class Link { */ uint8_t _ipvPref; + /** + * The physical-layer MTU for this link + */ + uint16_t _mtu; + /** * User-specified capacity of this link */ diff --git a/node/Path.hpp b/node/Path.hpp index 8304f27d8..eb268695d 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -92,6 +92,7 @@ public: _valid(true), _eligible(false), _bonded(false), + _mtu(0), _givenLinkSpeed(0), _relativeQuality(0), _latency(0xffff), @@ -112,6 +113,7 @@ public: _valid(true), _eligible(false), _bonded(false), + _mtu(0), _givenLinkSpeed(0), _relativeQuality(0), _latency(0xffff), @@ -334,6 +336,11 @@ public: */ inline unsigned int bonded() const { return _bonded; } + /** + * @return Whether the user-specified MTU for this path (determined by MTU for parent link) + */ + inline unsigned int mtu() const { return _mtu; } + /** * @return Given link capacity as reported by the bonding layer */ @@ -370,6 +377,7 @@ private: volatile bool _valid; volatile bool _eligible; volatile bool _bonded; + volatile uint16_t _mtu; volatile uint32_t _givenLinkSpeed; volatile float _relativeQuality; diff --git a/node/Switch.cpp b/node/Switch.cpp index ae870a278..9a81e532e 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -1009,7 +1009,8 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId) Mutex::Lock _l(peer->_paths_m); for(int i=0;i_paths[i].p && peer->_paths[i].p->alive(now)) { - _sendViaSpecificPath(tPtr,peer,peer->_paths[i].p,now,packet,encrypt,flowId); + uint16_t userSpecifiedMtu = peer->_paths[i].p->mtu(); + _sendViaSpecificPath(tPtr,peer,peer->_paths[i].p, userSpecifiedMtu,now,packet,encrypt,flowId); } } return true; @@ -1025,7 +1026,8 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId) } } if (viaPath) { - _sendViaSpecificPath(tPtr,peer,viaPath,now,packet,encrypt,flowId); + uint16_t userSpecifiedMtu = viaPath->mtu(); + _sendViaSpecificPath(tPtr,peer,viaPath,userSpecifiedMtu,now,packet,encrypt,flowId); return true; } } @@ -1033,12 +1035,15 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId) return false; } -void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr peer,SharedPtr viaPath,int64_t now,Packet &packet,bool encrypt,int32_t flowId) +void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr peer,SharedPtr viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId) { unsigned int mtu = ZT_DEFAULT_PHYSMTU; uint64_t trustedPathId = 0; RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId); + if (userSpecifiedMtu > 0) { + mtu = userSpecifiedMtu; + } unsigned int chunkSize = std::min(packet.size(),mtu); packet.setFragmented(chunkSize < packet.size()); diff --git a/node/Switch.hpp b/node/Switch.hpp index 4ca1c0e50..9bb1fbeaf 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -207,7 +207,7 @@ public: private: bool _shouldUnite(const int64_t now,const Address &source,const Address &destination); bool _trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId = ZT_QOS_NO_FLOW); // packet is modified if return is true - void _sendViaSpecificPath(void *tPtr,SharedPtr peer,SharedPtr viaPath,int64_t now,Packet &packet,bool encrypt,int32_t flowId); + void _sendViaSpecificPath(void *tPtr,SharedPtr peer,SharedPtr viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId); const RuntimeEnvironment *const RR; int64_t _lastBeaconResponse; diff --git a/service/OneService.cpp b/service/OneService.cpp index bbd48c7aa..429ae20e4 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2075,6 +2075,7 @@ public: bool enabled = OSUtils::jsonInt(link["enabled"],true); uint32_t capacity = OSUtils::jsonInt(link["capacity"],0); uint8_t ipvPref = OSUtils::jsonInt(link["ipvPref"],0); + uint16_t mtu = OSUtils::jsonInt(link["mtu"],0); std::string failoverToStr(OSUtils::jsonString(link["failoverTo"],"")); // Mode std::string linkModeStr(OSUtils::jsonString(link["mode"],"spare")); @@ -2091,7 +2092,7 @@ public: failoverToStr = ""; enabled = false; } - _node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,capacity,enabled,linkMode,failoverToStr)); + _node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,mtu,capacity,enabled,linkMode,failoverToStr)); } std::string linkSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize")); if (linkSelectMethodStr == "always") {