github/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/ZeroTier/ZeroTierOne synced 2025-07-05 20:41:44 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add multi-core concurrent packet processing

Browse source
This commit is contained in:
Joseph Henry 2024-02-23 09:57:39 -08:00
parent ac6d532651
commit 683d332abc
No known key found for this signature in database
GPG key ID: C45B33FF5EBC9344
12 changed files with 400 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

120
osdep/BSDEthernetTap.cpp
View file

@ -39,7 +39,9 @@
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/route.h>
#include <pthread_np.h>
#include <sched.h>
#include <string>
#include <map>
#include <set>
@ -53,6 +55,7 @@
#include "BSDEthernetTap.hpp"
#define ZT_BASE32_CHARS "0123456789abcdefghijklmnopqrstuv"
#define ZT_TAP_BUF_SIZE (1024 * 16)
// ff:ff:ff:ff:ff:ff with no ADI
static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
@ -61,6 +64,7 @@ namespace ZeroTier {
BSDEthernetTap::BSDEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -69,6 +73,7 @@ BSDEthernetTap::BSDEthernetTap(
void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
void *arg) :
_handler(handler),
_concurrency(concurrency),
_arg(arg),
_nwid(nwid),
_mtu(mtu),
@ -195,11 +200,9 @@ BSDEthernetTap::BSDEthernetTap(
BSDEthernetTap::~BSDEthernetTap()
{
::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
Thread::join(_thread);
::close(_fd);
::close(_shutdownSignalPipe[0]);
::close(_shutdownSignalPipe[1]);
long cpid = (long)vfork();
if (cpid == 0) {
#ifdef ZT_TRACE
@ -211,6 +214,10 @@ BSDEthernetTap::~BSDEthernetTap()
int exitcode = -1;
::waitpid(cpid,&exitcode,0);
}
Thread::join(_thread);
for (std::thread &t : _rxThreads) {
t.join();
}
}
void BSDEthernetTap::setEnabled(bool en)
@ -418,53 +425,84 @@ void BSDEthernetTap::setMtu(unsigned int mtu)
void BSDEthernetTap::threadMain()
throw()
{
fd_set readfds,nullfds;
MAC to,from;
int n,nfds,r;
char getBuf[ZT_MAX_MTU + 64];
bool _enablePinning = false;
char* envvar = std::getenv("ZT_CPU_PINNING");
if (envvar) {
int tmp = atoi(envvar);
if (tmp > 0) {
_enablePinning = true;
}
}
// Wait for a moment after startup -- wait for Network to finish
// constructing itself.
Thread::sleep(500);
FD_ZERO(&readfds);
FD_ZERO(&nullfds);
nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
for (unsigned int i = 0; i < _concurrency; ++i) {
_rxThreads.push_back(std::thread([this, i, _enablePinning] {
r = 0;
for(;;) {
FD_SET(_shutdownSignalPipe[0],&readfds);
FD_SET(_fd,&readfds);
select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
break;
if (FD_ISSET(_fd,&readfds)) {
n = (int)::read(_fd,getBuf + r,sizeof(getBuf) - r);
if (n < 0) {
if ((errno != EINTR)&&(errno != ETIMEDOUT))
break;
} else {
// Some tap drivers like to send the ethernet frame and the
// payload in two chunks, so handle that by accumulating
// data until we have at least a frame.
r += n;
if (r > 14) {
if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
r = _mtu + 14;
if (_enabled) {
to.setTo(getBuf,6);
from.setTo(getBuf + 6,6);
unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]);
_handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14);
}
r = 0;
if (_enablePinning) {
int pinCore = i % _concurrency;
fprintf(stderr, "pinning thread %d to core %d\n", i, pinCore);
pthread_t self = pthread_self();
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(pinCore, &cpuset);
//int rc = sched_setaffinity(self, sizeof(cpu_set_t), &cpuset);
int rc = pthread_setaffinity_np(self, sizeof(cpu_set_t), &cpuset);
if (rc != 0)
{
fprintf(stderr, "failed to pin thread %d to core %d: %s\n", i, pinCore, strerror(errno));
exit(1);
}
}
}
uint8_t b[ZT_TAP_BUF_SIZE];
MAC to, from;
fd_set readfds, nullfds;
int n, nfds, r;
FD_ZERO(&readfds);
FD_ZERO(&nullfds);
nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
r = 0;
for(;;) {
FD_SET(_shutdownSignalPipe[0],&readfds);
FD_SET(_fd,&readfds);
select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
break;
if (FD_ISSET(_fd,&readfds)) {
n = (int)::read(_fd,b + r,sizeof(b) - r);
if (n < 0) {
if ((errno != EINTR)&&(errno != ETIMEDOUT))
break;
} else {
// Some tap drivers like to send the ethernet frame and the
// payload in two chunks, so handle that by accumulating
// data until we have at least a frame.
r += n;
if (r > 14) {
if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
r = _mtu + 14;
if (_enabled) {
to.setTo(b,6);
from.setTo(b + 6,6);
unsigned int etherType = ntohs(((const uint16_t *)b)[6]);
_handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(b + 14),r - 14);
}
r = 0;
}
}
}
}
}));
}
}

4
osdep/BSDEthernetTap.hpp
View file

@ -20,6 +20,7 @@
#include <string>
#include <vector>
#include <stdexcept>
#include <thread>
#include "../node/Constants.hpp"
#include "../node/MulticastGroup.hpp"
@ -34,6 +35,7 @@ class BSDEthernetTap : public EthernetTap
public:
BSDEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -62,6 +64,7 @@ public:
private:
void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
void *_arg;
unsigned int _concurrency;
uint64_t _nwid;
Thread _thread;
std::string _dev;
@ -73,6 +76,7 @@ private:
volatile bool _enabled;
mutable std::vector<InetAddress> _ifaddrs;
mutable uint64_t _lastIfAddrsUpdate;
std::vector<std::thread> _rxThreads;
};
} // namespace ZeroTier

15
osdep/EthernetTap.cpp
View file

@ -58,6 +58,7 @@ namespace ZeroTier {
std::shared_ptr<EthernetTap> EthernetTap::newInstance(
const char *tapDeviceType, // OS-specific, NULL for default
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -83,16 +84,16 @@ std::shared_ptr<EthernetTap> EthernetTap::newInstance(
// The "feth" virtual Ethernet device type appeared in Darwin 17.x.x. Older versions
// (Sierra and earlier) must use the a kernel extension.
if (strtol(osrelease,(char **)0,10) < 17) {
return std::shared_ptr<EthernetTap>(new MacKextEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new MacKextEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
} else {
return std::shared_ptr<EthernetTap>(new MacEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new MacEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
}
}
}
#endif // __APPLE__
#ifdef __LINUX__
return std::shared_ptr<EthernetTap>(new LinuxEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new LinuxEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
#endif // __LINUX__
#ifdef __WINDOWS__
@ -126,19 +127,19 @@ std::shared_ptr<EthernetTap> EthernetTap::newInstance(
_comInit = true;
}
}
return std::shared_ptr<EthernetTap>(new WindowsEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new WindowsEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
#endif // __WINDOWS__
#ifdef __FreeBSD__
return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
#endif // __FreeBSD__
#ifdef __NetBSD__
return std::shared_ptr<EthernetTap>(new NetBSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new NetBSDEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
#endif // __NetBSD__
#ifdef __OpenBSD__
return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,concurrency,mac,mtu,metric,nwid,friendlyName,handler,arg));
#endif // __OpenBSD__
#endif // ZT_SDK?

1
osdep/EthernetTap.hpp
View file

@ -33,6 +33,7 @@ public:
static std::shared_ptr<EthernetTap> newInstance(
const char *tapDeviceType, // OS-specific, NULL for default
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,

241
osdep/LinuxEthernetTap.cpp
View file

@ -60,7 +60,7 @@
#define IFNAMSIZ 16
#endif
#define ZT_TAP_BUF_SIZE 16384
#define ZT_TAP_BUF_SIZE (1024 * 16)
// ff:ff:ff:ff:ff:ff with no ADI
static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
@ -68,7 +68,7 @@ static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC
namespace ZeroTier {
// determine if we're running a really old linux kernel.
// Kernels in the 2.6.x series don't behave the same when bringing up
// Kernels in the 2.6.x series don't behave the same when bringing up
// the tap devices.
//
// Returns true if the kernel major version is < 3
@ -111,6 +111,7 @@ static void _base32_5_to_8(const uint8_t *in,char *out)
LinuxEthernetTap::LinuxEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -127,6 +128,7 @@ LinuxEthernetTap::LinuxEthernetTap(
_fd(0),
_enabled(true),
_run(true),
_concurrency(concurrency),
_lastIfAddrsUpdate(0)
{
static std::mutex s_tapCreateLock;
@ -220,135 +222,164 @@ LinuxEthernetTap::LinuxEthernetTap(
(void)::pipe(_shutdownSignalPipe);
_tapReaderThread = std::thread([this]{
uint8_t b[ZT_TAP_BUF_SIZE];
fd_set readfds,nullfds;
int n,nfds,r;
std::vector<void *> buffers;
struct ifreq ifr;
memset(&ifr,0,sizeof(ifr));
strcpy(ifr.ifr_name,_dev.c_str());
const int sock = socket(AF_INET,SOCK_DGRAM,0);
if (sock <= 0)
return;
if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
return;
bool _enablePinning = false;
char* envvar = std::getenv("ZT_CPU_PINNING");
if (envvar) {
int tmp = atoi(envvar);
if (tmp > 0) {
_enablePinning = true;
}
}
ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6);
if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
return;
}
for (unsigned int i = 0; i < _concurrency; ++i) {
_rxThreads.push_back(std::thread([this, i, _enablePinning] {
usleep(100000);
if (_enablePinning) {
int pinCore = i % _concurrency;
fprintf(stderr, "pinning thread %d to core %d\n", i, pinCore);
pthread_t self = pthread_self();
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(pinCore, &cpuset);
int rc = pthread_setaffinity_np(self, sizeof(cpu_set_t), &cpuset);
if (rc != 0)
{
fprintf(stderr, "failed to pin thread %d to core %d: %s\n", i, pinCore, strerror(errno));
exit(1);
}
}
uint8_t b[ZT_TAP_BUF_SIZE];
fd_set readfds, nullfds;
int n, nfds, r;
if (i == 0) {
struct ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
strcpy(ifr.ifr_name, _dev.c_str());
const int sock = socket(AF_INET, SOCK_DGRAM, 0);
if (sock <= 0)
return;
if (ioctl(sock, SIOCGIFFLAGS, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
return;
}
ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data, 6);
if (ioctl(sock, SIOCSIFHWADDR, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
return;
}
usleep(100000);
if (isOldLinuxKernel()) {
ifr.ifr_ifru.ifru_mtu = (int)_mtu;
if (ioctl(sock, SIOCSIFMTU, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
return;
}
usleep(100000);
}
ifr.ifr_flags |= IFF_MULTICAST;
ifr.ifr_flags |= IFF_UP;
if (ioctl(sock, SIOCSIFFLAGS, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
return;
}
usleep(100000);
if (! isOldLinuxKernel()) {
ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data, 6);
if (ioctl(sock, SIOCSIFHWADDR, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
return;
}
ifr.ifr_ifru.ifru_mtu = (int)_mtu;
if (ioctl(sock, SIOCSIFMTU, (void*)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
return;
}
}
fcntl(_fd, F_SETFL, O_NONBLOCK);
if (isOldLinuxKernel()) {
ifr.ifr_ifru.ifru_mtu = (int)_mtu;
if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
}
if (! _run) {
return;
}
usleep(100000);
}
FD_ZERO(&readfds);
FD_ZERO(&nullfds);
nfds = (int)std::max(_shutdownSignalPipe[0], _fd) + 1;
ifr.ifr_flags |= IFF_MULTICAST;
ifr.ifr_flags |= IFF_UP;
if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
return;
}
r = 0;
for (;;) {
FD_SET(_shutdownSignalPipe[0], &readfds);
FD_SET(_fd, &readfds);
select(nfds, &readfds, &nullfds, &nullfds, (struct timeval*)0);
usleep(100000);
if (FD_ISSET(_shutdownSignalPipe[0], &readfds)) {
break;
}
if (FD_ISSET(_fd, &readfds)) {
for (;;) {
// read until there are no more packets, then return to outer select() loop
n = (int)::read(_fd, b + r, ZT_TAP_BUF_SIZE - r);
if (n > 0) {
// Some tap drivers like to send the ethernet frame and the
// payload in two chunks, so handle that by accumulating
// data until we have at least a frame.
r += n;
if (r > 14) {
if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
r = _mtu + 14;
if (!isOldLinuxKernel()) {
ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6);
if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
return;
}
if (_enabled) {
MAC to(b, 6), from(b + 6, 6);
unsigned int etherType = Utils::ntoh(((const uint16_t*)b)[6]);
_handler(_arg, nullptr, _nwid, from, to, etherType, 0, (const void*)(b + 14), (unsigned int)(r - 14));
}
ifr.ifr_ifru.ifru_mtu = (int)_mtu;
if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
::close(sock);
printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
return;
}
}
fcntl(_fd,F_SETFL,O_NONBLOCK);
::close(sock);
if (!_run)
return;
FD_ZERO(&readfds);
FD_ZERO(&nullfds);
nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
r = 0;
for(;;) {
FD_SET(_shutdownSignalPipe[0],&readfds);
FD_SET(_fd,&readfds);
select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
if (FD_ISSET(_shutdownSignalPipe[0],&readfds))
break;
if (FD_ISSET(_fd,&readfds)) {
for(;;) { // read until there are no more packets, then return to outer select() loop
n = (int)::read(_fd,b + r,ZT_TAP_BUF_SIZE - r);
if (n > 0) {
// Some tap drivers like to send the ethernet frame and the
// payload in two chunks, so handle that by accumulating
// data until we have at least a frame.
r += n;
if (r > 14) {
if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
r = _mtu + 14;
if (_enabled) {
//_tapq.post(std::pair<void *,int>(buf,r));
//buf = nullptr;
MAC to(b, 6),from(b + 6, 6);
unsigned int etherType = Utils::ntoh(((const uint16_t *)b)[6]);
_handler(_arg, nullptr, _nwid, from, to, etherType, 0, (const void *)(b + 14),(unsigned int)(r - 14));
r = 0;
}
r = 0;
}
} else {
r = 0;
break;
else {
r = 0;
break;
}
}
}
}
}
});
}));
}
}
LinuxEthernetTap::~LinuxEthernetTap()
{
_run = false;
(void)::write(_shutdownSignalPipe[1],"\0",1);
_tapReaderThread.join();
::close(_fd);
::close(_shutdownSignalPipe[0]);
::close(_shutdownSignalPipe[1]);
for (std::thread &t : _rxThreads) {
t.join();
}
}
void LinuxEthernetTap::setEnabled(bool en)

8
osdep/LinuxEthernetTap.hpp
View file

@ -26,6 +26,7 @@
#include <mutex>
#include "../node/MulticastGroup.hpp"
#include "EthernetTap.hpp"
#include "BlockingQueue.hpp"
namespace ZeroTier {
@ -34,6 +35,7 @@ class LinuxEthernetTap : public EthernetTap
public:
LinuxEthernetTap(
const char *homePath,
unsigned int _concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -57,9 +59,6 @@ public:
virtual void setMtu(unsigned int mtu);
virtual void setDns(const char *domain, const std::vector<InetAddress> &servers) {}
private:
void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
void *_arg;
@ -69,13 +68,14 @@ private:
std::string _dev;
std::vector<MulticastGroup> _multicastGroups;
unsigned int _mtu;
unsigned int _concurrency;
int _fd;
int _shutdownSignalPipe[2];
std::atomic_bool _enabled;
std::atomic_bool _run;
std::thread _tapReaderThread;
mutable std::vector<InetAddress> _ifaddrs;
mutable uint64_t _lastIfAddrsUpdate;
std::vector<std::thread> _rxThreads;
};
} // namespace ZeroTier

22
osdep/MacEthernetTap.cpp
View file

@ -69,6 +69,7 @@ static bool fethMaxMtuAdjusted = false;
MacEthernetTap::MacEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -77,6 +78,7 @@ MacEthernetTap::MacEthernetTap(
void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len),
void *arg) :
_handler(handler),
_concurrency(concurrency),
_arg(arg),
_nwid(nwid),
_homePath(homePath),
@ -286,6 +288,9 @@ MacEthernetTap::~MacEthernetTap()
}
Thread::join(_thread);
for (std::thread &t : _rxThreads) {
t.join();
}
}
void MacEthernetTap::setEnabled(bool en) { _enabled = en; }
@ -474,17 +479,25 @@ void MacEthernetTap::setMtu(unsigned int mtu)
void MacEthernetTap::threadMain()
throw()
{
Thread::sleep(250);
for (unsigned int i = 0; i < _concurrency; ++i) {
_rxThreads.push_back(std::thread([this, i] {
fprintf(stderr, "starting thread %d\n", i);
char agentReadBuf[ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE];
char agentStderrBuf[256];
fd_set readfds,nullfds;
MAC to,from;
Thread::sleep(250);
const int nfds = std::max(std::max(_shutdownSignalPipe[0],_agentStdout),_agentStderr) + 1;
long agentReadPtr = 0;
fcntl(_agentStdout,F_SETFL,fcntl(_agentStdout,F_GETFL)|O_NONBLOCK);
fcntl(_agentStderr,F_SETFL,fcntl(_agentStderr,F_GETFL)|O_NONBLOCK);
if (i == 0) {
fcntl(_agentStdout,F_SETFL,fcntl(_agentStdout,F_GETFL)|O_NONBLOCK);
fcntl(_agentStderr,F_SETFL,fcntl(_agentStderr,F_GETFL)|O_NONBLOCK);
}
FD_ZERO(&readfds);
FD_ZERO(&nullfds);
@ -533,6 +546,7 @@ void MacEthernetTap::threadMain()
*/
}
}
}));}
::close(_agentStdin);
::close(_agentStdout);

4
osdep/MacEthernetTap.hpp
View file

@ -28,6 +28,7 @@
#include <stdexcept>
#include <string>
#include <vector>
#include <thread>
namespace ZeroTier {
@ -36,6 +37,7 @@ class MacEthernetTap : public EthernetTap
public:
MacEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -67,6 +69,7 @@ private:
uint64_t _nwid;
Thread _thread;
std::string _homePath;
unsigned int _concurrency;
std::string _dev;
std::vector<MulticastGroup> _multicastGroups;
Mutex _putLock;
@ -79,6 +82,7 @@ private:
volatile bool _enabled;
mutable std::vector<InetAddress> _ifaddrs;
mutable uint64_t _lastIfAddrsUpdate;
std::vector<std::thread> _rxThreads;
};

2
osdep/MacEthernetTapAgent.c
View file

@ -32,7 +32,7 @@
* All this stuff is basically undocumented. A lot of tracing through
* the Darwin/XNU kernel source was required to figure out how to make
* this actually work.
*
*
* We hope to develop a DriverKit-based driver in the near-mid future to
* replace this weird hack, but it works for now through Big Sur in our
* testing.

6
osdep/MacKextEthernetTap.cpp
View file

@ -306,6 +306,7 @@ static Mutex globalTapCreateLock;
MacKextEthernetTap::MacKextEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -317,6 +318,7 @@ MacKextEthernetTap::MacKextEthernetTap(
_arg(arg),
_nwid(nwid),
_homePath(homePath),
_concurrency(concurrency),
_mtu(mtu),
_metric(metric),
_fd(0),
@ -447,7 +449,9 @@ MacKextEthernetTap::~MacKextEthernetTap()
::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
Thread::join(_thread);
for (std::thread &t : _rxThreads) {
t.join();
}
::close(_fd);
::close(_shutdownSignalPipe[0]);
::close(_shutdownSignalPipe[1]);

4
osdep/MacKextEthernetTap.hpp
View file

@ -20,6 +20,7 @@
#include <stdexcept>
#include <string>
#include <vector>
#include <thread>
#include "../node/Constants.hpp"
#include "../node/MAC.hpp"
@ -36,6 +37,7 @@ class MacKextEthernetTap : public EthernetTap
public:
MacKextEthernetTap(
const char *homePath,
unsigned int concurrency,
const MAC &mac,
unsigned int mtu,
unsigned int metric,
@ -70,11 +72,13 @@ private:
std::string _homePath;
std::string _dev;
std::vector<MulticastGroup> _multicastGroups;
unsigned int _concurrency;
unsigned int _mtu;
unsigned int _metric;
int _fd;
int _shutdownSignalPipe[2];
volatile bool _enabled;
std::vector<std::thread> _rxThreads;
};
} // namespace ZeroTier