clang-format

This commit is contained in:
Adam Ierymenko 2025-07-03 11:26:23 -04:00
parent d45f280cb7
commit ba2a4a605c
No known key found for this signature in database
GPG key ID: C8877CF2D7A5D7F3
140 changed files with 19214 additions and 17403 deletions

View file

@ -15,16 +15,16 @@
#define ZT_AES_HPP
#include "Constants.hpp"
#include "Utils.hpp"
#include "SHA512.hpp"
#include "Utils.hpp"
// Uncomment to disable all hardware acceleration (usually for testing)
//#define ZT_AES_NO_ACCEL
// #define ZT_AES_NO_ACCEL
#if !defined(ZT_AES_NO_ACCEL) && defined(ZT_ARCH_X64)
#if ! defined(ZT_AES_NO_ACCEL) && defined(ZT_ARCH_X64)
#define ZT_AES_AESNI 1
#endif
#if !defined(ZT_AES_NO_ACCEL) && defined(ZT_ARCH_ARM_HAS_NEON) && defined(ZT_ARCH_ARM_HAS_CRYPTO)
#if ! defined(ZT_AES_NO_ACCEL) && defined(ZT_ARCH_ARM_HAS_NEON) && defined(ZT_ARCH_ARM_HAS_CRYPTO)
#define ZT_AES_NEON 1
#endif
@ -40,9 +40,8 @@ namespace ZeroTier {
* This includes hardware acceleration for certain processors. The software
* mode is fallback and is significantly slower.
*/
class AES
{
public:
class AES {
public:
/**
* @return True if this system has hardware AES acceleration
*/
@ -63,39 +62,44 @@ public:
* Create an un-initialized AES instance (must call init() before use)
*/
ZT_INLINE AES() noexcept
{}
{
}
/**
* Create an AES instance with the given key
*
* @param key 256-bit key
*/
explicit ZT_INLINE AES(const void *const key) noexcept
{ this->init(key); }
explicit ZT_INLINE AES(const void* const key) noexcept
{
this->init(key);
}
ZT_INLINE ~AES()
{ Utils::burn(&p_k, sizeof(p_k)); }
{
Utils::burn(&p_k, sizeof(p_k));
}
/**
* Set (or re-set) this AES256 cipher's key
*
* @param key 256-bit / 32-byte key
*/
ZT_INLINE void init(const void *const key) noexcept
ZT_INLINE void init(const void* const key) noexcept
{
#ifdef ZT_AES_AESNI
if (likely(Utils::CPUID.aes)) {
p_init_aesni(reinterpret_cast<const uint8_t *>(key));
p_init_aesni(reinterpret_cast<const uint8_t*>(key));
return;
}
#endif
#ifdef ZT_AES_NEON
if (Utils::ARMCAP.aes) {
p_init_armneon_crypto(reinterpret_cast<const uint8_t *>(key));
p_init_armneon_crypto(reinterpret_cast<const uint8_t*>(key));
return;
}
#endif
p_initSW(reinterpret_cast<const uint8_t *>(key));
p_initSW(reinterpret_cast<const uint8_t*>(key));
}
/**
@ -104,7 +108,7 @@ public:
* @param in Input block
* @param out Output block (can be same as input)
*/
ZT_INLINE void encrypt(const void *const in, void *const out) const noexcept
ZT_INLINE void encrypt(const void* const in, void* const out) const noexcept
{
#ifdef ZT_AES_AESNI
if (likely(Utils::CPUID.aes)) {
@ -118,7 +122,7 @@ public:
return;
}
#endif
p_encryptSW(reinterpret_cast<const uint8_t *>(in), reinterpret_cast<uint8_t *>(out));
p_encryptSW(reinterpret_cast<const uint8_t*>(in), reinterpret_cast<uint8_t*>(out));
}
/**
@ -127,7 +131,7 @@ public:
* @param in Input block
* @param out Output block (can be same as input)
*/
ZT_INLINE void decrypt(const void *const in, void *const out) const noexcept
ZT_INLINE void decrypt(const void* const in, void* const out) const noexcept
{
#ifdef ZT_AES_AESNI
if (likely(Utils::CPUID.aes)) {
@ -141,7 +145,7 @@ public:
return;
}
#endif
p_decryptSW(reinterpret_cast<const uint8_t *>(in), reinterpret_cast<uint8_t *>(out));
p_decryptSW(reinterpret_cast<const uint8_t*>(in), reinterpret_cast<uint8_t*>(out));
}
class GMACSIVEncryptor;
@ -150,12 +154,11 @@ public:
/**
* Streaming GMAC calculator
*/
class GMAC
{
class GMAC {
friend class GMACSIVEncryptor;
friend class GMACSIVDecryptor;
public:
public:
/**
* @return True if this system has hardware GMAC acceleration
*/
@ -177,8 +180,9 @@ public:
*
* @param aes Keyed AES instance to use
*/
ZT_INLINE GMAC(const AES &aes) : _aes(aes)
{}
ZT_INLINE GMAC(const AES& aes) : _aes(aes)
{
}
/**
* Reset and initialize for a new GMAC calculation
@ -192,12 +196,12 @@ public:
// We fill the least significant 32 bits in the _iv field with 1 since in GCM mode
// this would hold the counter, but we're not doing GCM. The counter is therefore
// always 1.
#ifdef ZT_AES_AESNI // also implies an x64 processor
*reinterpret_cast<uint64_t *>(_iv) = *reinterpret_cast<const uint64_t *>(iv);
*reinterpret_cast<uint32_t *>(_iv + 8) = *reinterpret_cast<const uint64_t *>(iv + 8);
*reinterpret_cast<uint32_t *>(_iv + 12) = 0x01000000; // 0x00000001 in big-endian byte order
#ifdef ZT_AES_AESNI // also implies an x64 processor
*reinterpret_cast<uint64_t*>(_iv) = *reinterpret_cast<const uint64_t*>(iv);
*reinterpret_cast<uint32_t*>(_iv + 8) = *reinterpret_cast<const uint64_t*>(iv + 8);
*reinterpret_cast<uint32_t*>(_iv + 12) = 0x01000000; // 0x00000001 in big-endian byte order
#else
for(int i=0;i<12;++i) {
for (int i = 0; i < 12; ++i) {
_iv[i] = iv[i];
}
_iv[12] = 0;
@ -215,7 +219,7 @@ public:
* @param data Bytes to process
* @param len Length of input
*/
void update(const void *data, unsigned int len) noexcept;
void update(const void* data, unsigned int len) noexcept;
/**
* Process any remaining cached bytes and generate tag
@ -226,19 +230,19 @@ public:
*/
void finish(uint8_t tag[16]) noexcept;
private:
private:
#ifdef ZT_AES_AESNI
void p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept;
void p_aesNIUpdate(const uint8_t* in, unsigned int len) noexcept;
void p_aesNIFinish(uint8_t tag[16]) noexcept;
#endif
#ifdef ZT_AES_NEON
void p_armUpdate(const uint8_t *in, unsigned int len) noexcept;
void p_armUpdate(const uint8_t* in, unsigned int len) noexcept;
void p_armFinish(uint8_t tag[16]) noexcept;
#endif
const AES &_aes;
const AES& _aes;
unsigned int _rp;
unsigned int _len;
uint8_t _r[16]; // remainder
uint8_t _r[16]; // remainder
uint8_t _iv[16];
uint64_t _y[2];
};
@ -249,14 +253,14 @@ public:
* NOTE: this doesn't support overflow of the counter in the least significant 32 bits.
* AES-GMAC-CTR doesn't need this, so we don't support it as an optimization.
*/
class CTR
{
class CTR {
friend class GMACSIVEncryptor;
friend class GMACSIVDecryptor;
public:
ZT_INLINE CTR(const AES &aes) noexcept: _aes(aes)
{}
public:
ZT_INLINE CTR(const AES& aes) noexcept : _aes(aes)
{
}
/**
* Initialize this CTR instance to encrypt a new stream
@ -264,10 +268,10 @@ public:
* @param iv Unique initialization vector and initial 32-bit counter (least significant 32 bits, big-endian)
* @param output Buffer to which to store output (MUST be large enough for total bytes processed!)
*/
ZT_INLINE void init(const uint8_t iv[16], void *const output) noexcept
ZT_INLINE void init(const uint8_t iv[16], void* const output) noexcept
{
Utils::copy< 16 >(_ctr, iv);
_out = reinterpret_cast<uint8_t *>(output);
Utils::copy<16>(_ctr, iv);
_out = reinterpret_cast<uint8_t*>(output);
_len = 0;
}
@ -278,11 +282,11 @@ public:
* @param ic Initial counter (must be in big-endian byte order!)
* @param output Buffer to which to store output (MUST be large enough for total bytes processed!)
*/
ZT_INLINE void init(const uint8_t iv[12], const uint32_t ic, void *const output) noexcept
ZT_INLINE void init(const uint8_t iv[12], const uint32_t ic, void* const output) noexcept
{
Utils::copy< 12 >(_ctr, iv);
reinterpret_cast<uint32_t *>(_ctr)[3] = ic;
_out = reinterpret_cast<uint8_t *>(output);
Utils::copy<12>(_ctr, iv);
reinterpret_cast<uint32_t*>(_ctr)[3] = ic;
_out = reinterpret_cast<uint8_t*>(output);
_len = 0;
}
@ -292,7 +296,7 @@ public:
* @param input Input data
* @param len Length of input
*/
void crypt(const void *input, unsigned int len) noexcept;
void crypt(const void* input, unsigned int len) noexcept;
/**
* Finish any remaining bytes if total bytes processed wasn't a multiple of 16
@ -301,16 +305,16 @@ public:
*/
void finish() noexcept;
private:
private:
#ifdef ZT_AES_AESNI
void p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) noexcept;
void p_aesNICrypt(const uint8_t* in, uint8_t* out, unsigned int len) noexcept;
#endif
#ifdef ZT_AES_NEON
void p_armCrypt(const uint8_t *in, uint8_t *out, unsigned int len) noexcept;
void p_armCrypt(const uint8_t* in, uint8_t* out, unsigned int len) noexcept;
#endif
const AES &_aes;
const AES& _aes;
uint64_t _ctr[2];
uint8_t *_out;
uint8_t* _out;
unsigned int _len;
};
@ -325,19 +329,19 @@ public:
* This supports encryption of a maximum of 2^31 bytes of data per
* call to init().
*/
class GMACSIVEncryptor
{
public:
class GMACSIVEncryptor {
public:
/**
* Create a new AES-GMAC-SIV encryptor keyed with the provided AES instances
*
* @param k0 First of two AES instances keyed with K0
* @param k1 Second of two AES instances keyed with K1
*/
ZT_INLINE GMACSIVEncryptor(const AES &k0, const AES &k1) noexcept :
_gmac(k0),
_ctr(k1)
{}
ZT_INLINE GMACSIVEncryptor(const AES& k0, const AES& k1) noexcept
: _gmac(k0)
, _ctr(k1)
{
}
/**
* Initialize AES-GMAC-SIV
@ -345,7 +349,7 @@ public:
* @param iv IV in network byte order (byte order in which it will appear on the wire)
* @param output Pointer to buffer to receive ciphertext, must be large enough for all to-be-processed data!
*/
ZT_INLINE void init(const uint64_t iv, void *const output) noexcept
ZT_INLINE void init(const uint64_t iv, void* const output) noexcept
{
// Output buffer to receive the result of AES-CTR encryption.
_output = output;
@ -353,7 +357,7 @@ public:
// Initialize GMAC with 64-bit IV (and remaining 32 bits padded to zero).
_tag[0] = iv;
_tag[1] = 0;
_gmac.init(reinterpret_cast<const uint8_t *>(_tag));
_gmac.init(reinterpret_cast<const uint8_t*>(_tag));
}
/**
@ -367,7 +371,7 @@ public:
* @param aad Additional authenticated data
* @param len Length of AAD in bytes
*/
ZT_INLINE void aad(const void *const aad, unsigned int len) noexcept
ZT_INLINE void aad(const void* const aad, unsigned int len) noexcept
{
// Feed ADD into GMAC first
_gmac.update(aad, len);
@ -385,8 +389,10 @@ public:
* @param input Plaintext chunk
* @param len Length of plaintext chunk
*/
ZT_INLINE void update1(const void *const input, const unsigned int len) noexcept
{ _gmac.update(input, len); }
ZT_INLINE void update1(const void* const input, const unsigned int len) noexcept
{
_gmac.update(input, len);
}
/**
* Finish first pass, compute CTR IV, initialize second pass.
@ -395,7 +401,7 @@ public:
{
// Compute 128-bit GMAC tag.
uint64_t tmp[2];
_gmac.finish(reinterpret_cast<uint8_t *>(tmp));
_gmac.finish(reinterpret_cast<uint8_t*>(tmp));
// Shorten to 64 bits, concatenate with message IV, and encrypt with AES to
// yield the CTR IV and opaque IV/MAC blob. In ZeroTier's use of GMAC-SIV
@ -415,7 +421,7 @@ public:
// and so 2^31 should be considered the input limit.
tmp[0] = _tag[0];
tmp[1] = _tag[1] & ZT_CONST_TO_BE_UINT64(0xffffffff7fffffffULL);
_ctr.init(reinterpret_cast<const uint8_t *>(tmp), _output);
_ctr.init(reinterpret_cast<const uint8_t*>(tmp), _output);
}
/**
@ -427,8 +433,10 @@ public:
* @param input Plaintext chunk
* @param len Length of plaintext chunk
*/
ZT_INLINE void update2(const void *const input, const unsigned int len) noexcept
{ _ctr.crypt(input, len); }
ZT_INLINE void update2(const void* const input, const unsigned int len) noexcept
{
_ctr.crypt(input, len);
}
/**
* Finish second pass and return a pointer to the opaque 128-bit IV+MAC block
@ -438,14 +446,14 @@ public:
*
* @return Pointer to 128-bit opaque IV+MAC (packed into two 64-bit integers)
*/
ZT_INLINE const uint64_t *finish2()
ZT_INLINE const uint64_t* finish2()
{
_ctr.finish();
return _tag;
}
private:
void *_output;
private:
void* _output;
uint64_t _tag[2];
AES::GMAC _gmac;
AES::CTR _ctr;
@ -456,13 +464,13 @@ public:
*
* GMAC-SIV decryption is single-pass. AAD (if any) must be processed first.
*/
class GMACSIVDecryptor
{
public:
ZT_INLINE GMACSIVDecryptor(const AES &k0, const AES &k1) noexcept:
_ctr(k1),
_gmac(k0)
{}
class GMACSIVDecryptor {
public:
ZT_INLINE GMACSIVDecryptor(const AES& k0, const AES& k1) noexcept
: _ctr(k1)
, _gmac(k0)
{
}
/**
* Initialize decryptor for a new message
@ -470,18 +478,18 @@ public:
* @param tag 128-bit combined IV/MAC originally created by GMAC-SIV encryption
* @param output Buffer in which to write output plaintext (must be large enough!)
*/
ZT_INLINE void init(const uint64_t tag[2], void *const output) noexcept
ZT_INLINE void init(const uint64_t tag[2], void* const output) noexcept
{
uint64_t tmp[2];
tmp[0] = tag[0];
tmp[1] = tag[1] & ZT_CONST_TO_BE_UINT64(0xffffffff7fffffffULL);
_ctr.init(reinterpret_cast<const uint8_t *>(tmp), output);
_ctr.init(reinterpret_cast<const uint8_t*>(tmp), output);
_ctr._aes.decrypt(tag, _ivMac);
tmp[0] = _ivMac[0];
tmp[1] = 0;
_gmac.init(reinterpret_cast<const uint8_t *>(tmp));
_gmac.init(reinterpret_cast<const uint8_t*>(tmp));
_output = output;
_decryptedLen = 0;
@ -493,7 +501,7 @@ public:
* @param aad Additional authenticated data
* @param len Length of AAD in bytes
*/
ZT_INLINE void aad(const void *const aad, unsigned int len) noexcept
ZT_INLINE void aad(const void* const aad, unsigned int len) noexcept
{
_gmac.update(aad, len);
len &= 0xfU;
@ -510,7 +518,7 @@ public:
* @param input Input ciphertext
* @param len Length of ciphertext
*/
ZT_INLINE void update(const void *const input, const unsigned int len) noexcept
ZT_INLINE void update(const void* const input, const unsigned int len) noexcept
{
_ctr.crypt(input, len);
_decryptedLen += len;
@ -527,52 +535,48 @@ public:
uint64_t gmacTag[2];
_gmac.update(_output, _decryptedLen);
_gmac.finish(reinterpret_cast<uint8_t *>(gmacTag));
_gmac.finish(reinterpret_cast<uint8_t*>(gmacTag));
return (gmacTag[0] ^ gmacTag[1]) == _ivMac[1];
}
private:
private:
uint64_t _ivMac[2];
AES::CTR _ctr;
AES::GMAC _gmac;
void *_output;
void* _output;
unsigned int _decryptedLen;
};
private:
private:
static const uint32_t Te0[256];
static const uint32_t Te4[256];
static const uint32_t Td0[256];
static const uint8_t Td4[256];
static const uint32_t rcon[15];
void p_initSW(const uint8_t *key) noexcept;
void p_encryptSW(const uint8_t *in, uint8_t *out) const noexcept;
void p_decryptSW(const uint8_t *in, uint8_t *out) const noexcept;
void p_initSW(const uint8_t* key) noexcept;
void p_encryptSW(const uint8_t* in, uint8_t* out) const noexcept;
void p_decryptSW(const uint8_t* in, uint8_t* out) const noexcept;
union
{
union {
#ifdef ZT_AES_AESNI
struct
{
struct {
__m128i k[28];
__m128i h[4]; // h, hh, hhh, hhhh
__m128i h2[4]; // _mm_xor_si128(_mm_shuffle_epi32(h, 78), h), etc.
__m128i h[4]; // h, hh, hhh, hhhh
__m128i h2[4]; // _mm_xor_si128(_mm_shuffle_epi32(h, 78), h), etc.
} ni;
#endif
#ifdef ZT_AES_NEON
struct
{
uint64_t hsw[2]; // in case it has AES but not PMULL, not sure if that ever happens
struct {
uint64_t hsw[2]; // in case it has AES but not PMULL, not sure if that ever happens
uint8x16_t ek[15];
uint8x16_t dk[15];
uint8x16_t h;
} neon;
#endif
struct
{
struct {
uint64_t h[2];
uint32_t ek[60];
uint32_t dk[60];
@ -580,18 +584,18 @@ private:
} p_k;
#ifdef ZT_AES_AESNI
void p_init_aesni(const uint8_t *key) noexcept;
void p_encrypt_aesni(const void *in, void *out) const noexcept;
void p_decrypt_aesni(const void *in, void *out) const noexcept;
void p_init_aesni(const uint8_t* key) noexcept;
void p_encrypt_aesni(const void* in, void* out) const noexcept;
void p_decrypt_aesni(const void* in, void* out) const noexcept;
#endif
#ifdef ZT_AES_NEON
void p_init_armneon_crypto(const uint8_t *key) noexcept;
void p_encrypt_armneon_crypto(const void *in, void *out) const noexcept;
void p_decrypt_armneon_crypto(const void *in, void *out) const noexcept;
void p_init_armneon_crypto(const uint8_t* key) noexcept;
void p_encrypt_armneon_crypto(const void* in, void* out) const noexcept;
void p_decrypt_armneon_crypto(const void* in, void* out) const noexcept;
#endif
};
} // namespace ZeroTier
} // namespace ZeroTier
#endif