diff --git a/.github/workflows/validate-linux.sh b/.github/workflows/validate-linux.sh deleted file mode 100755 index a661f6f6c..000000000 --- a/.github/workflows/validate-linux.sh +++ /dev/null @@ -1,497 +0,0 @@ -#!/bin/bash - -# This test script joins Earth and pokes some stuff - -TEST_NETWORK=8056c2e21c000001 -RUN_LENGTH=30 -TEST_FINISHED=false -ZTO_VER=$(git describe --tags $(git rev-list --tags --max-count=1)) -ZTO_COMMIT=$(git rev-parse HEAD) -ZTO_COMMIT_SHORT=$(git rev-parse --short HEAD) -TEST_DIR_PREFIX="$ZTO_VER-$ZTO_COMMIT_SHORT-test-results" - -TEST_OK=0 -TEST_FAIL=1 - -echo "Performing test on: $ZTO_VER-$ZTO_COMMIT_SHORT" -TEST_FILEPATH_PREFIX="$TEST_DIR_PREFIX/$ZTO_COMMIT_SHORT" -mkdir $TEST_DIR_PREFIX - -# How long we will wait for ZT to come online before considering it a failure -MAX_WAIT_SECS=30 - -ZT_PORT_NODE_1=9996 -ZT_PORT_NODE_2=9997 - -################################################################################ -# Multi-node connectivity and performance test # -################################################################################ - -test() { - - echo -e "\nPerforming pre-flight checks" - - check_exit_on_invalid_identity - - echo -e "\nRunning test for $RUN_LENGTH seconds" - - export NS1="ip netns exec ns1" - export NS2="ip netns exec ns2" - - export ZT1="$NS1 ./zerotier-cli -p9996 -D$(pwd)/node1" - # Specify custom port on one node to ensure that feature works - export ZT2="$NS2 ./zerotier-cli -p9997 -D$(pwd)/node2" - - echo -e "\nSetting up network namespaces..." - echo "Setting up ns1" - - ip netns add ns1 - $NS1 ip link set dev lo up - ip link add veth0 type veth peer name veth1 - ip link set veth1 netns ns1 - ip addr add 192.168.0.1/24 dev veth0 - ip link set dev veth0 up - - $NS1 ip addr add 192.168.0.2/24 dev veth1 - $NS1 ip link set dev veth1 up - - # Add default route - $NS1 ip route add default via 192.168.0.1 - - iptables -t nat -A POSTROUTING -s 192.168.0.0/255.255.255.0 \ - -o eth0 -j MASQUERADE - iptables -A FORWARD -i eth0 -o veth0 -j ACCEPT - iptables -A FORWARD -o eth0 -i veth0 -j ACCEPT - - echo "Setting up ns2" - ip netns add ns2 - $NS2 ip link set dev lo up - ip link add veth2 type veth peer name veth3 - ip link set veth3 netns ns2 - ip addr add 192.168.1.1/24 dev veth2 - ip link set dev veth2 up - - $NS2 ip addr add 192.168.1.2/24 dev veth3 - $NS2 ip link set dev veth3 up - $NS2 ip route add default via 192.168.1.1 - - iptables -t nat -A POSTROUTING -s 192.168.1.0/255.255.255.0 \ - -o eth0 -j MASQUERADE - iptables -A FORWARD -i eth0 -o veth2 -j ACCEPT - iptables -A FORWARD -o eth0 -i veth2 -j ACCEPT - - # Allow forwarding - sysctl -w net.ipv4.ip_forward=1 - - ################################################################################ - # Memory Leak Check # - ################################################################################ - - export FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log" - - echo -e "\nStarting a ZeroTier instance in each namespace..." - - export time_test_start=$(date +%s) - - # Spam the CLI as ZeroTier is starting - spam_cli 100 - - echo "Starting memory leak check" - $NS1 sudo valgrind --demangle=yes --exit-on-first-error=yes \ - --error-exitcode=1 \ - --xml=yes \ - --xml-file=$FILENAME_MEMORY_LOG \ - --leak-check=full \ - ./zerotier-one node1 -p$ZT_PORT_NODE_1 -U >>node_1.log 2>&1 & - - # Second instance, not run in memory profiler - # Don't set up internet access until _after_ zerotier is running - # This has been a source of stuckness in the past. - $NS2 ip addr del 192.168.1.2/24 dev veth3 - $NS2 sudo ./zerotier-one node2 -U -p$ZT_PORT_NODE_2 >>node_2.log 2>&1 & - - sleep 10; # New HTTP control plane is a bit sluggish, so we delay here - - check_bind_to_correct_ports $ZT_PORT_NODE_1 - check_bind_to_correct_ports $ZT_PORT_NODE_2 - - $NS2 ip addr add 192.168.1.2/24 dev veth3 - $NS2 ip route add default via 192.168.1.1 - - echo -e "\nPing from host to namespaces" - - ping -c 3 192.168.0.1 - ping -c 3 192.168.1.1 - - echo -e "\nPing from namespace to host" - - $NS1 ping -c 3 192.168.0.1 - $NS1 ping -c 3 192.168.0.1 - $NS2 ping -c 3 192.168.0.2 - $NS2 ping -c 3 192.168.0.2 - - echo -e "\nPing from ns1 to ns2" - - $NS1 ping -c 3 192.168.0.1 - - echo -e "\nPing from ns2 to ns1" - - $NS2 ping -c 3 192.168.0.1 - - ################################################################################ - # Online Check # - ################################################################################ - - echo "Waiting for ZeroTier to come online before attempting test..." - node1_online=false - node2_online=false - both_instances_online=false - time_zt_node1_start=$(date +%s) - time_zt_node2_start=$(date +%s) - - for ((s = 0; s <= $MAX_WAIT_SECS; s++)); do - node1_online="$($ZT1 -j info | jq '.online' 2>/dev/null)" - node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)" - echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online" - if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]; then - export both_instances_online=true - export time_to_both_nodes_online=$(date +%s) - break - fi - sleep 1 - done - - echo -e "\n\nContents of ZeroTier home paths:" - - ls -lga node1 - tree node1 - ls -lga node2 - tree node2 - - echo -e "\n\nRunning ZeroTier processes:" - echo -e "\nNode 1:\n" - $NS1 ps aux | grep zerotier-one - echo -e "\nNode 2:\n" - $NS2 ps aux | grep zerotier-one - - echo -e "\n\nStatus of each instance:" - - echo -e "\n\nNode 1:\n" - $ZT1 status - echo -e "\n\nNode 2:\n" - $ZT2 status - - if [[ "$both_instances_online" != "true" ]]; then - exit_test_and_generate_report $TEST_FAIL "one or more nodes failed to come online" - fi - - echo -e "\nJoining networks" - - $ZT1 join $TEST_NETWORK - $ZT2 join $TEST_NETWORK - - sleep 10 - - node1_ip4=$($ZT1 get $TEST_NETWORK ip4) - node2_ip4=$($ZT2 get $TEST_NETWORK ip4) - - echo "node1_ip4=$node1_ip4" - echo "node2_ip4=$node2_ip4" - - echo -e "\nPinging each node" - - PING12_FILENAME="$TEST_FILEPATH_PREFIX-ping-1-to-2.txt" - PING21_FILENAME="$TEST_FILEPATH_PREFIX-ping-2-to-1.txt" - - $NS1 ping -c 16 $node2_ip4 >$PING12_FILENAME - $NS2 ping -c 16 $node1_ip4 >$PING21_FILENAME - - ping_loss_percent_1_to_2=$(cat $PING12_FILENAME | - grep "packet loss" | awk '{print $6}' | sed 's/%//') - ping_loss_percent_2_to_1=$(cat $PING21_FILENAME | - grep "packet loss" | awk '{print $6}' | sed 's/%//') - - # Normalize loss value - export ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc) - export ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc) - - ################################################################################ - # CLI Check # - ################################################################################ - - echo "Testing basic CLI functionality..." - - spam_cli 10 - - $ZT1 join $TEST_NETWORK - - $ZT1 -h - $ZT1 -v - $ZT1 status - $ZT1 info - $ZT1 listnetworks - $ZT1 peers - $ZT1 listpeers - - $ZT1 -j status - $ZT1 -j info - $ZT1 -j listnetworks - $ZT1 -j peers - $ZT1 -j listpeers - - $ZT1 dump - - $ZT1 get $TEST_NETWORK allowDNS - $ZT1 get $TEST_NETWORK allowDefault - $ZT1 get $TEST_NETWORK allowGlobal - $ZT1 get $TEST_NETWORK allowManaged - $ZT1 get $TEST_NETWORK bridge - $ZT1 get $TEST_NETWORK broadcastEnabled - $ZT1 get $TEST_NETWORK dhcp - $ZT1 get $TEST_NETWORK id - $ZT1 get $TEST_NETWORK mac - $ZT1 get $TEST_NETWORK mtu - $ZT1 get $TEST_NETWORK name - $ZT1 get $TEST_NETWORK netconfRevision - $ZT1 get $TEST_NETWORK nwid - $ZT1 get $TEST_NETWORK portDeviceName - $ZT1 get $TEST_NETWORK portError - $ZT1 get $TEST_NETWORK status - $ZT1 get $TEST_NETWORK type - - # Test an invalid command - $ZT1 get $TEST_NETWORK derpderp - - # TODO: Validate JSON - - # Performance Test - - export FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json" - - echo -e "\nBeginning performance test:" - - echo -e "\nStarting server:" - - echo "$NS1 iperf3 -s &" - sleep 1 - - echo -e "\nStarting client:" - sleep 1 - - echo "$NS2 iperf3 --json -c $node1_ip4 > $FILENAME_PERF_JSON" - - cat $FILENAME_PERF_JSON - - # Let ZeroTier idle long enough for various timers - - echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..." - sleep $RUN_LENGTH - - echo -e "\nLeaving networks" - - $ZT1 leave $TEST_NETWORK - $ZT2 leave $TEST_NETWORK - - sleep 5 - - exit_test_and_generate_report $TEST_OK "completed test" -} - -################################################################################ -# Generate report # -################################################################################ - -exit_test_and_generate_report() { - - echo -e "\nStopping memory check..." - sudo pkill -15 -f valgrind - sleep 10 - - time_test_end=$(date +%s) - - echo "Exiting test with reason: $2 ($1)" - - # Collect ZeroTier dump files - - echo -e "\nCollecting ZeroTier dump files" - - node1_id=$($ZT1 -j status | jq -r .address) - node2_id=$($ZT2 -j status | jq -r .address) - - $ZT1 dump - mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt" - - $ZT2 dump - mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt" - - # Copy ZeroTier stdout/stderr logs - - cp node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt" - cp node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt" - - # Generate report - - cat $FILENAME_MEMORY_LOG - - DEFINITELY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \ - $FILENAME_MEMORY_LOG | grep "definitely" | awk '{print $1;}') - POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \ - $FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}') - - # Generate coverage report artifact and summary - - FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json" - FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html" - - echo -e "\nGenerating coverage test report..." - - gcovr -r . --exclude ext --json-summary $FILENAME_COVERAGE_JSON \ - --html >$FILENAME_COVERAGE_HTML - - cat $FILENAME_COVERAGE_JSON - - COVERAGE_LINE_COVERED=$(cat $FILENAME_COVERAGE_JSON | jq .line_covered) - COVERAGE_LINE_TOTAL=$(cat $FILENAME_COVERAGE_JSON | jq .line_total) - COVERAGE_LINE_PERCENT=$(cat $FILENAME_COVERAGE_JSON | jq .line_percent) - - COVERAGE_LINE_COVERED="${COVERAGE_LINE_COVERED:-0}" - COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}" - COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}" - - # Default values - - DEFINITELY_LOST="${DEFINITELY_LOST:-0}" - POSSIBLY_LOST="${POSSIBLY_LOST:-0}" - ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}" - ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}" - time_to_both_nodes_online="${time_to_both_nodes_online:--1}" - - # Summarize and emit json for trend reporting - - FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json" - - time_length_test=$((time_test_end - time_test_start)) - if [[ $time_to_both_nodes_online != -1 ]]; - then - time_to_both_nodes_online=$((time_to_both_nodes_online - time_test_start)) - fi - #time_length_zt_join=$((time_zt_join_end-time_zt_join_start)) - #time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start)) - #time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start)) - - summary=$( - cat <$FILENAME_SUMMARY - cat $FILENAME_SUMMARY - - exit 0 -} - -################################################################################ -# CLI Check # -################################################################################ - -spam_cli() { - echo "Spamming CLI..." - # Rapidly spam the CLI with joins/leaves - - MAX_TRIES="${1:-10}" - - for ((s = 0; s <= MAX_TRIES; s++)); do - $ZT1 status - $ZT2 status - sleep 0.1 - done - - SPAM_TRIES=128 - - for ((s = 0; s <= SPAM_TRIES; s++)); do - $ZT1 join $TEST_NETWORK - done - - for ((s = 0; s <= SPAM_TRIES; s++)); do - $ZT1 leave $TEST_NETWORK - done - - for ((s = 0; s <= SPAM_TRIES; s++)); do - $ZT1 leave $TEST_NETWORK - $ZT1 join $TEST_NETWORK - done -} - -################################################################################ -# Check for proper exit on load of invalid identity # -################################################################################ - -check_exit_on_invalid_identity() { - echo "Checking ZeroTier exits on invalid identity..." - mkdir -p $(pwd)/exit_test - ZT1="sudo ./zerotier-one -p9999 $(pwd)/exit_test" - echo "asdfasdfasdfasdf" > $(pwd)/exit_test/identity.secret - echo "asdfasdfasdfasdf" > $(pwd)/exit_test/authtoken.secret - - echo "Launch ZeroTier with an invalid identity" - $ZT1 & - my_pid=$! - - echo "Waiting 5 seconds" - sleep 5 - - # check if process is running - kill -0 $my_pid - if [ $? -eq 0 ]; then - exit_test_and_generate_report $TEST_FAIL "Exit test FAILED: Process still running after being fed an invalid identity" - fi -} - -################################################################################ -# Check that we're binding to the primary port for TCP/TCP6/UDP # -################################################################################ - -check_bind_to_correct_ports() { - PORT_NUMBER=$1 - echo "Checking bound ports:" - sudo netstat -anp | grep "$PORT_NUMBER" | grep "zerotier" - if [[ $(sudo netstat -anp | grep "$PORT_NUMBER" | grep "zerotier" | grep "tcp") ]]; - then - : - else - exit_test_and_generate_report $TEST_FAIL "ZeroTier did not bind to tcp/$1" - fi - if [[ $(sudo netstat -anp | grep "$PORT_NUMBER" | grep "zerotier" | grep "tcp6") ]]; - then - : - else - exit_test_and_generate_report $TEST_FAIL "ZeroTier did not bind to tcp6/$1" - fi - if [[ $(sudo netstat -anp | grep "$PORT_NUMBER" | grep "zerotier" | grep "udp") ]]; - then - : - else - exit_test_and_generate_report $TEST_FAIL "ZeroTier did not bind to udp/$1" - fi -} - -test "$@"