# -*- bash -*- # Netavark binary to run NETAVARK=${NETAVARK:-./bin/netavark} NETAVARK_CONNECTION_TESTER=${NETAVARK_CONNECTION_TESTER:-./bin/netavark-connection-tester} TESTSDIR=${TESTSDIR:-$(dirname ${BASH_SOURCE})} # export RUST_BACKTRACE so that we get a helpful stack trace export RUST_BACKTRACE=full # this will cause tests to fail because stdou/stderr are not separate # export RUST_LOG=netavark=debug HOST_NS_PID= CONTAINER_NS_PIDS=() function create_container_ns() { CONTAINER_NS_PIDS+=("$(create_netns)") } function basic_setup() { HOST_NS_PID=$(create_netns) create_container_ns # make sure to set DBUS_SYSTEM_BUS_ADDRESS to an empty value # netavark will try to use firewalld connection when possible # because we run in a separate netns we cannot use firewalld # firewalld run in the host netns and not our custom netns # thus the firewall rules end up in the wrong netns # unsetting does not work, it would use the default address export DBUS_SYSTEM_BUS_ADDRESS= NETAVARK_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} netavark_bats.XXXXXX) # hack to make aardvark-dns run when really root or when running as user with # podman unshare --rootless-netns; since netavark runs aardvark with systemd-run # it needs to know if it should use systemd user instance or not. # iptables are still setup identically. rootless=false if [[ ! -e "/run/dbus/system_bus_socket" ]]; then rootless=true fi mkdir -p "$NETAVARK_TMPDIR/config" run_in_host_netns ip link set lo up } function basic_teardown() { teardown_firewalld kill -9 $HOST_NS_PID for i in "${!CONTAINER_NS_PIDS[@]}"; do kill -9 "${CONTAINER_NS_PIDS[$i]}" done rm -rf "$NETAVARK_TMPDIR" } function setup_firewalld() { # first, create a new dbus session DBUS_SYSTEM_BUS_ADDRESS=unix:path=$NETAVARK_TMPDIR/netavark-firewalld run_in_host_netns dbus-daemon --address="$DBUS_SYSTEM_BUS_ADDRESS" --print-pid --config-file="${TESTSDIR}/testfiles/firewalld-dbus.conf" DBUS_PID="$output" # export DBUS_SYSTEM_BUS_ADDRESS so firewalld and netavark will use the correct socket export DBUS_SYSTEM_BUS_ADDRESS # second, start firewalld in the netns with the dbus socket # do not use run_in_host_netns because we want to run this in background # use --nopid (we cannot change the pid file location), --nofork do not run as daemon so we can kill it by pid # change --system-config to make sure that we do not write any config files to the host location nsenter -n -t $HOST_NS_PID firewalld --nopid --nofork --system-config "$NETAVARK_TMPDIR" &>"$NETAVARK_TMPDIR/firewalld.log" & FIREWALLD_PID=$! echo "firewalld pid: $FIREWALLD_PID" # wait for firewalld to become ready timeout=5 while [ $timeout -gt 0 ]; do # query firewalld with firewall-cmd expected_rc="?" run_in_host_netns firewall-cmd --state if [ "$status" -eq 0 ]; then break fi sleep 1 timeout=$(($timeout - 1)) if [ $timeout -eq 0 ]; then cat "$NETAVARK_TMPDIR/firewalld.log" die "failed to start firewalld - timeout" fi done } function teardown_firewalld() { if [ -n "${NETAVARK_FIREWALLD_RELOAD_PID}" ]; then kill -9 $NETAVARK_FIREWALLD_RELOAD_PID fi if [ -n "${FIREWALLD_PID}" ]; then kill -9 $FIREWALLD_PID fi if [ -n "${DBUS_PID}" ]; then kill -9 $DBUS_PID fi unset DBUS_SYSTEM_BUS_ADDRESS } # Provide the above as default methods. function setup() { basic_setup } function teardown() { basic_teardown } function create_netns() { # create a new netns and mountns and run a sleep process to keep it alive # we have to redirect stdout/err to /dev/null otherwise bats will hang unshare -nm --propagation private sleep inf &>/dev/null & # netavark writes to /run/sysctl.d, mount a tmpfs to not leak stuff on the host mkdir -p /run/sysctl.d nsenter -n -m -w -t $! mount -t tmpfs none /run/sysctl.d echo $! } function get_container_netns_path() { local which="0" if [[ $# -eq 1 ]]; then which=$1 fi echo /proc/"${CONTAINER_NS_PIDS[$which]}"/ns/net } ################ # run_netavark # Invoke $NETAVARK, with timeout, using BATS 'run' ################ # # This is the preferred mechanism for invoking netavark: first, it # it joins the test network namespace before it invokes $NETAVARK, # which may be 'netavark' or '/some/path/netavark'. function run_netavark() { run_in_host_netns $NETAVARK --rootless "$rootless" \ --config "$NETAVARK_TMPDIR/config" "$@" } function run_netavark_firewalld_reload() { # need to use nsetner as this will be run in the background nsenter -n -t $HOST_NS_PID $NETAVARK --config "$NETAVARK_TMPDIR/config" firewalld-reload & NETAVARK_FIREWALLD_RELOAD_PID=$! } ################ # run_in_container_netns # Run args in container netns ################ # function run_in_container_netns() { local i="0" isnum='^[0-9]+$' if [[ $1 =~ $isnum ]]; then i=$1 shift 1 fi run_helper nsenter -n -m -w -t "${CONTAINER_NS_PIDS[$i]}" "$@" } ################ # run_in_host_netns # Run args in host netns ################ function run_in_host_netns() { run_helper nsenter -n -m -w -t $HOST_NS_PID "$@" } #### Functions below are taken from podman and buildah and adapted to netavark. ################ # run_helper # Invoke args, with timeout, using BATS 'run' ################ # # Second, we use 'timeout' to abort (with a diagnostic) if something # takes too long; this is preferable to a CI hang. # # Third, we log the command run and its output. This doesn't normally # appear in BATS output, but it will if there's an error. # # Next, we check exit status. Since the normal desired code is 0, # that's the default; but the expected_rc var can override: # # expected_rc=125 run_helper nonexistent-subcommand # expected_rc=? run_helper some-other-command # let our caller check status # # Since we use the BATS 'run' mechanism, $output and $status will be # defined for our caller. # function run_helper() { # expected_rc if unset set default to 0 expected_rc="${expected_rc-0}" if [ "$expected_rc" == "?" ]; then expected_rc= fi # Remember command args, for possible use in later diagnostic messages MOST_RECENT_COMMAND="$*" # stdout is only emitted upon error; this echo is to help a debugger echo "$_LOG_PROMPT $*" # BATS hangs if a subprocess remains and keeps FD 3 open; this happens # if a process crashes unexpectedly without cleaning up subprocesses. run timeout --foreground -v --kill=10 10 "$@" 3>/dev/null # without "quotes", multiple lines are glommed together into one if [ -n "$output" ]; then echo "$output" fi if [ "$status" -ne 0 ]; then echo -n "[ rc=$status " if [ -n "$expected_rc" ]; then if [ "$status" -eq "$expected_rc" ]; then echo -n "(expected) " else echo -n "(** EXPECTED $expected_rc **) " fi fi echo "]" fi if [ "$status" -eq 124 ]; then if expr "$output" : ".*timeout: sending" >/dev/null; then # It's possible for a subtest to _want_ a timeout if [[ "$expected_rc" != "124" ]]; then echo "*** TIMED OUT ***" false fi fi fi if [ -n "$expected_rc" ]; then if [ "$status" -ne "$expected_rc" ]; then die "exit code is $status; expected $expected_rc" fi fi # unset unset expected_rc } ######### # die # Abort with helpful message ######### function die() { # FIXME: handle multi-line output echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 echo "#| FAIL: $*" >&2 echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 false } ############ # assert # Compare actual vs expected string; fail if mismatch ############ # # Compares string (default: $output) against the given string argument. # By default we do an exact-match comparison against $output, but there # are two different ways to invoke us, each with an optional description: # # xpect "EXPECT" [DESCRIPTION] # xpect "RESULT" "OP" "EXPECT" [DESCRIPTION] # # The first form (one or two arguments) does an exact-match comparison # of "$output" against "EXPECT". The second (three or four args) compares # the first parameter against EXPECT, using the given OPerator. If present, # DESCRIPTION will be displayed on test failure. # # Examples: # # xpect "this is exactly what we expect" # xpect "${lines[0]}" =~ "^abc" "first line begins with abc" # function assert() { local actual_string="$output" local operator='==' local expect_string="$1" local testname="$2" case "${#*}" in 0) die "Internal error: 'assert' requires one or more arguments" ;; 1 | 2) ;; 3 | 4) actual_string="$1" operator="$2" expect_string="$3" testname="$4" ;; *) die "Internal error: too many arguments to 'assert'" ;; esac # Comparisons. # Special case: there is no !~ operator, so fake it via '! x =~ y' local not= local actual_op="$operator" if [[ $operator == '!~' ]]; then not='!' actual_op='=~' fi if [[ $operator == '=' || $operator == '==' ]]; then # Special case: we can't use '=' or '==' inside [[ ... ]] because # the right-hand side is treated as a pattern... and '[xy]' will # not compare literally. There seems to be no way to turn that off. if [ "$actual_string" = "$expect_string" ]; then return fi elif [[ $operator == '!=' ]]; then # Same special case as above if [ "$actual_string" != "$expect_string" ]; then return fi else if eval "[[ $not \$actual_string $actual_op \$expect_string ]]"; then return elif [ $? -gt 1 ]; then die "Internal error: could not process 'actual' $operator 'expect'" fi fi # Test has failed. Get a descriptive test name. if [ -z "$testname" ]; then testname="${MOST_RECENT_BUILDAH_COMMAND:-[no test name given]}" fi # Display optimization: the typical case for 'expect' is an # exact match ('='), but there are also '=~' or '!~' or '-ge' # and the like. Omit the '=' but show the others; and always # align subsequent output lines for ease of comparison. local op='' local ws='' if [ "$operator" != '==' ]; then op="$operator " ws=$(printf "%*s" ${#op} "") fi # This is a multi-line message, which may in turn contain multi-line # output, so let's format it ourself, readably local actual_split IFS=$'\n' read -rd '' -a actual_split <<<"$actual_string" || true printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 printf "#| FAIL: %s\n" "$testname" >&2 printf "#| expected: %s'%s'\n" "$op" "$expect_string" >&2 printf "#| actual: %s'%s'\n" "$ws" "${actual_split[0]}" >&2 local line for line in "${actual_split[@]:1}"; do printf "#| > %s'%s'\n" "$ws" "$line" >&2 done printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 false } ################# # assert_json # Compare actual json vs expected string; fail if mismatch ################# # assert_json works like assert except that it accepts one extra parameter, # the jq query string. # There are two different ways to invoke us, each with an optional description: # # xpect "JQ_QUERY" "EXPECT" [DESCRIPTION] # xpect "JSON_STRING" "JQ_QUERY" "OP" "EXPECT" [DESCRIPTION] # Important this function will overwrite $output, so if you need to use the value # more than once you need to safe it in another variable. function assert_json() { local actual_json="$output" local operator='==' local jq_query="$1" local expect_string="$2" local testname="$3" case "${#*}" in 0 | 1) die "Internal error: 'assert_json' requires two or more arguments" ;; 2 | 3) ;; 4 | 5) actual_json="$1" jq_query="$2" operator="$3" expect_string="$4" testname="$5" ;; *) die "Internal error: too many arguments to 'assert_json'" ;; esac run_helper jq -r "$jq_query" <<<"$actual_json" assert "$output" "$operator" "$expect_string" "$testname" } ################## # test_port_fw # test port forwarding ################## # test port forwarding # by default this will create a ipv4 config with tcp as protocol # # The following arguments are supported, the order does not matter: # ip={4, 6, dual} # proto={tcp,udp,sctp} or some comma separated list of the protocols # hostip=$ip the ip which is used for binding on the host # hostport=$port the port which is binded on the host # containerport=$port the port which is binded in the container # range=$num >=1 specify a port range which will forward hostport+range ports # connectip=$ip the ip which is used to connect to in the connection test # firewalld_reload={false,true} call firewall-cmd --reload to check for port rules # function test_port_fw() { local ipv4=true local ipv6=false local proto=tcp local host_ip="" local host_port="" local container_port="" local range=1 local connect_ip="" local firewalld_reload=false # parse arguments while [[ "$#" -gt 0 ]]; do IFS='=' read -r arg value <<<"$1" case "$arg" in ip) case "$value" in 4) ipv4=true ;; 6) ipv6=true ipv4=false ;; dual) ipv6=true ;; *) die "unknown argument '$value' for ip=" ;; esac ;; proto) proto="$value" ;; hostip) host_ip="$value" ;; connectip) connect_ip="$value" ;; hostport) host_port="$value" ;; containerport) container_port="$value" ;; range) range="$value" ;; firewalld_reload) firewalld_reload="$value" ;; *) die "unknown argument for '$arg' test_port_fw" ;; esac shift done if [ -z "$host_port" ]; then host_port=$(random_port) fi if [ -z "$container_port" ]; then container_port=$(random_port) fi local container_id=$(random_string 64) local container_name="name-$(random_string 10)" local static_ips="" local subnets="" if [ $ipv4 = true ]; then ipv4_subnet=$(random_subnet) ipv4_gateway=$(gateway_from_subnet $ipv4_subnet) ipv4_container_ip=$(random_ip_in_subnet $ipv4_subnet) static_ips="\"$ipv4_container_ip\"" subnets="{\"subnet\":\"$ipv4_subnet\",\"gateway\":\"$ipv4_gateway\"}" fi if [ $ipv6 = true ]; then ipv6_subnet=$(random_subnet 6) ipv6_gateway=$(gateway_from_subnet $ipv6_subnet) ipv6_container_ip=$(random_ip_in_subnet $ipv6_subnet) if [ $ipv4 = true ]; then # add comma for the json static_ips="$static_ips, " subnets="$subnets, " fi static_ips="$static_ips\"$ipv6_container_ip\"" subnets="$subnets {\"subnet\":\"$ipv6_subnet\",\"gateway\":\"$ipv6_gateway\"}" fi read -r -d '\0' config <