wgengine/router: add a test for linux router state transitions.

Signed-off-by: David Anderson <danderson@tailscale.com>

wgengine/router/router_linux.go

@@ -53,6 +53,22 @@ const (
 // avoid allocating Tailscale IPs from it, to avoid conflicts.
 const chromeOSVMRange = "100.115.92.0/23"
 
+type netfilterRunner interface {
+	Insert(table, chain string, pos int, args ...string) error
+	Append(table, chain string, args ...string) error
+	Exists(table, chain string, args ...string) (bool, error)
+	Delete(table, chain string, args ...string) error
+	ListChains(table string) ([]string, error)
+	ClearChain(table, chain string) error
+	NewChain(table, chain string) error
+	DeleteChain(table, chain string) error
+}
+
+type commandRunner interface {
+	run(...string) error
+	output(...string) ([]byte, error)
+}
+
 type linuxRouter struct {
 	logf             func(fmt string, args ...interface{})
 	tunname          string
@@ -62,7 +78,8 @@ type linuxRouter struct {
 	snatSubnetRoutes bool
 	netfilterMode    NetfilterMode
 
-	ipt4 *iptables.IPTables
+	ipt4 netfilterRunner
+	cmd  commandRunner
 }
 
 func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (Router, error) {
@@ -76,25 +93,37 @@ func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (
 		return nil, err
 	}
 
+	return newUserspaceRouterAdvanced(logf, tunname, ipt4, osCommandRunner{})
+}
+
+func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter netfilterRunner, cmd commandRunner) (Router, error) {
 	return &linuxRouter{
 		logf:          logf,
 		tunname:       tunname,
 		netfilterMode: NetfilterOff,
-		ipt4:          ipt4,
+		ipt4:          netfilter,
+		cmd:           cmd,
 	}, nil
 }
 
-func cmd(args ...string) error {
+type osCommandRunner struct{}
+
+func (o osCommandRunner) run(args ...string) error {
+	_, err := o.output(args...)
+	return err
+}
+
+func (o osCommandRunner) output(args ...string) ([]byte, error) {
 	if len(args) == 0 {
-		return errors.New("cmd: no argv[0]")
+		return nil, errors.New("cmd: no argv[0]")
 	}
 
 	out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("running %q failed: %v\n%s", strings.Join(args, " "), err, out)
+		return nil, fmt.Errorf("running %q failed: %v\n%s", strings.Join(args, " "), err, out)
 	}
 
-	return nil
+	return out, nil
 }
 
 func (r *linuxRouter) Up() error {
@@ -384,7 +413,7 @@ func (r *linuxRouter) restoreResolvConf() error {
 // address is already assigned to the interface, or if the addition
 // fails.
 func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
-	if err := cmd("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
+	if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
 		return fmt.Errorf("adding address %q to tunnel interface: %v", addr, err)
 	}
 	if err := r.addLoopbackRule(addr.IP); err != nil {
@@ -400,7 +429,7 @@ func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error {
 	if err := r.delLoopbackRule(addr.IP); err != nil {
 		return err
 	}
-	if err := cmd("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
+	if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
 		return fmt.Errorf("deleting address %q from tunnel interface: %v", addr, err)
 	}
 	return nil
@@ -434,14 +463,14 @@ func (r *linuxRouter) delLoopbackRule(addr netaddr.IP) error {
 // interface. Fails if the route already exists, or if adding the
 // route fails.
 func (r *linuxRouter) addRoute(cidr netaddr.IPPrefix) error {
-	return cmd("ip", "route", "add", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
+	return r.cmd.run("ip", "route", "add", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
 }
 
 // delRoute removes the route for cidr pointing to the tunnel
 // interface. Fails if the route doesn't exist, or if removing the
 // route fails.
 func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error {
-	return cmd("ip", "route", "del", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
+	return r.cmd.run("ip", "route", "del", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
 }
 
 // addSubnetRule adds a netfilter rule that allows traffic to flow
@@ -480,13 +509,13 @@ func (r *linuxRouter) delSubnetRule(cidr netaddr.IPPrefix) error {
 // upInterface brings up the tunnel interface and adds it to the
 // Tailscale interface group.
 func (r *linuxRouter) upInterface() error {
-	return cmd("ip", "link", "set", "dev", r.tunname, "group", "10000", "up")
+	return r.cmd.run("ip", "link", "set", "dev", r.tunname, "group", "10000", "up")
 }
 
 // downInterface sets the tunnel interface administratively down, and
 // returns it to the default interface group.
 func (r *linuxRouter) downInterface() error {
-	return cmd("ip", "link", "set", "dev", r.tunname, "group", "0", "down")
+	return r.cmd.run("ip", "link", "set", "dev", r.tunname, "group", "0", "down")
 }
 
 // addBypassRule adds the policy routing rule that avoids tailscaled
@@ -496,13 +525,13 @@ func (r *linuxRouter) addBypassRule() error {
 	if err := r.delBypassRule(); err != nil {
 		return err
 	}
-	return cmd("ip", "rule", "add", "fwmark", tailscaleBypassMark, "priority", "10000", "table", "main", "suppress_ifgroup", "10000")
+	return r.cmd.run("ip", "rule", "add", "fwmark", tailscaleBypassMark, "priority", "10000", "table", "main", "suppress_ifgroup", "10000")
 }
 
 // delBypassrule removes the policy routing rule that avoids
 // tailscaled routing loops, if it exists.
 func (r *linuxRouter) delBypassRule() error {
-	out, err := exec.Command("ip", "rule", "list", "priority", "10000").CombinedOutput()
+	out, err := r.cmd.output("ip", "rule", "list", "priority", "10000")
 	if err != nil {
 		// Busybox ships an `ip` binary that doesn't understand
 		// uncommon rules. Try to detect this explicitly, and steer
@@ -523,7 +552,7 @@ func (r *linuxRouter) delBypassRule() error {
 	if !bytes.Contains(out, []byte(" fwmark "+tailscaleBypassMark)) {
 		return fmt.Errorf("ip rule 10000 doesn't look like a Tailscale policy rule: %q", string(out))
 	}
-	return cmd("ip", "rule", "del", "priority", "10000")
+	return r.cmd.run("ip", "rule", "del", "priority", "10000")
 }
 
 // addNetfilterBase adds custom Tailscale chains to netfilter, along

wgengine/router/router_linux_test.go

@@ -0,0 +1,516 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"inet.af/netaddr"
+)
+
+func mustCIDR(s string) netaddr.IPPrefix {
+	pfx, err := netaddr.ParseIPPrefix(s)
+	if err != nil {
+		panic(err)
+	}
+	return pfx
+}
+
+func mustCIDRs(ss ...string) []netaddr.IPPrefix {
+	var ret []netaddr.IPPrefix
+	for _, s := range ss {
+		ret = append(ret, mustCIDR(s))
+	}
+	return ret
+}
+
+func TestRouterStates(t *testing.T) {
+	states := []struct {
+		name string
+		in   *Config
+		want string
+	}{
+		{
+			name: "no config",
+			in:   nil,
+			want: `
+up
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+`,
+		},
+		{
+			name: "local addr only",
+			in: &Config{
+				LocalAddrs: mustCIDRs("100.101.102.103/10"),
+			},
+			want: `
+up
+ip addr add 100.101.102.103/10 dev tailscale0
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+`,
+		},
+
+		{
+			name: "addr and routes",
+			in: &Config{
+				LocalAddrs: mustCIDRs("100.101.102.103/10"),
+				Routes:     mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
+			},
+			want: `
+up
+ip addr add 100.101.102.103/10 dev tailscale0
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip route add 192.168.16.0/24 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+`,
+		},
+
+		{
+			name: "addr and routes and subnet routes",
+			in: &Config{
+				LocalAddrs:   mustCIDRs("100.101.102.103/10"),
+				Routes:       mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
+				SubnetRoutes: mustCIDRs("200.0.0.0/8"),
+			},
+			want: `
+up
+ip addr add 100.101.102.103/10 dev tailscale0
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip route add 192.168.16.0/24 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+`,
+		},
+
+		{
+			name: "addr and routes and subnet routes with netfilter",
+			in: &Config{
+				LocalAddrs:       mustCIDRs("100.101.102.104/10"),
+				Routes:           mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				SubnetRoutes:     mustCIDRs("200.0.0.0/8"),
+				SNATSubnetRoutes: true,
+				NetfilterMode:    NetfilterOn,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/FORWARD -j ts-forward
+filter/INPUT -j ts-input
+filter/ts-forward -o tailscale0 -s 200.0.0.0/8 -j ACCEPT
+filter/ts-forward -i tailscale0 -d 200.0.0.0/8 -j MARK --set-mark 0x10000/0x10000
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+nat/POSTROUTING -j ts-postrouting
+nat/ts-postrouting -m mark --mark 0x10000/0x10000 -j MASQUERADE
+`,
+		},
+		{
+			name: "addr and routes with netfilter",
+			in: &Config{
+				LocalAddrs:    mustCIDRs("100.101.102.104/10"),
+				Routes:        mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				NetfilterMode: NetfilterOn,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/FORWARD -j ts-forward
+filter/INPUT -j ts-input
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+nat/POSTROUTING -j ts-postrouting
+`,
+		},
+
+		{
+			name: "addr and routes and subnet routes with netfilter but no SNAT",
+			in: &Config{
+				LocalAddrs:       mustCIDRs("100.101.102.104/10"),
+				Routes:           mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				SubnetRoutes:     mustCIDRs("200.0.0.0/8"),
+				SNATSubnetRoutes: false,
+				NetfilterMode:    NetfilterOn,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/FORWARD -j ts-forward
+filter/INPUT -j ts-input
+filter/ts-forward -o tailscale0 -s 200.0.0.0/8 -j ACCEPT
+filter/ts-forward -i tailscale0 -d 200.0.0.0/8 -j MARK --set-mark 0x10000/0x10000
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+nat/POSTROUTING -j ts-postrouting
+`,
+		},
+		{
+			name: "addr and routes with netfilter",
+			in: &Config{
+				LocalAddrs:    mustCIDRs("100.101.102.104/10"),
+				Routes:        mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				NetfilterMode: NetfilterOn,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/FORWARD -j ts-forward
+filter/INPUT -j ts-input
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+nat/POSTROUTING -j ts-postrouting
+`,
+		},
+
+		{
+			name: "addr and routes with half netfilter",
+			in: &Config{
+				LocalAddrs:    mustCIDRs("100.101.102.104/10"),
+				Routes:        mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				NetfilterMode: NetfilterNoDivert,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+`,
+		},
+		{
+			name: "addr and routes with netfilter",
+			in: &Config{
+				LocalAddrs:    mustCIDRs("100.101.102.104/10"),
+				Routes:        mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
+				NetfilterMode: NetfilterOn,
+			},
+			want: `
+up
+ip addr add 100.101.102.104/10 dev tailscale0
+ip route add 10.0.0.0/8 dev tailscale0 scope global
+ip route add 100.100.100.100/32 dev tailscale0 scope global
+ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
+filter/FORWARD -j ts-forward
+filter/INPUT -j ts-input
+filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j DROP
+filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
+filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
+filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
+nat/POSTROUTING -j ts-postrouting
+`,
+		},
+	}
+
+	fake := NewFakeOS(t)
+	router, err := newUserspaceRouterAdvanced(t.Logf, "tailscale0", fake, fake)
+	if err != nil {
+		t.Fatalf("failed to create router: %v", err)
+	}
+	if err := router.Up(); err != nil {
+		t.Fatalf("failed to up router: %v", err)
+	}
+
+	testState := func(t *testing.T, i int) {
+		t.Helper()
+		if err := router.Set(states[i].in); err != nil {
+			t.Fatalf("failed to set router config: %v", err)
+		}
+		got := fake.String()
+		want := strings.TrimSpace(states[i].want)
+		if diff := cmp.Diff(got, want); diff != "" {
+			t.Fatalf("unexpected OS state (-got+want):\n%s", diff)
+		}
+	}
+
+	for i, state := range states {
+		t.Run(state.name, func(t *testing.T) { testState(t, i) })
+	}
+
+	// Cycle through a bunch of states in pseudorandom order, to
+	// verify that we transition cleanly from state to state no matter
+	// the order.
+	for randRun := 0; randRun < 5*len(states); randRun++ {
+		i := rand.Intn(len(states))
+		state := states[i]
+		t.Run(state.name, func(t *testing.T) { testState(t, i) })
+	}
+}
+
+// fakeOS implements netfilterRunner and commandRunner, but captures
+// changes without touching the OS.
+type fakeOS struct {
+	t         *testing.T
+	up        bool
+	ips       []string
+	routes    []string
+	rules     []string
+	netfilter map[string][]string
+}
+
+func NewFakeOS(t *testing.T) *fakeOS {
+	return &fakeOS{
+		t: t,
+		netfilter: map[string][]string{
+			"filter/INPUT":    nil,
+			"filter/OUTPUT":   nil,
+			"filter/FORWARD":  nil,
+			"nat/PREROUTING":  nil,
+			"nat/OUTPUT":      nil,
+			"nat/POSTROUTING": nil,
+		},
+	}
+}
+
+var errExec = errors.New("execution failed")
+
+func (o *fakeOS) String() string {
+	var b strings.Builder
+	if o.up {
+		b.WriteString("up\n")
+	} else {
+		b.WriteString("down\n")
+	}
+
+	for _, ip := range o.ips {
+		fmt.Fprintf(&b, "ip addr add %s\n", ip)
+	}
+
+	for _, route := range o.routes {
+		fmt.Fprintf(&b, "ip route add %s\n", route)
+	}
+
+	for _, rule := range o.rules {
+		fmt.Fprintf(&b, "ip rule add %s\n", rule)
+	}
+
+	var chains []string
+	for chain := range o.netfilter {
+		chains = append(chains, chain)
+	}
+	sort.Strings(chains)
+	for _, chain := range chains {
+		for _, rule := range o.netfilter[chain] {
+			fmt.Fprintf(&b, "%s %s\n", chain, rule)
+		}
+	}
+	return b.String()[:len(b.String())-1]
+}
+
+func (o *fakeOS) Insert(table, chain string, pos int, args ...string) error {
+	k := table + "/" + chain
+	if rules, ok := o.netfilter[k]; ok {
+		if pos > len(rules)+1 {
+			o.t.Errorf("bad position %d in %s", pos, k)
+			return errExec
+		}
+		rules = append(rules, "")
+		copy(rules[pos:], rules[pos-1:])
+		rules[pos-1] = strings.Join(args, " ")
+		o.netfilter[k] = rules
+	} else {
+		o.t.Errorf("unknown table/chain %s", k)
+		return errExec
+	}
+	return nil
+}
+
+func (o *fakeOS) Append(table, chain string, args ...string) error {
+	k := table + "/" + chain
+	return o.Insert(table, chain, len(o.netfilter[k])+1, args...)
+}
+
+func (o *fakeOS) Exists(table, chain string, args ...string) (bool, error) {
+	k := table + "/" + chain
+	if rules, ok := o.netfilter[k]; ok {
+		for _, rule := range rules {
+			if rule == strings.Join(args, " ") {
+				return true, nil
+			}
+		}
+		return false, nil
+	} else {
+		o.t.Errorf("unknown table/chain %s", k)
+		return false, errExec
+	}
+}
+
+func (o *fakeOS) Delete(table, chain string, args ...string) error {
+	k := table + "/" + chain
+	if rules, ok := o.netfilter[k]; ok {
+		for i, rule := range rules {
+			if rule == strings.Join(args, " ") {
+				rules = append(rules[:i], rules[i+1:]...)
+				o.netfilter[k] = rules
+				return nil
+			}
+		}
+		o.t.Errorf("delete of unknown rule %q from %s", strings.Join(args, " "), k)
+		return errExec
+	} else {
+		o.t.Errorf("unknown table/chain %s", k)
+		return errExec
+	}
+}
+
+func (o *fakeOS) ListChains(table string) (ret []string, err error) {
+	for chain := range o.netfilter {
+		pfx := table + "/"
+		if strings.HasPrefix(chain, pfx) {
+			ret = append(ret, chain[len(pfx):])
+		}
+	}
+	return ret, nil
+}
+
+func (o *fakeOS) ClearChain(table, chain string) error {
+	k := table + "/" + chain
+	if _, ok := o.netfilter[k]; ok {
+		o.netfilter[k] = nil
+		return nil
+	} else {
+		o.t.Errorf("unknown table/chain %s", k)
+		return errExec
+	}
+}
+
+func (o *fakeOS) NewChain(table, chain string) error {
+	k := table + "/" + chain
+	if _, ok := o.netfilter[k]; ok {
+		o.t.Errorf("table/chain %s already exists", k)
+		return errExec
+	}
+	o.netfilter[k] = nil
+	return nil
+}
+
+func (o *fakeOS) DeleteChain(table, chain string) error {
+	k := table + "/" + chain
+	if rules, ok := o.netfilter[k]; ok {
+		if len(rules) != 0 {
+			o.t.Errorf("%s is not empty", k)
+			return errExec
+		}
+		delete(o.netfilter, k)
+		return nil
+	} else {
+		o.t.Errorf("%s does not exist", k)
+		return errExec
+	}
+}
+
+func (o *fakeOS) run(args ...string) error {
+	unexpected := func() error {
+		o.t.Errorf("unexpected invocation %q", strings.Join(args, " "))
+		return errors.New("unrecognized invocation")
+	}
+	if args[0] != "ip" {
+		return unexpected()
+	}
+
+	rest := strings.Join(args[3:], " ")
+
+	var l *[]string
+	switch args[1] {
+	case "link":
+		got := strings.Join(args[2:], " ")
+		switch got {
+		case "set dev tailscale0 group 10000 up":
+			o.up = true
+		case "set dev tailscale0 group 0 down":
+			o.up = false
+		default:
+			return unexpected()
+		}
+		return nil
+	case "addr":
+		l = &o.ips
+	case "route":
+		l = &o.routes
+	case "rule":
+		l = &o.rules
+	default:
+		return unexpected()
+	}
+
+	switch args[2] {
+	case "add":
+		for _, el := range *l {
+			if el == rest {
+				o.t.Errorf("can't add %q, already present", rest)
+				return errors.New("already exists")
+			}
+		}
+		*l = append(*l, rest)
+		sort.Strings(*l)
+	case "del":
+		found := false
+		for i, el := range *l {
+			if el == rest {
+				found = true
+				*l = append((*l)[:i], (*l)[i+1:]...)
+				break
+			}
+		}
+		if !found {
+			o.t.Errorf("can't delete %q, not present", rest)
+			return errors.New("not present")
+		}
+	default:
+		return unexpected()
+	}
+
+	return nil
+}
+
+func (o *fakeOS) output(args ...string) ([]byte, error) {
+	want := "ip rule list priority 10000"
+	got := strings.Join(args, " ")
+	if got != want {
+		o.t.Errorf("unexpected command that wants output: %v", got)
+		return nil, errExec
+	}
+
+	var ret []string
+	for _, rule := range o.rules {
+		if strings.Contains(rule, "10000") {
+			ret = append(ret, rule)
+		}
+	}
+	return []byte(strings.Join(ret, "\n")), nil
+}