cmd/containerboot: allow disabling secret storage in k8s.

In some configurations, user explicitly do not want to store
tailscale state in k8s secrets, because doing that leads to
some annoying permission issues with sidecar containers.
With this change, TS_KUBE_SECRET="" and TS_STATE_DIR=/foo
will force storage to file when running in kubernetes.

Fixes #6704.

Signed-off-by: David Anderson <danderson@tailscale.com>
(cherry picked from commit af3127711a)

VERSION.txt

@@ -1 +1 @@
-1.33.0
+1.34.1

cmd/containerboot/kube.go

@@ -21,8 +21,84 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"tailscale.com/util/multierr"
 )
 
+// checkSecretPermissions checks the secret access permissions of the current
+// pod. It returns an error if the basic permissions tailscale needs are
+// missing, and reports whether the patch permission is additionally present.
+//
+// Errors encountered during the access checking process are logged, but ignored
+// so that the pod tries to fail alive if the permissions exist and there's just
+// something wrong with SelfSubjectAccessReviews. There shouldn't be, pods
+// should always be able to use SSARs to assess their own permissions, but since
+// we didn't use to check permissions this way we'll be cautious in case some
+// old version of k8s deviates from the current behavior.
+func checkSecretPermissions(ctx context.Context, secretName string) (canPatch bool, err error) {
+	var errs []error
+	for _, verb := range []string{"get", "update"} {
+		ok, err := checkPermission(ctx, verb, secretName)
+		if err != nil {
+			log.Printf("error checking %s permission on secret %s: %v", verb, secretName, err)
+		} else if !ok {
+			errs = append(errs, fmt.Errorf("missing %s permission on secret %q", verb, secretName))
+		}
+	}
+	if len(errs) > 0 {
+		return false, multierr.New(errs...)
+	}
+	ok, err := checkPermission(ctx, "patch", secretName)
+	if err != nil {
+		log.Printf("error checking patch permission on secret %s: %v", secretName, err)
+		return false, nil
+	}
+	return ok, nil
+}
+
+// checkPermission reports whether the current pod has permission to use the
+// given verb (e.g. get, update, patch) on secretName.
+func checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
+	sar := map[string]any{
+		"apiVersion": "authorization.k8s.io/v1",
+		"kind":       "SelfSubjectAccessReview",
+		"spec": map[string]any{
+			"resourceAttributes": map[string]any{
+				"namespace": kubeNamespace,
+				"verb":      verb,
+				"resource":  "secrets",
+				"name":      secretName,
+			},
+		},
+	}
+	bs, err := json.Marshal(sar)
+	if err != nil {
+		return false, err
+	}
+	req, err := http.NewRequest("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", bytes.NewReader(bs))
+	if err != nil {
+		return false, err
+	}
+	resp, err := doKubeRequest(ctx, req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+	bs, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return false, err
+	}
+	var res struct {
+		Status struct {
+			Allowed bool `json:"allowed"`
+		} `json:"status"`
+	}
+	if err := json.Unmarshal(bs, &res); err != nil {
+		return false, err
+	}
+	return res.Status.Allowed, nil
+}
+
 // findKeyInKubeSecret inspects the kube secret secretName for a data
 // field called "authkey", and returns its value if present.
 func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
@@ -84,7 +160,7 @@ func storeDeviceID(ctx context.Context, secretName, deviceID string) error {
 	}
 
 	m := map[string]map[string]string{
-		"stringData": map[string]string{
+		"stringData": {
 			"device_id": deviceID,
 		},
 	}
@@ -193,8 +269,8 @@ func doKubeRequest(ctx context.Context, r *http.Request) (*http.Response, error)
 	if err != nil {
 		return nil, err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return resp, fmt.Errorf("got non-200 status code %d", resp.StatusCode)
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+		return resp, fmt.Errorf("got non-200/201 status code %d", resp.StatusCode)
 	}
 	return resp, nil
 }

cmd/containerboot/main.go

@@ -4,13 +4,13 @@
 
 //go:build linux
 
-// The containerboot binary is a wrapper for starting tailscaled in a
-// container. It handles reading the desired mode of operation out of
-// environment variables, bringing up and authenticating Tailscale,
-// and any other kubernetes-specific side jobs.
+// The containerboot binary is a wrapper for starting tailscaled in a container.
+// It handles reading the desired mode of operation out of environment
+// variables, bringing up and authenticating Tailscale, and any other
+// kubernetes-specific side jobs.
 //
-// As with most container things, configuration is passed through
-// environment variables. All configuration is optional.
+// As with most container things, configuration is passed through environment
+// variables. All configuration is optional.
 //
 //   - TS_AUTH_KEY: the authkey to use for login.
 //   - TS_ROUTES: subnet routes to advertise.
@@ -37,9 +37,13 @@
 //     compatibility), forcibly log in every time the
 //     container starts.
 //
-// When running on Kubernetes, TS_KUBE_SECRET takes precedence over
-// TS_STATE_DIR. Additionally, if TS_AUTH_KEY is not provided and the
-// TS_KUBE_SECRET contains an "authkey" field, that key is used.
+// When running on Kubernetes, containerboot defaults to storing state in the
+// "tailscale" kube secret. To store state on local disk instead, set
+// TS_KUBE_SECRET="" and TS_STATE_DIR=/path/to/storage/dir. The state dir should
+// be persistent storage.
+//
+// Additionally, if TS_AUTH_KEY is not set and the TS_KUBE_SECRET contains an
+// "authkey" field, that key is used as the tailscale authkey.
 package main
 
 import (
@@ -116,30 +120,53 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.AuthKey == "" {
-		key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret)
+	if cfg.InKubernetes && cfg.KubeSecret != "" {
+		canPatch, err := checkSecretPermissions(ctx, cfg.KubeSecret)
 		if err != nil {
-			log.Fatalf("Getting authkey from kube secret: %v", err)
+			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
 		}
-		if key != "" {
-			log.Print("Using authkey found in kube secret")
-			cfg.AuthKey = key
-		} else {
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		cfg.KubernetesCanPatch = canPatch
+
+		if cfg.AuthKey == "" {
+			key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret)
+			if err != nil {
+				log.Fatalf("Getting authkey from kube secret: %v", err)
+			}
+			if key != "" {
+				// This behavior of pulling authkeys from kube secrets was added
+				// at the same time as the patch permission, so we can enforce
+				// that we must be able to patch out the authkey after
+				// authenticating if you want to use this feature. This avoids
+				// us having to deal with the case where we might leave behind
+				// an unnecessary reusable authkey in a secret, like a rake in
+				// the grass.
+				if !cfg.KubernetesCanPatch {
+					log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+				}
+				log.Print("Using authkey found in kube secret")
+				cfg.AuthKey = key
+			} else {
+				log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+			}
 		}
 	}
 
-	st, daemonPid, err := startAndAuthTailscaled(ctx, cfg)
+	client, daemonPid, err := startTailscaled(ctx, cfg)
 	if err != nil {
 		log.Fatalf("failed to bring up tailscale: %v", err)
 	}
 
+	st, err := authTailscaled(ctx, client, cfg)
+	if err != nil {
+		log.Fatalf("failed to auth tailscale: %v", err)
+	}
+
 	if cfg.ProxyTo != "" {
 		if err := installIPTablesRule(ctx, cfg.ProxyTo, st.TailscaleIPs); err != nil {
 			log.Fatalf("installing proxy rules: %v", err)
 		}
 	}
-	if cfg.InKubernetes && cfg.KubeSecret != "" {
+	if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" {
 		if err := storeDeviceID(ctx, cfg.KubeSecret, string(st.Self.ID)); err != nil {
 			log.Fatalf("storing device ID in kube secret: %v", err)
 		}
@@ -173,10 +200,7 @@ func main() {
 	}
 }
 
-// startAndAuthTailscaled starts the tailscale daemon and attempts to
-// auth it, according to the settings in cfg. If successful, returns
-// tailscaled's Status and pid.
-func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Status, int, error) {
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, int, error) {
 	args := tailscaledArgs(cfg)
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, unix.SIGTERM, unix.SIGINT)
@@ -198,8 +222,7 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 		cmd.Process.Signal(unix.SIGTERM)
 	}()
 
-	// Wait for the socket file to appear, otherwise 'tailscale up'
-	// can fail.
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
 	log.Printf("Waiting for tailscaled socket")
 	for {
 		if ctx.Err() != nil {
@@ -215,39 +238,46 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 		break
 	}
 
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process.Pid, nil
+}
+
+// startAndAuthTailscaled starts the tailscale daemon and attempts to
+// auth it, according to the settings in cfg. If successful, returns
+// tailscaled's Status and pid.
+func authTailscaled(ctx context.Context, client *tailscale.LocalClient, cfg *settings) (*ipnstate.Status, error) {
 	didLogin := false
 	if !cfg.AuthOnce {
 		if err := tailscaleUp(ctx, cfg); err != nil {
-			return nil, 0, fmt.Errorf("couldn't log in: %v", err)
+			return nil, fmt.Errorf("couldn't log in: %v", err)
 		}
 		didLogin = true
 	}
 
-	tsClient := tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
 	// Poll for daemon state until it goes to either Running or
 	// NeedsLogin. The latter only happens if cfg.AuthOnce is true,
 	// because in that case we only try to auth when it's necessary to
 	// reach the running state.
 	for {
 		if ctx.Err() != nil {
-			return nil, 0, ctx.Err()
+			return nil, ctx.Err()
 		}
 
 		loopCtx, cancel := context.WithTimeout(ctx, time.Second)
-		st, err := tsClient.Status(loopCtx)
+		st, err := client.Status(loopCtx)
 		cancel()
 		if err != nil {
-			return nil, 0, fmt.Errorf("Getting tailscaled state: %w", err)
+			return nil, fmt.Errorf("Getting tailscaled state: %w", err)
 		}
 
 		switch st.BackendState {
 		case "Running":
 			if len(st.TailscaleIPs) > 0 {
-				return st, cmd.Process.Pid, nil
+				return st, nil
 			}
 			log.Printf("No Tailscale IPs assigned yet")
 		case "NeedsLogin":
@@ -256,7 +286,7 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 				// LocalAPI, so we still have to shell out to the
 				// tailscale CLI for this bit.
 				if err := tailscaleUp(ctx, cfg); err != nil {
-					return nil, 0, fmt.Errorf("couldn't log in: %v", err)
+					return nil, fmt.Errorf("couldn't log in: %v", err)
 				}
 				didLogin = true
 			}
@@ -275,7 +305,7 @@ func tailscaledArgs(cfg *settings) []string {
 	case cfg.InKubernetes && cfg.KubeSecret != "":
 		args = append(args, "--state=kube:"+cfg.KubeSecret, "--statedir=/tmp")
 	case cfg.StateDir != "":
-		args = append(args, "--state="+cfg.StateDir)
+		args = append(args, "--statedir="+cfg.StateDir)
 	default:
 		args = append(args, "--state=mem:", "--statedir=/tmp")
 	}
@@ -433,27 +463,28 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Addr)
 
 // settings is all the configuration for containerboot.
 type settings struct {
-	AuthKey         string
-	Routes          string
-	ProxyTo         string
-	DaemonExtraArgs string
-	ExtraArgs       string
-	InKubernetes    bool
-	UserspaceMode   bool
-	StateDir        string
-	AcceptDNS       bool
-	KubeSecret      string
-	SOCKSProxyAddr  string
-	HTTPProxyAddr   string
-	Socket          string
-	AuthOnce        bool
-	Root            string
+	AuthKey            string
+	Routes             string
+	ProxyTo            string
+	DaemonExtraArgs    string
+	ExtraArgs          string
+	InKubernetes       bool
+	UserspaceMode      bool
+	StateDir           string
+	AcceptDNS          bool
+	KubeSecret         string
+	SOCKSProxyAddr     string
+	HTTPProxyAddr      string
+	Socket             string
+	AuthOnce           bool
+	Root               string
+	KubernetesCanPatch bool
 }
 
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {
-	if v := os.Getenv(name); v != "" {
+	if v, ok := os.LookupEnv(name); ok {
 		return v
 	}
 	return defVal

cmd/containerboot/main_test.go

@@ -33,6 +33,7 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
 )
 
 func TestContainerBoot(t *testing.T) {
@@ -97,31 +98,45 @@ func TestContainerBoot(t *testing.T) {
 	// step. Right now all of containerboot's modes either converge
 	// with no further interaction needed, or with one extra step
 	// only.
-	tests := []struct {
-		Name           string
-		Env            map[string]string
-		KubeSecret     map[string]string
-		WantArgs1      []string        // Wait for containerboot to run these commands...
-		Status1        ipnstate.Status // ... then report this status in LocalAPI.
-		WantArgs2      []string        // If non-nil, wait for containerboot to run these additional commands...
-		Status2        ipnstate.Status // ... then report this status in LocalAPI.
+	type phase struct {
+		// Make LocalAPI report this status, then wait for the Wants below to be
+		// satisfied. A zero Status is a valid state for a just-started
+		// tailscaled.
+		Status ipnstate.Status
+
+		// WantCmds is the commands that containerboot should run in this phase.
+		WantCmds []string
+		// WantKubeSecret is the secret keys/values that should exist in the
+		// kube secret.
 		WantKubeSecret map[string]string
-		WantFiles      map[string]string
+		// WantFiles files that should exist in the container and their
+		// contents.
+		WantFiles map[string]string
+	}
+	tests := []struct {
+		Name          string
+		Env           map[string]string
+		KubeSecret    map[string]string
+		KubeDenyPatch bool
+		Phases        []phase
 	}{
 		{
 			// Out of the box default: runs in userspace mode, ephemeral storage, interactive login.
 			Name: "no_args",
 			Env:  nil,
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
-			},
-			// The tailscale up call blocks until auth is complete, so
-			// by the time it returns the next converged state is
-			// Running.
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -130,13 +145,19 @@ func TestContainerBoot(t *testing.T) {
 			Env: map[string]string{
 				"TS_AUTH_KEY": "tskey-key",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -145,13 +166,19 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_STATE_DIR": filepath.Join(d, "tmp"),
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -160,17 +187,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY": "tskey-key",
 				"TS_ROUTES":   "1.2.3.0/24,10.20.30.0/24",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-			},
-			WantFiles: map[string]string{
-				"proc/sys/net/ipv4/ip_forward":          "0",
-				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "0",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
 			},
 		},
 		{
@@ -180,17 +213,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "1.2.3.0/24,10.20.30.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-			},
-			WantFiles: map[string]string{
-				"proc/sys/net/ipv4/ip_forward":          "1",
-				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
 			},
 		},
 		{
@@ -200,17 +239,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1::/64",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-			},
-			WantFiles: map[string]string{
-				"proc/sys/net/ipv4/ip_forward":          "0",
-				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "0",
+						"proc/sys/net/ipv6/conf/all/forwarding": "1",
+					},
+				},
 			},
 		},
 		{
@@ -220,17 +265,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1.2.3.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-			},
-			WantFiles: map[string]string{
-				"proc/sys/net/ipv4/ip_forward":          "1",
-				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "1",
+					},
+				},
 			},
 		},
 		{
@@ -240,20 +291,22 @@ func TestContainerBoot(t *testing.T) {
 				"TS_DEST_IP":   "1.2.3.4",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-			},
-			WantArgs2: []string{
-				"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
-			},
-			Status2: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+					WantCmds: []string{
+						"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
+					},
+				},
 			},
 		},
 		{
@@ -262,18 +315,26 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_AUTH_ONCE": "true",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "NeedsLogin",
-			},
-			WantArgs2: []string{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status2: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "NeedsLogin",
+					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -285,20 +346,83 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-				Self: &ipnstate.PeerStatus{
-					ID: tailcfg.StableNodeID("myID"),
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":   "tskey-key",
+						"device_id": "myID",
+					},
 				},
 			},
-			WantKubeSecret: map[string]string{
-				"authkey":   "tskey-key",
-				"device_id": "myID",
+		},
+		{
+			Name: "kube_disk_storage",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				// Explicitly set to an empty value, to override the default of "tailscale".
+				"TS_KUBE_SECRET": "",
+				"TS_STATE_DIR":   filepath.Join(d, "tmp"),
+				"TS_AUTH_KEY":    "tskey-key",
+			},
+			KubeSecret: map[string]string{},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{},
+				},
+				{
+					Notify:         runningNotify,
+					WantKubeSecret: map[string]string{},
+				},
+			},
+		},
+		{
+			Name: "kube_storage_no_patch",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_AUTH_KEY":                   "tskey-key",
+			},
+			KubeSecret:    map[string]string{},
+			KubeDenyPatch: true,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{},
+				},
 			},
 		},
 		{
@@ -312,24 +436,38 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "NeedsLogin",
-			},
-			WantArgs2: []string{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-			},
-			Status2: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
-				Self: &ipnstate.PeerStatus{
-					ID: tailcfg.StableNodeID("myID"),
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "NeedsLogin",
+					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{
+						"device_id": "myID",
+					},
 				},
-			},
-			WantKubeSecret: map[string]string{
-				"device_id": "myID",
 			},
 		},
 		{
@@ -338,16 +476,22 @@ func TestContainerBoot(t *testing.T) {
 				"TS_SOCKS5_SERVER":              "localhost:1080",
 				"TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
-			},
-			// The tailscale up call blocks until auth is complete, so
-			// by the time it returns the next converged state is
-			// Running.
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					},
+				},
+				{
+					// The tailscale up call blocks until auth is complete, so
+					// by the time it returns the next converged state is
+					// Running.
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -355,13 +499,19 @@ func TestContainerBoot(t *testing.T) {
 			Env: map[string]string{
 				"TS_ACCEPT_DNS": "true",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -370,13 +520,18 @@ func TestContainerBoot(t *testing.T) {
 				"TS_EXTRA_ARGS":            "--widget=rotated",
 				"TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets",
 			},
-			WantArgs1: []string{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
-			},
-			Status1: ipnstate.Status{
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
+					},
+				}, {
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 	}
@@ -392,6 +547,7 @@ func TestContainerBoot(t *testing.T) {
 			for k, v := range test.KubeSecret {
 				kube.SetSecret(k, v)
 			}
+			kube.SetPatching(!test.KubeDenyPatch)
 
 			cmd := exec.Command(boot)
 			cmd.Env = []string{
@@ -419,35 +575,45 @@ func TestContainerBoot(t *testing.T) {
 				cmd.Process.Wait()
 			}()
 
-			waitArgs(t, 2*time.Second, d, argFile, strings.Join(test.WantArgs1, "\n"))
-			lapi.SetStatus(test.Status1)
-			if test.WantArgs2 != nil {
-				waitArgs(t, 2*time.Second, d, argFile, strings.Join(append(test.WantArgs1, test.WantArgs2...), "\n"))
-				lapi.SetStatus(test.Status2)
+			var wantCmds []string
+			for _, p := range test.Phases {
+				lapi.SetStatus(p.Status)
+				wantCmds = append(wantCmds, p.WantCmds...)
+				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
+				err := tstest.WaitFor(2*time.Second, func() error {
+					if p.WantKubeSecret != nil {
+						got := kube.Secret()
+						if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" {
+							return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff)
+						}
+					} else {
+						got := kube.Secret()
+						if len(got) > 0 {
+							return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got)
+						}
+					}
+					return nil
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = tstest.WaitFor(2*time.Second, func() error {
+					for path, want := range p.WantFiles {
+						gotBs, err := os.ReadFile(filepath.Join(d, path))
+						if err != nil {
+							return fmt.Errorf("reading wanted file %q: %v", path, err)
+						}
+						if got := strings.TrimSpace(string(gotBs)); got != want {
+							return fmt.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
+						}
+					}
+					return nil
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
-
-			if test.WantKubeSecret != nil {
-				got := kube.Secret()
-				if diff := cmp.Diff(got, test.WantKubeSecret); diff != "" {
-					t.Fatalf("unexpected kube secret data (-got+want):\n%s", diff)
-				}
-			} else {
-				got := kube.Secret()
-				if len(got) != 0 {
-					t.Fatalf("kube secret unexpectedly not empty, got %#v", got)
-				}
-			}
-
-			for path, want := range test.WantFiles {
-				gotBs, err := os.ReadFile(filepath.Join(d, path))
-				if err != nil {
-					t.Fatalf("reading wanted file %q: %v", path, err)
-				}
-				if got := strings.TrimSpace(string(gotBs)); got != want {
-					t.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
-				}
-			}
 		})
 	}
 }
@@ -612,7 +778,8 @@ type kubeServer struct {
 	srv *httptest.Server
 
 	sync.Mutex
-	secret map[string]string
+	secret   map[string]string
+	canPatch bool
 }
 
 func (k *kubeServer) Secret() map[string]string {
@@ -631,6 +798,12 @@ func (k *kubeServer) SetSecret(key, val string) {
 	k.secret[key] = val
 }
 
+func (k *kubeServer) SetPatching(canPatch bool) {
+	k.Lock()
+	defer k.Unlock()
+	k.canPatch = canPatch
+}
+
 func (k *kubeServer) Reset() {
 	k.Lock()
 	defer k.Unlock()
@@ -674,10 +847,39 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Authorization") != "Bearer bearer_token" {
 		panic("client didn't provide bearer token in request")
 	}
-	if r.URL.Path != "/api/v1/namespaces/default/secrets/tailscale" {
+	switch r.URL.Path {
+	case "/api/v1/namespaces/default/secrets/tailscale":
+		k.serveSecret(w, r)
+	case "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews":
+		k.serveSSAR(w, r)
+	default:
 		panic(fmt.Sprintf("unhandled fake kube api path %q", r.URL.Path))
 	}
+}
 
+func (k *kubeServer) serveSSAR(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Spec struct {
+			ResourceAttributes struct {
+				Verb string `json:"verb"`
+			} `json:"resourceAttributes"`
+		} `json:"spec"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		panic(fmt.Sprintf("decoding SSAR request: %v", err))
+	}
+	ok := true
+	if req.Spec.ResourceAttributes.Verb == "patch" {
+		k.Lock()
+		defer k.Unlock()
+		ok = k.canPatch
+	}
+	// Just say yes to all SARs, we don't enforce RBAC.
+	w.Header().Set("Content-Type", "application/json")
+	fmt.Fprintf(w, `{"status":{"allowed":%v}}`, ok)
+}
+
+func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 	bs, err := io.ReadAll(r.Body)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError)
@@ -688,7 +890,7 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case "GET":
 		w.Header().Set("Content-Type", "application/json")
 		ret := map[string]map[string]string{
-			"data": map[string]string{},
+			"data": {},
 		}
 		k.Lock()
 		defer k.Unlock()
@@ -703,6 +905,11 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			panic("encode failed")
 		}
 	case "PATCH":
+		k.Lock()
+		defer k.Unlock()
+		if !k.canPatch {
+			panic("containerboot tried to patch despite not being allowed")
+		}
 		switch r.Header.Get("Content-Type") {
 		case "application/json-patch+json":
 			req := []struct {
@@ -712,8 +919,6 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if err := json.Unmarshal(bs, &req); err != nil {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
-			k.Lock()
-			defer k.Unlock()
 			for _, op := range req {
 				if op.Op != "remove" {
 					panic(fmt.Sprintf("unsupported json-patch op %q", op.Op))
@@ -730,8 +935,6 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if err := json.Unmarshal(bs, &req); err != nil {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
-			k.Lock()
-			defer k.Unlock()
 			for key, val := range req.Data {
 				k.secret[key] = val
 			}

cmd/tailscale/cli/ssh.go

@@ -21,6 +21,7 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/paths"
 	"tailscale.com/version"
 )
 
@@ -37,7 +38,7 @@ most users running the Tailscale SSH server will prefer to just use the normal
 
 The 'tailscale ssh' wrapper adds a few things:
 
-* It resolves the destination server name in its arugments using MagicDNS,
+* It resolves the destination server name in its arguments using MagicDNS,
   even if --accept-dns=false.
 * It works in userspace-networking mode, by supplying a ProxyCommand to the
   system 'ssh' command that connects via a pipe through tailscaled.
@@ -110,10 +111,15 @@ func runSSH(ctx context.Context, args []string) error {
 	// So don't use it for now. MagicDNS is usually working on macOS anyway
 	// and they're not in userspace mode, so 'nc' isn't very useful.
 	if runtime.GOOS != "darwin" {
+		socketArg := ""
+		if rootArgs.socket != "" && rootArgs.socket != paths.DefaultTailscaledSocket() {
+			socketArg = fmt.Sprintf("--socket=%q", rootArgs.socket)
+		}
+
 		argv = append(argv,
-			"-o", fmt.Sprintf("ProxyCommand %q --socket=%q nc %%h %%p",
+			"-o", fmt.Sprintf("ProxyCommand %q %s nc %%h %%p",
 				tailscaleBin,
-				rootArgs.socket,
+				socketArg,
 			))
 	}
 

ipn/ipnlocal/peerapi.go

@@ -607,6 +607,7 @@ func peerAPIRequestShouldGetSecurityHeaders(r *http.Request) bool {
 
 func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if err := h.validatePeerAPIRequest(r); err != nil {
+		metricInvalidRequests.Add(1)
 		h.logf("invalid request from %v: %v", h.remoteAddr, err)
 		http.Error(w, "invalid peerapi request", http.StatusForbidden)
 		return
@@ -617,10 +618,12 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Content-Type-Options", "nosniff")
 	}
 	if strings.HasPrefix(r.URL.Path, "/v0/put/") {
+		metricPutCalls.Add(1)
 		h.handlePeerPut(w, r)
 		return
 	}
 	if strings.HasPrefix(r.URL.Path, "/dns-query") {
+		metricDNSCalls.Add(1)
 		h.handleDNSQuery(w, r)
 		return
 	}
@@ -641,12 +644,14 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handleServeDNSFwd(w, r)
 		return
 	case "/v0/wol":
+		metricWakeOnLANCalls.Add(1)
 		h.handleWakeOnLAN(w, r)
 		return
 	case "/v0/interfaces":
 		h.handleServeInterfaces(w, r)
 		return
 	case "/v0/ingress":
+		metricIngressCalls.Add(1)
 		h.handleServeIngress(w, r)
 		return
 	}
@@ -1370,3 +1375,13 @@ func (fl *fakePeerAPIListener) Accept() (net.Conn, error) {
 }
 
 func (fl *fakePeerAPIListener) Addr() net.Addr { return fl.addr }
+
+var (
+	metricInvalidRequests = clientmetric.NewCounter("peerapi_invalid_requests")
+
+	// Non-debug PeerAPI endpoints.
+	metricPutCalls       = clientmetric.NewCounter("peerapi_put")
+	metricDNSCalls       = clientmetric.NewCounter("peerapi_dns")
+	metricWakeOnLANCalls = clientmetric.NewCounter("peerapi_wol")
+	metricIngressCalls   = clientmetric.NewCounter("peerapi_ingress")
+)

ipn/localapi/localapi.go

@@ -146,6 +146,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if r.Referer() != "" || r.Header.Get("Origin") != "" || !validHost(r.Host) {
+		metricInvalidRequests.Add(1)
 		http.Error(w, "invalid localapi request", http.StatusForbidden)
 		return
 	}
@@ -156,10 +157,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if h.RequiredPassword != "" {
 		_, pass, ok := r.BasicAuth()
 		if !ok {
+			metricInvalidRequests.Add(1)
 			http.Error(w, "auth required", http.StatusUnauthorized)
 			return
 		}
 		if pass != h.RequiredPassword {
+			metricInvalidRequests.Add(1)
 			http.Error(w, "bad password", http.StatusForbidden)
 			return
 		}
@@ -900,6 +903,8 @@ func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
 //
 //   - PUT /localapi/v0/file-put/:stableID/:escaped-filename
 func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
+	metricFilePutCalls.Add(1)
+
 	if !h.PermitWrite {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
@@ -1437,3 +1442,10 @@ func defBool(a string, def bool) bool {
 	}
 	return v
 }
+
+var (
+	metricInvalidRequests = clientmetric.NewCounter("localapi_invalid_requests")
+
+	// User-visible LocalAPI endpoints.
+	metricFilePutCalls = clientmetric.NewCounter("localapi_file_put")
+)

wgengine/magicsock/magicsock.go

@@ -3008,13 +3008,14 @@ func (c *Conn) ParseEndpoint(nodeKeyStr string) (conn.Endpoint, error) {
 // RebindingUDPConn is a UDP socket that can be re-bound.
 // Unix has no notion of re-binding a socket, so we swap it out for a new one.
 type RebindingUDPConn struct {
-	// pconnAtomic is the same as pconn, but doesn't require acquiring mu. It's
-	// used for reads/writes and only upon failure do the reads/writes then
-	// check pconn (after acquiring mu) to see if there's been a rebind
-	// meanwhile.
+	// pconnAtomic is a pointer to the value stored in pconn, but doesn't
+	// require acquiring mu. It's used for reads/writes and only upon failure
+	// do the reads/writes then check pconn (after acquiring mu) to see if
+	// there's been a rebind meanwhile.
 	// pconn isn't really needed, but makes some of the code simpler
-	// to keep it in a type safe form.
-	pconnAtomic syncs.AtomicValue[nettype.PacketConn]
+	// to keep it distinct.
+	// Neither is expected to be nil, sockets are bound on creation.
+	pconnAtomic atomic.Pointer[nettype.PacketConn]
 
 	mu    sync.Mutex // held while changing pconn (and pconnAtomic)
 	pconn nettype.PacketConn
@@ -3023,7 +3024,7 @@ type RebindingUDPConn struct {
 
 func (c *RebindingUDPConn) setConnLocked(p nettype.PacketConn) {
 	c.pconn = p
-	c.pconnAtomic.Store(p)
+	c.pconnAtomic.Store(&p)
 	c.port = uint16(c.localAddrLocked().Port)
 }
 
@@ -3038,7 +3039,7 @@ func (c *RebindingUDPConn) currentConn() nettype.PacketConn {
 // It returns the number of bytes copied and the source address.
 func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
 		n, addr, err := pconn.ReadFrom(b)
 		if err != nil && pconn != c.currentConn() {
 			continue
@@ -3056,7 +3057,7 @@ func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 // when c's underlying connection is a net.UDPConn.
 func (c *RebindingUDPConn) ReadFromNetaddr(b []byte) (n int, ipp netip.AddrPort, err error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
 
 		// Optimization: Treat *net.UDPConn specially.
 		// This lets us avoid allocations by calling ReadFromUDPAddrPort.
@@ -3122,13 +3123,10 @@ func (c *RebindingUDPConn) closeLocked() error {
 
 func (c *RebindingUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
-
+		pconn := *c.pconnAtomic.Load()
 		n, err := pconn.WriteTo(b, addr)
-		if err != nil {
-			if pconn != c.currentConn() {
-				continue
-			}
+		if err != nil && pconn != c.currentConn() {
+			continue
 		}
 		return n, err
 	}
@@ -3136,13 +3134,10 @@ func (c *RebindingUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
 
 func (c *RebindingUDPConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
-
+		pconn := *c.pconnAtomic.Load()
 		n, err := pconn.WriteToUDPAddrPort(b, addr)
-		if err != nil {
-			if pconn != c.currentConn() {
-				continue
-			}
+		if err != nil && pconn != c.currentConn() {
+			continue
 		}
 		return n, err
 	}
@@ -3334,7 +3329,7 @@ type endpoint struct {
 	publicKey    key.NodePublic // peer public key (for WireGuard + DERP)
 	publicKeyHex string         // cached output of publicKey.UntypedHexString
 	fakeWGAddr   netip.AddrPort // the UDP address we tell wireguard-go we're using
-	nodeAddr     netip.Addr     // the node's first tailscale address (only used for logging)
+	nodeAddr     netip.Addr     // the node's first tailscale address; used for logging & wireguard rate-limiting (Issue 6686)
 
 	// mu protects all following fields.
 	mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu
@@ -3521,7 +3516,7 @@ func (de *endpoint) ClearSrc()           {}
 func (de *endpoint) SrcToString() string { panic("unused") } // unused by wireguard-go
 func (de *endpoint) SrcIP() netip.Addr   { panic("unused") } // unused by wireguard-go
 func (de *endpoint) DstToString() string { return de.publicKeyHex }
-func (de *endpoint) DstIP() netip.Addr   { panic("unused") }
+func (de *endpoint) DstIP() netip.Addr   { return de.nodeAddr } // see tailscale/tailscale#6686
 func (de *endpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr) }
 
 // addrForSendLocked returns the address(es) that should be used for

wgengine/magicsock/magicsock_test.go

@@ -1803,3 +1803,16 @@ func TestDiscoMagicMatches(t *testing.T) {
 		t.Errorf("last 2 bytes of disco magic don't match, got %v want %v", discoMagic2, m2)
 	}
 }
+
+func TestRebindingUDPConn(t *testing.T) {
+	// Test that RebindingUDPConn can be re-bound to different connection
+	// types.
+	c := RebindingUDPConn{}
+	realConn, err := net.ListenPacket("udp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer realConn.Close()
+	c.setConnLocked(realConn.(nettype.PacketConn))
+	c.setConnLocked(newBlockForeverConn())
+}

wgengine/router/router_linux_test.go

@@ -11,8 +11,10 @@ import (
 	"net/netip"
 	"os"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 
@@ -341,7 +343,7 @@ ip route add throw 192.168.0.0/24 table 52` + basic,
 			t.Fatalf("failed to set router config: %v", err)
 		}
 		got := fake.String()
-		want := strings.TrimSpace(states[i].want)
+		want := adjustFwmask(t, strings.TrimSpace(states[i].want))
 		if diff := cmp.Diff(got, want); diff != "" {
 			t.Fatalf("unexpected OS state (-got+want):\n%s", diff)
 		}
@@ -922,3 +924,23 @@ func TestCIDRDiff(t *testing.T) {
 		}
 	}
 }
+
+var (
+	fwmaskSupported     bool
+	fwmaskSupportedOnce sync.Once
+	fwmaskAdjustRe      = regexp.MustCompile(`(?m)(fwmark 0x[0-9a-f]+)/0x[0-9a-f]+`)
+)
+
+// adjustFwmask removes the "/0xmask" string from fwmask stanzas if the
+// installed 'ip' binary does not support that format.
+func adjustFwmask(t *testing.T, s string) string {
+	t.Helper()
+	fwmaskSupportedOnce.Do(func() {
+		fwmaskSupported, _ = ipCmdSupportsFwmask()
+	})
+	if fwmaskSupported {
+		return s
+	}
+
+	return fwmaskAdjustRe.ReplaceAllString(s, "$1")
+}