cmd/containerboot: allow disabling secret storage in k8s.

In some configurations, user explicitly do not want to store
tailscale state in k8s secrets, because doing that leads to
some annoying permission issues with sidecar containers.
With this change, TS_KUBE_SECRET="" and TS_STATE_DIR=/foo
will force storage to file when running in kubernetes.

Fixes #6704.

Signed-off-by: David Anderson <danderson@tailscale.com>
(cherry picked from commit af3127711a)

VERSION.txt

@@ -1 +1 @@
-1.33.0
+1.34.1

cmd/containerboot/kube.go

@@ -21,8 +21,84 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"tailscale.com/util/multierr"
 )
 
+// checkSecretPermissions checks the secret access permissions of the current
+// pod. It returns an error if the basic permissions tailscale needs are
+// missing, and reports whether the patch permission is additionally present.
+//
+// Errors encountered during the access checking process are logged, but ignored
+// so that the pod tries to fail alive if the permissions exist and there's just
+// something wrong with SelfSubjectAccessReviews. There shouldn't be, pods
+// should always be able to use SSARs to assess their own permissions, but since
+// we didn't use to check permissions this way we'll be cautious in case some
+// old version of k8s deviates from the current behavior.
+func checkSecretPermissions(ctx context.Context, secretName string) (canPatch bool, err error) {
+	var errs []error
+	for _, verb := range []string{"get", "update"} {
+		ok, err := checkPermission(ctx, verb, secretName)
+		if err != nil {
+			log.Printf("error checking %s permission on secret %s: %v", verb, secretName, err)
+		} else if !ok {
+			errs = append(errs, fmt.Errorf("missing %s permission on secret %q", verb, secretName))
+		}
+	}
+	if len(errs) > 0 {
+		return false, multierr.New(errs...)
+	}
+	ok, err := checkPermission(ctx, "patch", secretName)
+	if err != nil {
+		log.Printf("error checking patch permission on secret %s: %v", secretName, err)
+		return false, nil
+	}
+	return ok, nil
+}
+
+// checkPermission reports whether the current pod has permission to use the
+// given verb (e.g. get, update, patch) on secretName.
+func checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
+	sar := map[string]any{
+		"apiVersion": "authorization.k8s.io/v1",
+		"kind":       "SelfSubjectAccessReview",
+		"spec": map[string]any{
+			"resourceAttributes": map[string]any{
+				"namespace": kubeNamespace,
+				"verb":      verb,
+				"resource":  "secrets",
+				"name":      secretName,
+			},
+		},
+	}
+	bs, err := json.Marshal(sar)
+	if err != nil {
+		return false, err
+	}
+	req, err := http.NewRequest("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", bytes.NewReader(bs))
+	if err != nil {
+		return false, err
+	}
+	resp, err := doKubeRequest(ctx, req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+	bs, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return false, err
+	}
+	var res struct {
+		Status struct {
+			Allowed bool `json:"allowed"`
+		} `json:"status"`
+	}
+	if err := json.Unmarshal(bs, &res); err != nil {
+		return false, err
+	}
+	return res.Status.Allowed, nil
+}
+
 // findKeyInKubeSecret inspects the kube secret secretName for a data
 // field called "authkey", and returns its value if present.
 func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
@@ -84,7 +160,7 @@ func storeDeviceID(ctx context.Context, secretName, deviceID string) error {
 	}
 
 	m := map[string]map[string]string{
-		"stringData": map[string]string{
+		"stringData": {
 			"device_id": deviceID,
 		},
 	}
@@ -193,8 +269,8 @@ func doKubeRequest(ctx context.Context, r *http.Request) (*http.Response, error)
 	if err != nil {
 		return nil, err
 	}
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
-		return resp, fmt.Errorf("got non-200 status code %d", resp.StatusCode)
+		return resp, fmt.Errorf("got non-200/201 status code %d", resp.StatusCode)
 	}
 	return resp, nil
 }

cmd/containerboot/main.go

@@ -4,13 +4,13 @@
 
 //go:build linux
 
-// The containerboot binary is a wrapper for starting tailscaled in a
+// The containerboot binary is a wrapper for starting tailscaled in a container.
-// container. It handles reading the desired mode of operation out of
+// It handles reading the desired mode of operation out of environment
-// environment variables, bringing up and authenticating Tailscale,
+// variables, bringing up and authenticating Tailscale, and any other
-// and any other kubernetes-specific side jobs.
+// kubernetes-specific side jobs.
 //
-// As with most container things, configuration is passed through
+// As with most container things, configuration is passed through environment
-// environment variables. All configuration is optional.
+// variables. All configuration is optional.
 //
 //   - TS_AUTH_KEY: the authkey to use for login.
 //   - TS_ROUTES: subnet routes to advertise.
@@ -37,9 +37,13 @@
 //     compatibility), forcibly log in every time the
 //     container starts.
 //
-// When running on Kubernetes, TS_KUBE_SECRET takes precedence over
+// When running on Kubernetes, containerboot defaults to storing state in the
-// TS_STATE_DIR. Additionally, if TS_AUTH_KEY is not provided and the
+// "tailscale" kube secret. To store state on local disk instead, set
-// TS_KUBE_SECRET contains an "authkey" field, that key is used.
+// TS_KUBE_SECRET="" and TS_STATE_DIR=/path/to/storage/dir. The state dir should
+// be persistent storage.
+//
+// Additionally, if TS_AUTH_KEY is not set and the TS_KUBE_SECRET contains an
+// "authkey" field, that key is used as the tailscale authkey.
 package main
 
 import (
@@ -116,30 +120,53 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.AuthKey == "" {
+	if cfg.InKubernetes && cfg.KubeSecret != "" {
-		key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret)
+		canPatch, err := checkSecretPermissions(ctx, cfg.KubeSecret)
 		if err != nil {
-			log.Fatalf("Getting authkey from kube secret: %v", err)
+			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
 		}
-		if key != "" {
+		cfg.KubernetesCanPatch = canPatch
-			log.Print("Using authkey found in kube secret")
+
-			cfg.AuthKey = key
+		if cfg.AuthKey == "" {
-		} else {
+			key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret)
-			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+			if err != nil {
+				log.Fatalf("Getting authkey from kube secret: %v", err)
+			}
+			if key != "" {
+				// This behavior of pulling authkeys from kube secrets was added
+				// at the same time as the patch permission, so we can enforce
+				// that we must be able to patch out the authkey after
+				// authenticating if you want to use this feature. This avoids
+				// us having to deal with the case where we might leave behind
+				// an unnecessary reusable authkey in a secret, like a rake in
+				// the grass.
+				if !cfg.KubernetesCanPatch {
+					log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+				}
+				log.Print("Using authkey found in kube secret")
+				cfg.AuthKey = key
+			} else {
+				log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+			}
 		}
 	}
 
-	st, daemonPid, err := startAndAuthTailscaled(ctx, cfg)
+	client, daemonPid, err := startTailscaled(ctx, cfg)
 	if err != nil {
 		log.Fatalf("failed to bring up tailscale: %v", err)
 	}
 
+	st, err := authTailscaled(ctx, client, cfg)
+	if err != nil {
+		log.Fatalf("failed to auth tailscale: %v", err)
+	}
+
 	if cfg.ProxyTo != "" {
 		if err := installIPTablesRule(ctx, cfg.ProxyTo, st.TailscaleIPs); err != nil {
 			log.Fatalf("installing proxy rules: %v", err)
 		}
 	}
-	if cfg.InKubernetes && cfg.KubeSecret != "" {
+	if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" {
 		if err := storeDeviceID(ctx, cfg.KubeSecret, string(st.Self.ID)); err != nil {
 			log.Fatalf("storing device ID in kube secret: %v", err)
 		}
@@ -173,10 +200,7 @@ func main() {
 	}
 }
 
-// startAndAuthTailscaled starts the tailscale daemon and attempts to
+func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, int, error) {
-// auth it, according to the settings in cfg. If successful, returns
-// tailscaled's Status and pid.
-func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Status, int, error) {
 	args := tailscaledArgs(cfg)
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, unix.SIGTERM, unix.SIGINT)
@@ -198,8 +222,7 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 		cmd.Process.Signal(unix.SIGTERM)
 	}()
 
-	// Wait for the socket file to appear, otherwise 'tailscale up'
+	// Wait for the socket file to appear, otherwise API ops will racily fail.
-	// can fail.
 	log.Printf("Waiting for tailscaled socket")
 	for {
 		if ctx.Err() != nil {
@@ -215,39 +238,46 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 		break
 	}
 
+	tsClient := &tailscale.LocalClient{
+		Socket:        cfg.Socket,
+		UseSocketOnly: true,
+	}
+
+	return tsClient, cmd.Process.Pid, nil
+}
+
+// startAndAuthTailscaled starts the tailscale daemon and attempts to
+// auth it, according to the settings in cfg. If successful, returns
+// tailscaled's Status and pid.
+func authTailscaled(ctx context.Context, client *tailscale.LocalClient, cfg *settings) (*ipnstate.Status, error) {
 	didLogin := false
 	if !cfg.AuthOnce {
 		if err := tailscaleUp(ctx, cfg); err != nil {
-			return nil, 0, fmt.Errorf("couldn't log in: %v", err)
+			return nil, fmt.Errorf("couldn't log in: %v", err)
 		}
 		didLogin = true
 	}
 
-	tsClient := tailscale.LocalClient{
-		Socket:        cfg.Socket,
-		UseSocketOnly: true,
-	}
-
 	// Poll for daemon state until it goes to either Running or
 	// NeedsLogin. The latter only happens if cfg.AuthOnce is true,
 	// because in that case we only try to auth when it's necessary to
 	// reach the running state.
 	for {
 		if ctx.Err() != nil {
-			return nil, 0, ctx.Err()
+			return nil, ctx.Err()
 		}
 
 		loopCtx, cancel := context.WithTimeout(ctx, time.Second)
-		st, err := tsClient.Status(loopCtx)
+		st, err := client.Status(loopCtx)
 		cancel()
 		if err != nil {
-			return nil, 0, fmt.Errorf("Getting tailscaled state: %w", err)
+			return nil, fmt.Errorf("Getting tailscaled state: %w", err)
 		}
 
 		switch st.BackendState {
 		case "Running":
 			if len(st.TailscaleIPs) > 0 {
-				return st, cmd.Process.Pid, nil
+				return st, nil
 			}
 			log.Printf("No Tailscale IPs assigned yet")
 		case "NeedsLogin":
@@ -256,7 +286,7 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 				// LocalAPI, so we still have to shell out to the
 				// tailscale CLI for this bit.
 				if err := tailscaleUp(ctx, cfg); err != nil {
-					return nil, 0, fmt.Errorf("couldn't log in: %v", err)
+					return nil, fmt.Errorf("couldn't log in: %v", err)
 				}
 				didLogin = true
 			}
@@ -275,7 +305,7 @@ func tailscaledArgs(cfg *settings) []string {
 	case cfg.InKubernetes && cfg.KubeSecret != "":
 		args = append(args, "--state=kube:"+cfg.KubeSecret, "--statedir=/tmp")
 	case cfg.StateDir != "":
-		args = append(args, "--state="+cfg.StateDir)
+		args = append(args, "--statedir="+cfg.StateDir)
 	default:
 		args = append(args, "--state=mem:", "--statedir=/tmp")
 	}
@@ -433,27 +463,28 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Addr)
 
 // settings is all the configuration for containerboot.
 type settings struct {
-	AuthKey         string
+	AuthKey            string
-	Routes          string
+	Routes             string
-	ProxyTo         string
+	ProxyTo            string
-	DaemonExtraArgs string
+	DaemonExtraArgs    string
-	ExtraArgs       string
+	ExtraArgs          string
-	InKubernetes    bool
+	InKubernetes       bool
-	UserspaceMode   bool
+	UserspaceMode      bool
-	StateDir        string
+	StateDir           string
-	AcceptDNS       bool
+	AcceptDNS          bool
-	KubeSecret      string
+	KubeSecret         string
-	SOCKSProxyAddr  string
+	SOCKSProxyAddr     string
-	HTTPProxyAddr   string
+	HTTPProxyAddr      string
-	Socket          string
+	Socket             string
-	AuthOnce        bool
+	AuthOnce           bool
-	Root            string
+	Root               string
+	KubernetesCanPatch bool
 }
 
 // defaultEnv returns the value of the given envvar name, or defVal if
 // unset.
 func defaultEnv(name, defVal string) string {
-	if v := os.Getenv(name); v != "" {
+	if v, ok := os.LookupEnv(name); ok {
 		return v
 	}
 	return defVal

cmd/containerboot/main_test.go

@@ -33,6 +33,7 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
 )
 
 func TestContainerBoot(t *testing.T) {
@@ -97,31 +98,45 @@ func TestContainerBoot(t *testing.T) {
 	// step. Right now all of containerboot's modes either converge
 	// with no further interaction needed, or with one extra step
 	// only.
-	tests := []struct {
+	type phase struct {
-		Name           string
+		// Make LocalAPI report this status, then wait for the Wants below to be
-		Env            map[string]string
+		// satisfied. A zero Status is a valid state for a just-started
-		KubeSecret     map[string]string
+		// tailscaled.
-		WantArgs1      []string        // Wait for containerboot to run these commands...
+		Status ipnstate.Status
-		Status1        ipnstate.Status // ... then report this status in LocalAPI.
+
-		WantArgs2      []string        // If non-nil, wait for containerboot to run these additional commands...
+		// WantCmds is the commands that containerboot should run in this phase.
-		Status2        ipnstate.Status // ... then report this status in LocalAPI.
+		WantCmds []string
+		// WantKubeSecret is the secret keys/values that should exist in the
+		// kube secret.
 		WantKubeSecret map[string]string
-		WantFiles      map[string]string
+		// WantFiles files that should exist in the container and their
+		// contents.
+		WantFiles map[string]string
+	}
+	tests := []struct {
+		Name          string
+		Env           map[string]string
+		KubeSecret    map[string]string
+		KubeDenyPatch bool
+		Phases        []phase
 	}{
 		{
 			// Out of the box default: runs in userspace mode, ephemeral storage, interactive login.
 			Name: "no_args",
 			Env:  nil,
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-			// The tailscale up call blocks until auth is complete, so
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
-			// by the time it returns the next converged state is
+					},
-			// Running.
+				},
-			Status1: ipnstate.Status{
+				{
-				BackendState: "Running",
+					Status: ipnstate.Status{
-				TailscaleIPs: tsIPs,
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -130,13 +145,19 @@ func TestContainerBoot(t *testing.T) {
 			Env: map[string]string{
 				"TS_AUTH_KEY": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -145,13 +166,19 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_STATE_DIR": filepath.Join(d, "tmp"),
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -160,17 +187,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY": "tskey-key",
 				"TS_ROUTES":   "1.2.3.0/24,10.20.30.0/24",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
-			},
+				{
-			WantFiles: map[string]string{
+					Status: ipnstate.Status{
-				"proc/sys/net/ipv4/ip_forward":          "0",
+						BackendState: "Running",
-				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "0",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
 			},
 		},
 		{
@@ -180,17 +213,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "1.2.3.0/24,10.20.30.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
-			},
+				{
-			WantFiles: map[string]string{
+					Status: ipnstate.Status{
-				"proc/sys/net/ipv4/ip_forward":          "1",
+						BackendState: "Running",
-				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "0",
+					},
+				},
 			},
 		},
 		{
@@ -200,17 +239,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1::/64",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
-			},
+				{
-			WantFiles: map[string]string{
+					Status: ipnstate.Status{
-				"proc/sys/net/ipv4/ip_forward":          "0",
+						BackendState: "Running",
-				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "0",
+						"proc/sys/net/ipv6/conf/all/forwarding": "1",
+					},
+				},
 			},
 		},
 		{
@@ -220,17 +265,23 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1.2.3.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
-			},
+				{
-			WantFiles: map[string]string{
+					Status: ipnstate.Status{
-				"proc/sys/net/ipv4/ip_forward":          "1",
+						BackendState: "Running",
-				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+						TailscaleIPs: tsIPs,
+					},
+					WantFiles: map[string]string{
+						"proc/sys/net/ipv4/ip_forward":          "1",
+						"proc/sys/net/ipv6/conf/all/forwarding": "1",
+					},
+				},
 			},
 		},
 		{
@@ -240,20 +291,22 @@ func TestContainerBoot(t *testing.T) {
 				"TS_DEST_IP":   "1.2.3.4",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
-			},
+				{
-			WantArgs2: []string{
+					Status: ipnstate.Status{
-				"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
+						BackendState: "Running",
-			},
+						TailscaleIPs: tsIPs,
-			Status2: ipnstate.Status{
+					},
-				BackendState: "Running",
+					WantCmds: []string{
-				TailscaleIPs: tsIPs,
+						"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
+					},
+				},
 			},
 		},
 		{
@@ -262,18 +315,26 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_AUTH_ONCE": "true",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				{
-			},
+					WantCmds: []string{
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-				BackendState: "NeedsLogin",
+					},
-			},
+				},
-			WantArgs2: []string{
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					Status: ipnstate.Status{
-			},
+						BackendState: "NeedsLogin",
-			Status2: ipnstate.Status{
+					},
-				BackendState: "Running",
+					WantCmds: []string{
-				TailscaleIPs: tsIPs,
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -285,20 +346,83 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+					WantKubeSecret: map[string]string{
-				Self: &ipnstate.PeerStatus{
+						"authkey": "tskey-key",
-					ID: tailcfg.StableNodeID("myID"),
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{
+						"authkey":   "tskey-key",
+						"device_id": "myID",
+					},
 				},
 			},
-			WantKubeSecret: map[string]string{
+		},
-				"authkey":   "tskey-key",
+		{
-				"device_id": "myID",
+			Name: "kube_disk_storage",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				// Explicitly set to an empty value, to override the default of "tailscale".
+				"TS_KUBE_SECRET": "",
+				"TS_STATE_DIR":   filepath.Join(d, "tmp"),
+				"TS_AUTH_KEY":    "tskey-key",
+			},
+			KubeSecret: map[string]string{},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{},
+				},
+				{
+					Notify:         runningNotify,
+					WantKubeSecret: map[string]string{},
+				},
+			},
+		},
+		{
+			Name: "kube_storage_no_patch",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_AUTH_KEY":                   "tskey-key",
+			},
+			KubeSecret:    map[string]string{},
+			KubeDenyPatch: true,
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{},
+				},
 			},
 		},
 		{
@@ -312,24 +436,38 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+				{
-			},
+					WantCmds: []string{
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-				BackendState: "NeedsLogin",
+					},
-			},
+					WantKubeSecret: map[string]string{
-			WantArgs2: []string{
+						"authkey": "tskey-key",
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
-			},
+				},
-			Status2: ipnstate.Status{
+				{
-				BackendState: "Running",
+					Status: ipnstate.Status{
-				TailscaleIPs: tsIPs,
+						BackendState: "NeedsLogin",
-				Self: &ipnstate.PeerStatus{
+					},
-					ID: tailcfg.StableNodeID("myID"),
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+					},
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+						Self: &ipnstate.PeerStatus{
+							ID: tailcfg.StableNodeID("myID"),
+						},
+					},
+					WantKubeSecret: map[string]string{
+						"device_id": "myID",
+					},
 				},
-			},
-			WantKubeSecret: map[string]string{
-				"device_id": "myID",
 			},
 		},
 		{
@@ -338,16 +476,22 @@ func TestContainerBoot(t *testing.T) {
 				"TS_SOCKS5_SERVER":              "localhost:1080",
 				"TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
-			// The tailscale up call blocks until auth is complete, so
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
-			// by the time it returns the next converged state is
+					},
-			// Running.
+				},
-			Status1: ipnstate.Status{
+				{
-				BackendState: "Running",
+					// The tailscale up call blocks until auth is complete, so
-				TailscaleIPs: tsIPs,
+					// by the time it returns the next converged state is
+					// Running.
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -355,13 +499,19 @@ func TestContainerBoot(t *testing.T) {
 			Env: map[string]string{
 				"TS_ACCEPT_DNS": "true",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				},
+				{
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 		{
@@ -370,13 +520,18 @@ func TestContainerBoot(t *testing.T) {
 				"TS_EXTRA_ARGS":            "--widget=rotated",
 				"TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
-				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
+				{
-				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
+					WantCmds: []string{
-			},
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
-			Status1: ipnstate.Status{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
-				BackendState: "Running",
+					},
-				TailscaleIPs: tsIPs,
+				}, {
+					Status: ipnstate.Status{
+						BackendState: "Running",
+						TailscaleIPs: tsIPs,
+					},
+				},
 			},
 		},
 	}
@@ -392,6 +547,7 @@ func TestContainerBoot(t *testing.T) {
 			for k, v := range test.KubeSecret {
 				kube.SetSecret(k, v)
 			}
+			kube.SetPatching(!test.KubeDenyPatch)
 
 			cmd := exec.Command(boot)
 			cmd.Env = []string{
@@ -419,35 +575,45 @@ func TestContainerBoot(t *testing.T) {
 				cmd.Process.Wait()
 			}()
 
-			waitArgs(t, 2*time.Second, d, argFile, strings.Join(test.WantArgs1, "\n"))
+			var wantCmds []string
-			lapi.SetStatus(test.Status1)
+			for _, p := range test.Phases {
-			if test.WantArgs2 != nil {
+				lapi.SetStatus(p.Status)
-				waitArgs(t, 2*time.Second, d, argFile, strings.Join(append(test.WantArgs1, test.WantArgs2...), "\n"))
+				wantCmds = append(wantCmds, p.WantCmds...)
-				lapi.SetStatus(test.Status2)
+				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
+				err := tstest.WaitFor(2*time.Second, func() error {
+					if p.WantKubeSecret != nil {
+						got := kube.Secret()
+						if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" {
+							return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff)
+						}
+					} else {
+						got := kube.Secret()
+						if len(got) > 0 {
+							return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got)
+						}
+					}
+					return nil
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = tstest.WaitFor(2*time.Second, func() error {
+					for path, want := range p.WantFiles {
+						gotBs, err := os.ReadFile(filepath.Join(d, path))
+						if err != nil {
+							return fmt.Errorf("reading wanted file %q: %v", path, err)
+						}
+						if got := strings.TrimSpace(string(gotBs)); got != want {
+							return fmt.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
+						}
+					}
+					return nil
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
 			}
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
-
-			if test.WantKubeSecret != nil {
-				got := kube.Secret()
-				if diff := cmp.Diff(got, test.WantKubeSecret); diff != "" {
-					t.Fatalf("unexpected kube secret data (-got+want):\n%s", diff)
-				}
-			} else {
-				got := kube.Secret()
-				if len(got) != 0 {
-					t.Fatalf("kube secret unexpectedly not empty, got %#v", got)
-				}
-			}
-
-			for path, want := range test.WantFiles {
-				gotBs, err := os.ReadFile(filepath.Join(d, path))
-				if err != nil {
-					t.Fatalf("reading wanted file %q: %v", path, err)
-				}
-				if got := strings.TrimSpace(string(gotBs)); got != want {
-					t.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
-				}
-			}
 		})
 	}
 }
@@ -612,7 +778,8 @@ type kubeServer struct {
 	srv *httptest.Server
 
 	sync.Mutex
-	secret map[string]string
+	secret   map[string]string
+	canPatch bool
 }
 
 func (k *kubeServer) Secret() map[string]string {
@@ -631,6 +798,12 @@ func (k *kubeServer) SetSecret(key, val string) {
 	k.secret[key] = val
 }
 
+func (k *kubeServer) SetPatching(canPatch bool) {
+	k.Lock()
+	defer k.Unlock()
+	k.canPatch = canPatch
+}
+
 func (k *kubeServer) Reset() {
 	k.Lock()
 	defer k.Unlock()
@@ -674,10 +847,39 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Authorization") != "Bearer bearer_token" {
 		panic("client didn't provide bearer token in request")
 	}
-	if r.URL.Path != "/api/v1/namespaces/default/secrets/tailscale" {
+	switch r.URL.Path {
+	case "/api/v1/namespaces/default/secrets/tailscale":
+		k.serveSecret(w, r)
+	case "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews":
+		k.serveSSAR(w, r)
+	default:
 		panic(fmt.Sprintf("unhandled fake kube api path %q", r.URL.Path))
 	}
+}
 
+func (k *kubeServer) serveSSAR(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Spec struct {
+			ResourceAttributes struct {
+				Verb string `json:"verb"`
+			} `json:"resourceAttributes"`
+		} `json:"spec"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		panic(fmt.Sprintf("decoding SSAR request: %v", err))
+	}
+	ok := true
+	if req.Spec.ResourceAttributes.Verb == "patch" {
+		k.Lock()
+		defer k.Unlock()
+		ok = k.canPatch
+	}
+	// Just say yes to all SARs, we don't enforce RBAC.
+	w.Header().Set("Content-Type", "application/json")
+	fmt.Fprintf(w, `{"status":{"allowed":%v}}`, ok)
+}
+
+func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
 	bs, err := io.ReadAll(r.Body)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError)
@@ -688,7 +890,7 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case "GET":
 		w.Header().Set("Content-Type", "application/json")
 		ret := map[string]map[string]string{
-			"data": map[string]string{},
+			"data": {},
 		}
 		k.Lock()
 		defer k.Unlock()
@@ -703,6 +905,11 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			panic("encode failed")
 		}
 	case "PATCH":
+		k.Lock()
+		defer k.Unlock()
+		if !k.canPatch {
+			panic("containerboot tried to patch despite not being allowed")
+		}
 		switch r.Header.Get("Content-Type") {
 		case "application/json-patch+json":
 			req := []struct {
@@ -712,8 +919,6 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if err := json.Unmarshal(bs, &req); err != nil {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
-			k.Lock()
-			defer k.Unlock()
 			for _, op := range req {
 				if op.Op != "remove" {
 					panic(fmt.Sprintf("unsupported json-patch op %q", op.Op))
@@ -730,8 +935,6 @@ func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if err := json.Unmarshal(bs, &req); err != nil {
 				panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
 			}
-			k.Lock()
-			defer k.Unlock()
 			for key, val := range req.Data {
 				k.secret[key] = val
 			}

cmd/tailscale/cli/ssh.go

@@ -21,6 +21,7 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/paths"
 	"tailscale.com/version"
 )
 
@@ -37,7 +38,7 @@ most users running the Tailscale SSH server will prefer to just use the normal
 
 The 'tailscale ssh' wrapper adds a few things:
 
-* It resolves the destination server name in its arugments using MagicDNS,
+* It resolves the destination server name in its arguments using MagicDNS,
   even if --accept-dns=false.
 * It works in userspace-networking mode, by supplying a ProxyCommand to the
   system 'ssh' command that connects via a pipe through tailscaled.
@@ -110,10 +111,15 @@ func runSSH(ctx context.Context, args []string) error {
 	// So don't use it for now. MagicDNS is usually working on macOS anyway
 	// and they're not in userspace mode, so 'nc' isn't very useful.
 	if runtime.GOOS != "darwin" {
+		socketArg := ""
+		if rootArgs.socket != "" && rootArgs.socket != paths.DefaultTailscaledSocket() {
+			socketArg = fmt.Sprintf("--socket=%q", rootArgs.socket)
+		}
+
 		argv = append(argv,
-			"-o", fmt.Sprintf("ProxyCommand %q --socket=%q nc %%h %%p",
+			"-o", fmt.Sprintf("ProxyCommand %q %s nc %%h %%p",
 				tailscaleBin,
-				rootArgs.socket,
+				socketArg,
 			))
 	}
 

ipn/ipnlocal/peerapi.go

@@ -607,6 +607,7 @@ func peerAPIRequestShouldGetSecurityHeaders(r *http.Request) bool {
 
 func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if err := h.validatePeerAPIRequest(r); err != nil {
+		metricInvalidRequests.Add(1)
 		h.logf("invalid request from %v: %v", h.remoteAddr, err)
 		http.Error(w, "invalid peerapi request", http.StatusForbidden)
 		return
@@ -617,10 +618,12 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Content-Type-Options", "nosniff")
 	}
 	if strings.HasPrefix(r.URL.Path, "/v0/put/") {
+		metricPutCalls.Add(1)
 		h.handlePeerPut(w, r)
 		return
 	}
 	if strings.HasPrefix(r.URL.Path, "/dns-query") {
+		metricDNSCalls.Add(1)
 		h.handleDNSQuery(w, r)
 		return
 	}
@@ -641,12 +644,14 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handleServeDNSFwd(w, r)
 		return
 	case "/v0/wol":
+		metricWakeOnLANCalls.Add(1)
 		h.handleWakeOnLAN(w, r)
 		return
 	case "/v0/interfaces":
 		h.handleServeInterfaces(w, r)
 		return
 	case "/v0/ingress":
+		metricIngressCalls.Add(1)
 		h.handleServeIngress(w, r)
 		return
 	}
@@ -1370,3 +1375,13 @@ func (fl *fakePeerAPIListener) Accept() (net.Conn, error) {
 }
 
 func (fl *fakePeerAPIListener) Addr() net.Addr { return fl.addr }
+
+var (
+	metricInvalidRequests = clientmetric.NewCounter("peerapi_invalid_requests")
+
+	// Non-debug PeerAPI endpoints.
+	metricPutCalls       = clientmetric.NewCounter("peerapi_put")
+	metricDNSCalls       = clientmetric.NewCounter("peerapi_dns")
+	metricWakeOnLANCalls = clientmetric.NewCounter("peerapi_wol")
+	metricIngressCalls   = clientmetric.NewCounter("peerapi_ingress")
+)

ipn/localapi/localapi.go

@@ -146,6 +146,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if r.Referer() != "" || r.Header.Get("Origin") != "" || !validHost(r.Host) {
+		metricInvalidRequests.Add(1)
 		http.Error(w, "invalid localapi request", http.StatusForbidden)
 		return
 	}
@@ -156,10 +157,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if h.RequiredPassword != "" {
 		_, pass, ok := r.BasicAuth()
 		if !ok {
+			metricInvalidRequests.Add(1)
 			http.Error(w, "auth required", http.StatusUnauthorized)
 			return
 		}
 		if pass != h.RequiredPassword {
+			metricInvalidRequests.Add(1)
 			http.Error(w, "bad password", http.StatusForbidden)
 			return
 		}
@@ -900,6 +903,8 @@ func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
 //
 //   - PUT /localapi/v0/file-put/:stableID/:escaped-filename
 func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
+	metricFilePutCalls.Add(1)
+
 	if !h.PermitWrite {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
@@ -1437,3 +1442,10 @@ func defBool(a string, def bool) bool {
 	}
 	return v
 }
+
+var (
+	metricInvalidRequests = clientmetric.NewCounter("localapi_invalid_requests")
+
+	// User-visible LocalAPI endpoints.
+	metricFilePutCalls = clientmetric.NewCounter("localapi_file_put")
+)

wgengine/magicsock/magicsock.go

@@ -3008,13 +3008,14 @@ func (c *Conn) ParseEndpoint(nodeKeyStr string) (conn.Endpoint, error) {
 // RebindingUDPConn is a UDP socket that can be re-bound.
 // Unix has no notion of re-binding a socket, so we swap it out for a new one.
 type RebindingUDPConn struct {
-	// pconnAtomic is the same as pconn, but doesn't require acquiring mu. It's
+	// pconnAtomic is a pointer to the value stored in pconn, but doesn't
-	// used for reads/writes and only upon failure do the reads/writes then
+	// require acquiring mu. It's used for reads/writes and only upon failure
-	// check pconn (after acquiring mu) to see if there's been a rebind
+	// do the reads/writes then check pconn (after acquiring mu) to see if
-	// meanwhile.
+	// there's been a rebind meanwhile.
 	// pconn isn't really needed, but makes some of the code simpler
-	// to keep it in a type safe form.
+	// to keep it distinct.
-	pconnAtomic syncs.AtomicValue[nettype.PacketConn]
+	// Neither is expected to be nil, sockets are bound on creation.
+	pconnAtomic atomic.Pointer[nettype.PacketConn]
 
 	mu    sync.Mutex // held while changing pconn (and pconnAtomic)
 	pconn nettype.PacketConn
@@ -3023,7 +3024,7 @@ type RebindingUDPConn struct {
 
 func (c *RebindingUDPConn) setConnLocked(p nettype.PacketConn) {
 	c.pconn = p
-	c.pconnAtomic.Store(p)
+	c.pconnAtomic.Store(&p)
 	c.port = uint16(c.localAddrLocked().Port)
 }
 
@@ -3038,7 +3039,7 @@ func (c *RebindingUDPConn) currentConn() nettype.PacketConn {
 // It returns the number of bytes copied and the source address.
 func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
 		n, addr, err := pconn.ReadFrom(b)
 		if err != nil && pconn != c.currentConn() {
 			continue
@@ -3056,7 +3057,7 @@ func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 // when c's underlying connection is a net.UDPConn.
 func (c *RebindingUDPConn) ReadFromNetaddr(b []byte) (n int, ipp netip.AddrPort, err error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
 
 		// Optimization: Treat *net.UDPConn specially.
 		// This lets us avoid allocations by calling ReadFromUDPAddrPort.
@@ -3122,13 +3123,10 @@ func (c *RebindingUDPConn) closeLocked() error {
 
 func (c *RebindingUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
-
 		n, err := pconn.WriteTo(b, addr)
-		if err != nil {
+		if err != nil && pconn != c.currentConn() {
-			if pconn != c.currentConn() {
+			continue
-				continue
-			}
 		}
 		return n, err
 	}
@@ -3136,13 +3134,10 @@ func (c *RebindingUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
 
 func (c *RebindingUDPConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (int, error) {
 	for {
-		pconn := c.pconnAtomic.Load()
+		pconn := *c.pconnAtomic.Load()
-
 		n, err := pconn.WriteToUDPAddrPort(b, addr)
-		if err != nil {
+		if err != nil && pconn != c.currentConn() {
-			if pconn != c.currentConn() {
+			continue
-				continue
-			}
 		}
 		return n, err
 	}
@@ -3334,7 +3329,7 @@ type endpoint struct {
 	publicKey    key.NodePublic // peer public key (for WireGuard + DERP)
 	publicKeyHex string         // cached output of publicKey.UntypedHexString
 	fakeWGAddr   netip.AddrPort // the UDP address we tell wireguard-go we're using
-	nodeAddr     netip.Addr     // the node's first tailscale address (only used for logging)
+	nodeAddr     netip.Addr     // the node's first tailscale address; used for logging & wireguard rate-limiting (Issue 6686)
 
 	// mu protects all following fields.
 	mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu
@@ -3521,7 +3516,7 @@ func (de *endpoint) ClearSrc()           {}
 func (de *endpoint) SrcToString() string { panic("unused") } // unused by wireguard-go
 func (de *endpoint) SrcIP() netip.Addr   { panic("unused") } // unused by wireguard-go
 func (de *endpoint) DstToString() string { return de.publicKeyHex }
-func (de *endpoint) DstIP() netip.Addr   { panic("unused") }
+func (de *endpoint) DstIP() netip.Addr   { return de.nodeAddr } // see tailscale/tailscale#6686
 func (de *endpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr) }
 
 // addrForSendLocked returns the address(es) that should be used for

wgengine/magicsock/magicsock_test.go

@@ -1803,3 +1803,16 @@ func TestDiscoMagicMatches(t *testing.T) {
 		t.Errorf("last 2 bytes of disco magic don't match, got %v want %v", discoMagic2, m2)
 	}
 }
+
+func TestRebindingUDPConn(t *testing.T) {
+	// Test that RebindingUDPConn can be re-bound to different connection
+	// types.
+	c := RebindingUDPConn{}
+	realConn, err := net.ListenPacket("udp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer realConn.Close()
+	c.setConnLocked(realConn.(nettype.PacketConn))
+	c.setConnLocked(newBlockForeverConn())
+}

wgengine/router/router_linux_test.go

@@ -11,8 +11,10 @@ import (
 	"net/netip"
 	"os"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 
@@ -341,7 +343,7 @@ ip route add throw 192.168.0.0/24 table 52` + basic,
 			t.Fatalf("failed to set router config: %v", err)
 		}
 		got := fake.String()
-		want := strings.TrimSpace(states[i].want)
+		want := adjustFwmask(t, strings.TrimSpace(states[i].want))
 		if diff := cmp.Diff(got, want); diff != "" {
 			t.Fatalf("unexpected OS state (-got+want):\n%s", diff)
 		}
@@ -922,3 +924,23 @@ func TestCIDRDiff(t *testing.T) {
 		}
 	}
 }
+
+var (
+	fwmaskSupported     bool
+	fwmaskSupportedOnce sync.Once
+	fwmaskAdjustRe      = regexp.MustCompile(`(?m)(fwmark 0x[0-9a-f]+)/0x[0-9a-f]+`)
+)
+
+// adjustFwmask removes the "/0xmask" string from fwmask stanzas if the
+// installed 'ip' binary does not support that format.
+func adjustFwmask(t *testing.T, s string) string {
+	t.Helper()
+	fwmaskSupportedOnce.Do(func() {
+		fwmaskSupported, _ = ipCmdSupportsFwmask()
+	})
+	if fwmaskSupported {
+		return s
+	}
+
+	return fwmaskAdjustRe.ReplaceAllString(s, "$1")
+}