ipn/store: add ability to store data as k8s secrets.

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/tailscale/depaware.txt

@@ -31,6 +31,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/kube                                           from tailscale.com/ipn
         tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/flowtrack                                  from tailscale.com/wgengine/filter+

cmd/tailscaled/depaware.txt

@@ -105,6 +105,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/ipnstate                                   from tailscale.com/ipn+
         tailscale.com/ipn/localapi                                   from tailscale.com/ipn/ipnserver
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
+        tailscale.com/kube                                           from tailscale.com/ipn
         tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled

cmd/tailscaled/tailscaled.go

@@ -111,7 +111,7 @@ func main() {
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")

docs/k8s/README.md

@@ -0,0 +1,20 @@
+# Using Kubernetes Secrets as the state store for Tailscale
+Tailscale supports using Kubernetes Secrets as the state store, however there is some configuration required in order for it to work.
+
+**Note: this only works if `tailscaled` runs inside a pod in the cluster.**
+
+1. Create a service account for Tailscale (optional)
+   ```
+   kubectl create -f sa.yaml
+   ```
+
+1. Create role and role bindings for the service account
+   ```
+   kubectl create -f role.yaml
+   kubectl create -f rolebinding.yaml
+   ```
+
+1. Launch `tailscaled` with a Kubernetes Secret as the state store.
+   ```
+   tailscaled --state=kube:tailscale
+   ```

docs/k8s/role.yaml

@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: default
+  name: tailscale
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resourceNames: ["tailscale"]
+  resources: ["secrets"]
+  verbs: ["create", "get", "update"]

docs/k8s/rolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: default
+  name: tailscale
+subjects:
+- kind: ServiceAccount
+  name: tailscale
+roleRef:
+  kind: Role
+  name: tailscale
+  apiGroup: rbac.authorization.k8s.io

docs/k8s/sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tailscale
+  namespace: default

ipn/ipnserver/server.go

@@ -613,9 +613,19 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 
 	var store ipn.StateStore
 	if opts.StatePath != "" {
-		store, err = ipn.NewFileStore(opts.StatePath)
-		if err != nil {
-			return fmt.Errorf("ipn.NewFileStore(%q): %v", opts.StatePath, err)
+		const kubePrefix = "kube:"
+		switch {
+		case strings.HasPrefix(opts.StatePath, kubePrefix):
+			secretName := strings.TrimPrefix(opts.StatePath, kubePrefix)
+			store, err = ipn.NewKubeStore(secretName)
+			if err != nil {
+				return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
+			}
+		default:
+			store, err = ipn.NewFileStore(opts.StatePath)
+			if err != nil {
+				return fmt.Errorf("ipn.NewFileStore(%q): %v", opts.StatePath, err)
+			}
 		}
 		if opts.AutostartStateKey == "" {
 			autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)

ipn/store.go

@@ -6,6 +6,7 @@ package ipn
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -14,8 +15,10 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"tailscale.com/atomicfile"
+	"tailscale.com/kube"
 )
 
 // ErrStateNotExist is returned by StateStore.ReadState when the
@@ -55,6 +58,75 @@ type StateStore interface {
 	WriteState(id StateKey, bs []byte) error
 }
 
+// KubeStore is a StateStore that uses a Kubernetes Secret for persistence.
+type KubeStore struct {
+	client     *kube.Client
+	secretName string
+}
+
+// NewKubeStore returns a new KubeStore that persists to the named secret.
+func NewKubeStore(secretName string) (*KubeStore, error) {
+	c, err := kube.New()
+	if err != nil {
+		return nil, err
+	}
+	return &KubeStore{
+		client:     c,
+		secretName: secretName,
+	}, nil
+}
+
+func (s *KubeStore) String() string { return "KubeStore" }
+
+// ReadState implements the StateStore interface.
+func (s *KubeStore) ReadState(id StateKey) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return nil, ErrStateNotExist
+		}
+		return nil, err
+	}
+	b, ok := secret.Data[string(id)]
+	if !ok {
+		return nil, ErrStateNotExist
+	}
+	return b, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *KubeStore) WriteState(id StateKey, bs []byte) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return s.client.CreateSecret(ctx, &kube.Secret{
+				TypeMeta: kube.TypeMeta{
+					APIVersion: "v1",
+					Kind:       "Secret",
+				},
+				ObjectMeta: kube.ObjectMeta{
+					Name: s.secretName,
+				},
+				Data: map[string][]byte{
+					string(id): bs,
+				},
+			})
+		}
+		return err
+	}
+	secret.Data[string(id)] = bs
+	if err := s.client.UpdateSecret(ctx, secret); err != nil {
+		return err
+	}
+	return err
+}
+
 // MemoryStore is a store that keeps state in memory only.
 type MemoryStore struct {
 	mu    sync.Mutex

kube/api.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package kube
+
+import "time"
+
+// Note: The API types are copied from k8s.io/api{,machinery} to not introduce a
+// module dependency on the Kubernetes API as it pulls in many more dependencies.
+
+// TypeMeta describes an individual object in an API response or request with
+// strings representing the type of the object and its API schema version.
+// Structures that are versioned or persisted should inline TypeMeta.
+type TypeMeta struct {
+	// Kind is a string value representing the REST resource this object represents.
+	// Servers may infer this from the endpoint the client submits requests to.
+	// Cannot be updated.
+	// In CamelCase.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	// +optional
+	Kind string `json:"kind,omitempty"`
+
+	// APIVersion defines the versioned schema of this representation of an object.
+	// Servers should convert recognized schemas to the latest internal value, and
+	// may reject unrecognized values.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	// +optional
+	APIVersion string `json:"apiVersion,omitempty"`
+}
+
+// ObjectMeta is metadata that all persisted resources must have, which
+// includes all objects users must create.
+type ObjectMeta struct {
+	// Name must be unique within a namespace. Is required when creating resources, although
+	// some resources may allow a client to request the generation of an appropriate name
+	// automatically. Name is primarily intended for creation idempotence and configuration
+	// definition.
+	// Cannot be updated.
+	// More info: http://kubernetes.io/docs/user-guide/identifiers#names
+	// +optional
+	Name string `json:"name"`
+
+	// Namespace defines the space within which each name must be unique. An empty namespace is
+	// equivalent to the "default" namespace, but "default" is the canonical representation.
+	// Not all objects are required to be scoped to a namespace - the value of this field for
+	// those objects will be empty.
+	//
+	// Must be a DNS_LABEL.
+	// Cannot be updated.
+	// More info: http://kubernetes.io/docs/user-guide/namespaces
+	// +optional
+	Namespace string `json:"namespace"`
+
+	// UID is the unique in time and space value for this object. It is typically generated by
+	// the server on successful creation of a resource and is not allowed to change on PUT
+	// operations.
+	//
+	// Populated by the system.
+	// Read-only.
+	// More info: http://kubernetes.io/docs/user-guide/identifiers#uids
+	// +optional
+	UID string `json:"uid,omitempty"`
+
+	// An opaque value that represents the internal version of this object that can
+	// be used by clients to determine when objects have changed. May be used for optimistic
+	// concurrency, change detection, and the watch operation on a resource or set of resources.
+	// Clients must treat these values as opaque and passed unmodified back to the server.
+	// They may only be valid for a particular resource or set of resources.
+	//
+	// Populated by the system.
+	// Read-only.
+	// Value must be treated as opaque by clients and .
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+	// +optional
+	ResourceVersion string `json:"resourceVersion,omitempty"`
+
+	// A sequence number representing a specific generation of the desired state.
+	// Populated by the system. Read-only.
+	// +optional
+	Generation int64 `json:"generation,omitempty"`
+
+	// CreationTimestamp is a timestamp representing the server time when this object was
+	// created. It is not guaranteed to be set in happens-before order across separate operations.
+	// Clients may not set this value. It is represented in RFC3339 form and is in UTC.
+	//
+	// Populated by the system.
+	// Read-only.
+	// Null for lists.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	CreationTimestamp time.Time `json:"creationTimestamp,omitempty"`
+
+	// DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This
+	// field is set by the server when a graceful deletion is requested by the user, and is not
+	// directly settable by a client. The resource is expected to be deleted (no longer visible
+	// from resource lists, and not reachable by name) after the time in this field, once the
+	// finalizers list is empty. As long as the finalizers list contains items, deletion is blocked.
+	// Once the deletionTimestamp is set, this value may not be unset or be set further into the
+	// future, although it may be shortened or the resource may be deleted prior to this time.
+	// For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react
+	// by sending a graceful termination signal to the containers in the pod. After that 30 seconds,
+	// the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup,
+	// remove the pod from the API. In the presence of network partitions, this object may still
+	// exist after this timestamp, until an administrator or automated process can determine the
+	// resource is fully terminated.
+	// If not set, graceful deletion of the object has not been requested.
+	//
+	// Populated by the system when a graceful deletion is requested.
+	// Read-only.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	DeletionTimestamp *time.Time `json:"deletionTimestamp,omitempty"`
+
+	// Number of seconds allowed for this object to gracefully terminate before
+	// it will be removed from the system. Only set when deletionTimestamp is also set.
+	// May only be shortened.
+	// Read-only.
+	// +optional
+	DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty"`
+
+	// Map of string keys and values that can be used to organize and categorize
+	// (scope and select) objects. May match selectors of replication controllers
+	// and services.
+	// More info: http://kubernetes.io/docs/user-guide/labels
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+
+	// Annotations is an unstructured key value map stored with a resource that may be
+	// set by external tools to store and retrieve arbitrary metadata. They are not
+	// queryable and should be preserved when modifying objects.
+	// More info: http://kubernetes.io/docs/user-guide/annotations
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// Secret holds secret data of a certain type. The total bytes of the values
+// in the Data field must be less than MaxSecretSize bytes.
+type Secret struct {
+	TypeMeta   `json:",inline"`
+	ObjectMeta `json:"metadata"`
+
+	// Data contains the secret data. Each key must consist of alphanumeric
+	// characters, '-', '_' or '.'. The serialized form of the secret data is a
+	// base64 encoded string, representing the arbitrary (possibly non-string)
+	// data value here. Described in https://tools.ietf.org/html/rfc4648#section-4
+	// +optional
+	Data map[string][]byte `json:"data,omitempty"`
+}
+
+// Status is a return value for calls that don't return other objects.
+type Status struct {
+	TypeMeta `json:",inline"`
+	// Status of the operation.
+	// One of: "Success" or "Failure".
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
+	// +optional
+	Status string `json:"status,omitempty"`
+
+	// A human-readable description of the status of this operation.
+	// +optional
+	Message string `json:"message,omitempty"`
+
+	// A machine-readable description of why this operation is in the
+	// "Failure" status. If this value is empty there
+	// is no information available. A Reason clarifies an HTTP status
+	// code but does not override it.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// Extended data associated with the reason.  Each reason may define its
+	// own extended details. This field is optional and the data returned
+	// is not guaranteed to conform to any schema except that defined by
+	// the reason type.
+	// +optional
+	Details *struct {
+		Name string `json:"name,omitempty"`
+		Kind string `json:"kind,omitempty"`
+	} `json:"details,omitempty"`
+
+	// Suggested HTTP return code for this status, 0 if not set.
+	// +optional
+	Code int `json:"code,omitempty"`
+}
+
+func (s *Status) Error() string {
+	return s.Message
+}

kube/client.go

@@ -0,0 +1,170 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package kube provides a client to interact with Kubernetes.
+// This package is Tailscale-internal and not meant for external consumption.
+// Further, the API should not be considered stable.
+package kube
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+const (
+	saPath     = "/var/run/secrets/kubernetes.io/serviceaccount"
+	defaultURL = "https://kubernetes.default.svc"
+)
+
+func readFile(n string) ([]byte, error) {
+	return os.ReadFile(filepath.Join(saPath, n))
+}
+
+// Client handles connections to Kubernetes.
+// It expects to be run inside a cluster.
+type Client struct {
+	mu          sync.Mutex
+	url         string
+	ns          string
+	client      *http.Client
+	token       string
+	tokenExpiry time.Time
+}
+
+// New returns a new client
+func New() (*Client, error) {
+	ns, err := readFile("namespace")
+	if err != nil {
+		return nil, err
+	}
+	caCert, err := readFile("ca.crt")
+	if err != nil {
+		return nil, err
+	}
+	cp := x509.NewCertPool()
+	if ok := cp.AppendCertsFromPEM(caCert); !ok {
+		return nil, fmt.Errorf("kube: error in creating root cert pool")
+	}
+	return &Client{
+		url: defaultURL,
+		ns:  string(ns),
+		client: &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					RootCAs: cp,
+				},
+			},
+		},
+	}, nil
+}
+
+func (c *Client) expireToken() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.tokenExpiry = time.Now()
+}
+
+func (c *Client) getOrRenewToken() (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	tk, te := c.token, c.tokenExpiry
+	if time.Now().Before(te) {
+		return tk, nil
+	}
+
+	tkb, err := readFile("token")
+	if err != nil {
+		return "", err
+	}
+	c.token = string(tkb)
+	c.tokenExpiry = time.Now().Add(30 * time.Minute)
+	return c.token, nil
+}
+
+func (c *Client) secretURL(name string) string {
+	if name == "" {
+		return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns)
+	}
+	return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets/%s", c.url, c.ns, name)
+}
+
+func getError(resp *http.Response) error {
+	if resp.StatusCode == 200 {
+		return nil
+	}
+	st := &Status{}
+	if err := json.NewDecoder(resp.Body).Decode(st); err != nil {
+		return err
+	}
+	return st
+}
+
+func (c *Client) doRequest(ctx context.Context, method, url string, in, out interface{}) error {
+	tk, err := c.getOrRenewToken()
+	if err != nil {
+		return err
+	}
+	var body io.Reader
+	if in != nil {
+		var b bytes.Buffer
+		if err := json.NewEncoder(&b).Encode(in); err != nil {
+			return err
+		}
+		body = &b
+	}
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return err
+	}
+	if body != nil {
+		req.Header.Add("Content-Type", "application/json")
+	}
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Authorization", "Bearer "+tk)
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if err := getError(resp); err != nil {
+		if st, ok := err.(*Status); ok && st.Code == 401 {
+			c.expireToken()
+		}
+		return err
+	}
+	if out != nil {
+		return json.NewDecoder(resp.Body).Decode(out)
+	}
+	return nil
+}
+
+// GetSecret fetches the secret from the Kubernetes API.
+func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) {
+	s := &Secret{Data: make(map[string][]byte)}
+	if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// CreateSecret creates a secret in the Kubernetes API.
+func (c *Client) CreateSecret(ctx context.Context, s *Secret) error {
+	s.Namespace = c.ns
+	return c.doRequest(ctx, "POST", c.secretURL(""), s, nil)
+}
+
+// UpdateSecret updates a secret in the Kubernetes API.
+func (c *Client) UpdateSecret(ctx context.Context, s *Secret) error {
+	return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil)
+}