ipn/ipnlocal: add support to store certs in k8s secrets

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/containerboot/main.go

@@ -90,8 +90,31 @@ func main() {
 		AuthOnce:        defaultBool("TS_AUTH_ONCE", false),
 		Root:            defaultEnv("TS_TEST_ONLY_ROOT", "/"),
 	}
+	funnelForwardPorts := strings.Split(defaultEnv("TS_FUNNEL_TCP_PORTFORWARD", ""), ",")
+	if len(funnelForwardPorts) > 0 {
+		ffp := make(map[uint16]uint16)
+		for _, p := range funnelForwardPorts {
+			if p == "" {
+				continue
+			}
+			from, to, ok := strings.Cut(p, ":")
+			if !ok {
+				log.Fatalf("TS_FUNNEL_TCP_PORTFORWARD: %q is not a valid port pair", p)
+			}
+			fp, err := strconv.ParseUint(from, 10, 16)
+			if err != nil {
+				log.Fatalf("TS_FUNNEL_TCP_PORTFORWARD: %v", err)
+			}
+			tp, err := strconv.ParseUint(to, 10, 16)
+			if err != nil {
+				log.Fatalf("TS_FUNNEL_TCP_PORTFORWARD: %v", err)
+			}
+			ffp[uint16(fp)] = uint16(tp)
+		}
+		cfg.FunnelTCPPorts = ffp
+	}
 
-	if cfg.ProxyTo != "" && cfg.UserspaceMode {
+	if cfg.ProxyTo != "" && cfg.UserspaceMode && len(cfg.FunnelTCPPorts) == 0 {
 		log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE")
 	}
 
@@ -240,10 +263,8 @@ authLoop:
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != ""
 		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
 		startupTasksDone  = false
-		currentIPs        deephash.Sum // tailscale IPs assigned to device
 		currentDeviceInfo deephash.Sum // device ID and fqdn
 	)
 	for {
@@ -261,11 +282,6 @@ authLoop:
 			log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
 		}
 		if n.NetMap != nil {
-			if cfg.ProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
-				if err := installIPTablesRule(ctx, cfg.ProxyTo, n.NetMap.Addresses); err != nil {
-					log.Fatalf("installing proxy rules: %v", err)
-				}
-			}
 			deviceInfo := []any{n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name}
 			if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
 				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name); err != nil {
@@ -274,7 +290,10 @@ authLoop:
 			}
 		}
 		if !startupTasksDone {
-			if (!wantProxy || currentIPs != deephash.Sum{}) && (!wantDeviceInfo || currentDeviceInfo != deephash.Sum{}) {
+			if (!wantDeviceInfo || currentDeviceInfo != deephash.Sum{}) {
+				if err := configureForwarding(ctx, client, cfg); err != nil {
+					log.Fatalf("configuring forwarding: %v", err)
+				}
 				// This log message is used in tests to detect when all
 				// post-auth configuration is done.
 				log.Println("Startup complete, waiting for shutdown signal")
@@ -305,6 +324,35 @@ authLoop:
 	}
 }
 
+func configureForwarding(ctx context.Context, client *tailscale.LocalClient, cfg *settings) error {
+	if cfg.ProxyTo == "" {
+		return nil
+	}
+	st, err := client.StatusWithoutPeers(ctx)
+	if err != nil {
+		return err
+	}
+	if len(cfg.FunnelTCPPorts) == 0 {
+		return installIPTablesRule(ctx, cfg.ProxyTo, st.Self.TailscaleIPs)
+	}
+	if len(st.CertDomains) == 0 {
+		return errors.New("no cert domains, cannot configure TCP forwarding")
+	}
+	cd := st.CertDomains[0]
+	sc := &ipn.ServeConfig{
+		AllowFunnel: make(map[ipn.HostPort]bool),
+		TCP:         make(map[uint16]*ipn.TCPPortHandler),
+	}
+	for f, t := range cfg.FunnelTCPPorts {
+		sc.TCP[f] = &ipn.TCPPortHandler{
+			TCPForward:   fmt.Sprintf("%s:%d", cfg.ProxyTo, t),
+			TerminateTLS: cd,
+		}
+		sc.AllowFunnel[ipn.HostPort(fmt.Sprintf("%s:%d", cd, f))] = true
+	}
+	return client.SetServeConfig(ctx, sc)
+}
+
 func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, int, error) {
 	args := tailscaledArgs(cfg)
 	sigCh := make(chan os.Signal, 1)
@@ -488,7 +536,7 @@ func ensureIPForwarding(root, proxyTo, routes string) error {
 	return nil
 }
 
-func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
+func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Addr) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
 		return err
@@ -498,14 +546,11 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefi
 		argv0 = "ip6tables"
 	}
 	var local string
-	for _, pfx := range tsIPs {
-		if !pfx.IsSingleIP() {
+	for _, addr := range tsIPs {
+		if addr.Is4() != dst.Is4() {
 			continue
 		}
-		if pfx.Addr().Is4() != dst.Is4() {
-			continue
-		}
-		local = pfx.Addr().String()
+		local = addr.String()
 		break
 	}
 	if local == "" {
@@ -529,6 +574,7 @@ type settings struct {
 	Hostname           string
 	Routes             string
 	ProxyTo            string
+	FunnelTCPPorts     map[uint16]uint16 // from Tailscale port -> to ProxyTo port
 	DaemonExtraArgs    string
 	ExtraArgs          string
 	InKubernetes       bool

cmd/k8s-operator/manifests/proxy.yaml

@@ -27,8 +27,6 @@ spec:
         - name: tailscale
           imagePullPolicy: Always
           env:
-            - name: TS_USERSPACE
-              value: "false"
             - name: TS_AUTH_ONCE
               value: "true"
           securityContext:

cmd/k8s-operator/operator.go

@@ -235,9 +235,10 @@ const (
 
 	FinalizerName = "tailscale.com/finalizer"
 
-	AnnotationExpose   = "tailscale.com/expose"
-	AnnotationTags     = "tailscale.com/tags"
-	AnnotationHostname = "tailscale.com/hostname"
+	AnnotationExpose      = "tailscale.com/expose"
+	AnnotationTags        = "tailscale.com/tags"
+	AnnotationHostname    = "tailscale.com/hostname"
+	AnnotationFunnelPorts = "tailscale.com/funnel-tcp-portforward"
 )
 
 // ServiceReconciler is a simple ControllerManagedBy example implementation.
@@ -584,6 +585,21 @@ func (a *ServiceReconciler) reconcileSTS(ctx context.Context, logger *zap.Sugare
 			Name:  "TS_HOSTNAME",
 			Value: hostname,
 		})
+	if len(parentSvc.Annotations[AnnotationFunnelPorts]) > 0 {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_FUNNEL_TCP_PORTFORWARD",
+			Value: parentSvc.Annotations[AnnotationFunnelPorts],
+		})
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_USERSPACE",
+			Value: "true",
+		})
+	} else {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_USERSPACE",
+			Value: "false",
+		})
+	}
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,
 		Namespace: a.operatorNamespace,

ipn/ipnlocal/cert.go

@@ -32,6 +32,8 @@ import (
 
 	"golang.org/x/crypto/acme"
 	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
@@ -94,9 +96,9 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 		log.Printf("acme %T: %s", v, j)
 	}
 
-	if pair, ok := getCertPEMCached(dir, domain, now); ok {
+	if pair, ok := b.getCertPEMCached(dir, domain, now); ok {
 		future := now.AddDate(0, 0, 14)
-		if shouldStartDomainRenewal(dir, domain, future) {
+		if b.shouldStartDomainRenewal(dir, domain, future) {
 			logf("starting async renewal")
 			// Start renewal in the background.
 			go b.getCertPEM(context.Background(), logf, traceACME, dir, domain, future)
@@ -112,7 +114,7 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 	return pair, nil
 }
 
-func shouldStartDomainRenewal(dir, domain string, future time.Time) bool {
+func (b *LocalBackend) shouldStartDomainRenewal(dir, domain string, future time.Time) bool {
 	renewMu.Lock()
 	defer renewMu.Unlock()
 	now := time.Now()
@@ -122,7 +124,7 @@ func shouldStartDomainRenewal(dir, domain string, future time.Time) bool {
 		return false
 	}
 	lastRenewCheck[domain] = now
-	_, ok := getCertPEMCached(dir, domain, future)
+	_, ok := b.getCertPEMCached(dir, domain, future)
 	return !ok
 }
 
@@ -140,17 +142,36 @@ func certFile(dir, domain string) string { return filepath.Join(dir, domain+".cr
 // getCertPEMCached returns a non-nil keyPair and true if a cached
 // keypair for domain exists on disk in dir that is valid at the
 // provided now time.
-func getCertPEMCached(dir, domain string, now time.Time) (p *TLSCertKeyPair, ok bool) {
+func (b *LocalBackend) getCertPEMCached(dir, domain string, now time.Time) (p *TLSCertKeyPair, ok bool) {
 	if !validLookingCertDomain(domain) {
 		// Before we read files from disk using it, validate it's halfway
 		// reasonable looking.
 		return nil, false
 	}
-	if keyPEM, err := os.ReadFile(keyFile(dir, domain)); err == nil {
-		certPEM, _ := os.ReadFile(certFile(dir, domain))
-		if validCertPEM(domain, keyPEM, certPEM, now) {
-			return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, true
+	var keyPEM, certPEM []byte
+	var err error
+	if hostinfo.GetEnvType() == hostinfo.Kubernetes {
+		var err error
+		certPEM, err = b.store.ReadState(ipn.StateKey(domain + ".crt"))
+		if err != nil {
+			return nil, false
 		}
+		keyPEM, err = b.store.ReadState(ipn.StateKey(domain + ".key"))
+		if err != nil {
+			return nil, false
+		}
+	} else {
+		certPEM, err = os.ReadFile(keyFile(dir, domain))
+		if err != nil {
+			return nil, false
+		}
+		keyPEM, err = os.ReadFile(certFile(dir, domain))
+		if err != nil {
+			return nil, false
+		}
+	}
+	if validCertPEM(domain, keyPEM, certPEM, now) {
+		return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, true
 	}
 	return nil, false
 }
@@ -159,7 +180,7 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceAC
 	acmeMu.Lock()
 	defer acmeMu.Unlock()
 
-	if p, ok := getCertPEMCached(dir, domain, now); ok {
+	if p, ok := b.getCertPEMCached(dir, domain, now); ok {
 		return p, nil
 	}
 
@@ -274,8 +295,14 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceAC
 	if err := encodeECDSAKey(&privPEM, certPrivKey); err != nil {
 		return nil, err
 	}
-	if err := os.WriteFile(keyFile(dir, domain), privPEM.Bytes(), 0600); err != nil {
-		return nil, err
+	if hostinfo.GetEnvType() == hostinfo.Kubernetes {
+		if err := b.store.WriteState(ipn.StateKey(domain+".key"), privPEM.Bytes()); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := os.WriteFile(keyFile(dir, domain), privPEM.Bytes(), 0600); err != nil {
+			return nil, err
+		}
 	}
 
 	csr, err := certRequest(certPrivKey, domain, nil)
@@ -297,8 +324,14 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceAC
 			return nil, err
 		}
 	}
-	if err := os.WriteFile(certFile(dir, domain), certPEM.Bytes(), 0644); err != nil {
-		return nil, err
+	if hostinfo.GetEnvType() == hostinfo.Kubernetes {
+		if err := b.store.WriteState(ipn.StateKey(domain+".crt"), certPEM.Bytes()); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := os.WriteFile(certFile(dir, domain), certPEM.Bytes(), 0644); err != nil {
+			return nil, err
+		}
 	}
 
 	return &TLSCertKeyPair{CertPEM: certPEM.Bytes(), KeyPEM: privPEM.Bytes()}, nil

ipn/ipnlocal/profiles.go

@@ -185,7 +185,7 @@ func init() {
 func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView) error {
 	prefs := prefsIn.AsStruct().View()
 	newPersist := prefs.Persist().AsStruct()
-	if newPersist == nil || newPersist.LoginName == "" {
+	if newPersist == nil || newPersist.NodeID == "" {
 		return pm.setPrefsLocked(prefs)
 	}
 	up := newPersist.UserProfile

ipn/ipnlocal/profiles_test.go

@@ -5,6 +5,7 @@ package ipnlocal
 
 import (
 	"fmt"
+	"strconv"
 	"testing"
 
 	"tailscale.com/ipn"
@@ -337,6 +338,7 @@ func TestProfileManagementWindows(t *testing.T) {
 				ID:        id,
 				LoginName: loginName,
 			},
+			NodeID: tailcfg.StableNodeID(strconv.Itoa(int(id))),
 		}
 		if err := pm.SetPrefs(p.View()); err != nil {
 			t.Fatal(err)

ipn/ipnlocal/state_test.go

@@ -473,6 +473,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(3)
 	cc.persist.LoginName = "user1"
 	cc.persist.UserProfile.LoginName = "user1"
+	cc.persist.NodeID = "node1"
 	cc.send(nil, "", true, &netmap.NetworkMap{})
 	{
 		nn := notifies.drain(3)
@@ -699,6 +700,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(3)
 	cc.persist.LoginName = "user2"
 	cc.persist.UserProfile.LoginName = "user2"
+	cc.persist.NodeID = "node2"
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
@@ -835,6 +837,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
 	cc.persist.UserProfile.LoginName = "user3"
+	cc.persist.NodeID = "node3"
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})

ipn/store/kubestore/store_kube.go

@@ -7,6 +7,7 @@ package kubestore
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"tailscale.com/ipn"
@@ -36,6 +37,7 @@ func (s *Store) String() string { return "kube.Store" }
 
 // ReadState implements the StateStore interface.
 func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
+	id = ipn.StateKey(strings.ReplaceAll(string(id), "/", "__"))
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -55,6 +57,7 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
 
 // WriteState implements the StateStore interface.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
+	id = ipn.StateKey(strings.ReplaceAll(string(id), "/", "__"))
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()