cmd/k8s-operator: WIP add support for egress proxy

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/containerboot/main.go

@@ -16,6 +16,8 @@
 //   - TS_ROUTES: subnet routes to advertise.
 //   - TS_DEST_IP: proxy all incoming Tailscale traffic to the given
 //     destination.
+//   - TS_EGRESS_IP: proxy all incoming non-Tailscale traffic to the given
+//     destination.
 //   - TS_TAILSCALED_EXTRA_ARGS: extra arguments to 'tailscaled'.
 //   - TS_EXTRA_ARGS: extra arguments to 'tailscale up'.
 //   - TS_USERSPACE: run with userspace networking (the default)
@@ -76,7 +78,8 @@ func main() {
 		AuthKey:         defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
 		Hostname:        defaultEnv("TS_HOSTNAME", ""),
 		Routes:          defaultEnv("TS_ROUTES", ""),
-		ProxyTo:         defaultEnv("TS_DEST_IP", ""),
+		IngressProxyTo:  defaultEnv("TS_DEST_IP", ""),
+		EgressProxyTo:   defaultEnv("TS_EGRESS_IP", ""),
 		DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
 		ExtraArgs:       defaultEnv("TS_EXTRA_ARGS", ""),
 		InKubernetes:    os.Getenv("KUBERNETES_SERVICE_HOST") != "",
@@ -91,7 +94,7 @@ func main() {
 		Root:            defaultEnv("TS_TEST_ONLY_ROOT", "/"),
 	}
 
-	if cfg.ProxyTo != "" && cfg.UserspaceMode {
+	if cfg.IngressProxyTo != "" && cfg.UserspaceMode {
 		log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE")
 	}
 
@@ -99,8 +102,8 @@ func main() {
 		if err := ensureTunFile(cfg.Root); err != nil {
 			log.Fatalf("Unable to create tuntap device file: %v", err)
 		}
-		if cfg.ProxyTo != "" || cfg.Routes != "" {
+		if cfg.IngressProxyTo != "" || cfg.Routes != "" {
-			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.Routes); err != nil {
+			if err := ensureIPForwarding(cfg.Root, cfg.IngressProxyTo, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
@@ -240,11 +243,13 @@ authLoop:
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != ""
+		wantProxy         = cfg.IngressProxyTo != "" || cfg.EgressProxyTo != ""
 		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
 		startupTasksDone  = false
 		currentIPs        deephash.Sum // tailscale IPs assigned to device
 		currentDeviceInfo deephash.Sum // device ID and fqdn
+
+		installedEgressProxy = false
 	)
 	for {
 		n, err := w.Next()
@@ -261,11 +266,17 @@ authLoop:
 			log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
 		}
 		if n.NetMap != nil {
-			if cfg.ProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
+			if cfg.IngressProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
-				if err := installIPTablesRule(ctx, cfg.ProxyTo, n.NetMap.Addresses); err != nil {
+				if err := installIngressForwardingRule(ctx, cfg.IngressProxyTo, n.NetMap.Addresses); err != nil {
 					log.Fatalf("installing proxy rules: %v", err)
 				}
 			}
+			if cfg.EgressProxyTo != "" && !installedEgressProxy {
+				if err := installEgressForwardingRule(ctx, cfg.EgressProxyTo); err != nil {
+					log.Fatalf("installing proxy rules: %v", err)
+				}
+				installedEgressProxy = true
+			}
 			deviceInfo := []any{n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name}
 			if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
 				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name); err != nil {
@@ -492,7 +503,28 @@ func ensureIPForwarding(root, proxyTo, routes string) error {
 	return nil
 }
 
-func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
+func installEgressForwardingRule(ctx context.Context, dstStr string) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	argv0 := "iptables"
+	if dst.Is6() {
+		argv0 = "ip6tables"
+	}
+	// Technically, if the control server ever changes the IPs assigned to this
+	// node, we'll slowly accumulate iptables rules. This shouldn't happen, so
+	// for now we'll live with it.
+	cmd := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "PREROUTING", "1", "-i", "!tailscale0", "-j", "DNAT", "--to-destination", dstStr)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("executing iptables failed: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
 		return err
@@ -532,7 +564,17 @@ type settings struct {
 	AuthKey  string
 	Hostname string
 	Routes   string
-	ProxyTo            string
+
+	// IngressProxyTo is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	IngressProxyTo string
+
+	// EgressProxyTo is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. If empty, no
+	// proxying is done. This is typically a Tailscale IP.
+	EgressProxyTo string
+
 	DaemonExtraArgs    string
 	ExtraArgs          string
 	InKubernetes       bool

cmd/k8s-operator/operator.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"crypto/tls"
 	_ "embed"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -274,6 +275,8 @@ const (
 	AnnotationExpose   = "tailscale.com/expose"
 	AnnotationTags     = "tailscale.com/tags"
 	AnnotationHostname = "tailscale.com/hostname"
+
+	AnnotationTarget = "tailscale.com/target"
 )
 
 // ServiceReconciler is a simple ControllerManagedBy example implementation.
@@ -321,12 +324,16 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) {
+	if !svc.DeletionTimestamp.IsZero() || (!a.shouldExpose(svc) && !a.hasTargetAnnotation(svc)) {
 		logger.Debugf("service is being deleted or should not be exposed, cleaning up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
 
-	return reconcile.Result{}, a.maybeProvision(ctx, logger, svc)
+	if a.hasTargetAnnotation(svc) {
+		return reconcile.Result{}, a.maybeProvisionEgress(ctx, logger, svc)
+	}
+
+	return reconcile.Result{}, a.maybeProvisionIngress(ctx, logger, svc)
 }
 
 // maybeCleanup removes any existing resources related to serving svc over tailscale.
@@ -402,12 +409,16 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	return nil
 }
 
-// maybeProvision ensures that svc is exposed over tailscale, taking any actions
+func (a *ServiceReconciler) maybeProvisionEgress(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+	return errors.New("unimplemented")
+}
+
+// maybeProvisionIngress ensures that svc is exposed over tailscale, taking any actions
 // necessary to reach that state.
 //
 // This function adds a finalizer to svc, ensuring that we can handle orderly
 // deprovisioning later.
-func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+func (a *ServiceReconciler) maybeProvisionIngress(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
 	hostname, err := nameForService(svc)
 	if err != nil {
 		return err
@@ -482,7 +493,7 @@ func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
 		return false
 	}
 
-	return a.hasLoadBalancerClass(svc) || a.hasAnnotation(svc)
+	return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
 }
 
 func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
@@ -492,9 +503,12 @@ func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
 		*svc.Spec.LoadBalancerClass == "tailscale"
 }
 
-func (a *ServiceReconciler) hasAnnotation(svc *corev1.Service) bool {
+func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
-	return svc != nil &&
+	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
-		svc.Annotations[AnnotationExpose] == "true"
+}
+
+func (a *ServiceReconciler) hasTargetAnnotation(svc *corev1.Service) bool {
+	return svc != nil && svc.Annotations[AnnotationTarget] != ""
 }
 
 func (a *ServiceReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (*corev1.Service, error) {