update DNAT routing rules

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/containerboot/main.go

@@ -16,6 +16,8 @@
 //   - TS_ROUTES: subnet routes to advertise.
 //   - TS_DEST_IP: proxy all incoming Tailscale traffic to the given
 //     destination.
+//   - TS_EGRESS_IP: proxy all incoming non-Tailscale traffic to the given
+//     destination.
 //   - TS_TAILSCALED_EXTRA_ARGS: extra arguments to 'tailscaled'.
 //   - TS_EXTRA_ARGS: extra arguments to 'tailscale up'.
 //   - TS_USERSPACE: run with userspace networking (the default)
@@ -76,7 +78,8 @@ func main() {
 		AuthKey:         defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
 		Hostname:        defaultEnv("TS_HOSTNAME", ""),
 		Routes:          defaultEnv("TS_ROUTES", ""),
-		ProxyTo:         defaultEnv("TS_DEST_IP", ""),
+		IngressProxyTo:  defaultEnv("TS_DEST_IP", ""),
+		EgressProxyTo:   defaultEnv("TS_EGRESS_IP", ""),
 		DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
 		ExtraArgs:       defaultEnv("TS_EXTRA_ARGS", ""),
 		InKubernetes:    os.Getenv("KUBERNETES_SERVICE_HOST") != "",
@@ -91,7 +94,7 @@ func main() {
 		Root:            defaultEnv("TS_TEST_ONLY_ROOT", "/"),
 	}
 
-	if cfg.ProxyTo != "" && cfg.UserspaceMode {
+	if cfg.IngressProxyTo != "" && cfg.UserspaceMode {
 		log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE")
 	}
 
@@ -99,8 +102,8 @@ func main() {
 		if err := ensureTunFile(cfg.Root); err != nil {
 			log.Fatalf("Unable to create tuntap device file: %v", err)
 		}
-		if cfg.ProxyTo != "" || cfg.Routes != "" {
+		if cfg.IngressProxyTo != "" || cfg.Routes != "" {
-			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.Routes); err != nil {
+			if err := ensureIPForwarding(cfg.Root, cfg.IngressProxyTo, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
@@ -240,11 +243,13 @@ authLoop:
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != ""
+		wantProxy         = cfg.IngressProxyTo != "" || cfg.EgressProxyTo != ""
 		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
 		startupTasksDone  = false
 		currentIPs        deephash.Sum // tailscale IPs assigned to device
 		currentDeviceInfo deephash.Sum // device ID and fqdn
+
+		installedEgressProxy = false
 	)
 	for {
 		n, err := w.Next()
@@ -261,11 +266,17 @@ authLoop:
 			log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
 		}
 		if n.NetMap != nil {
-			if cfg.ProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
+			if cfg.IngressProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
-				if err := installIPTablesRule(ctx, cfg.ProxyTo, n.NetMap.Addresses); err != nil {
+				if err := installIngressForwardingRule(ctx, cfg.IngressProxyTo, n.NetMap.Addresses); err != nil {
 					log.Fatalf("installing proxy rules: %v", err)
 				}
 			}
+			if cfg.EgressProxyTo != "" && !installedEgressProxy {
+				if err := installEgressForwardingRule(ctx, cfg.EgressProxyTo); err != nil {
+					log.Fatalf("installing proxy rules: %v", err)
+				}
+				installedEgressProxy = true
+			}
 			deviceInfo := []any{n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name}
 			if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
 				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name); err != nil {
@@ -492,7 +503,28 @@ func ensureIPForwarding(root, proxyTo, routes string) error {
 	return nil
 }
 
-func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
+func installEgressForwardingRule(ctx context.Context, dstStr string) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	argv0 := "iptables"
+	if dst.Is6() {
+		argv0 = "ip6tables"
+	}
+	// Technically, if the control server ever changes the IPs assigned to this
+	// node, we'll slowly accumulate iptables rules. This shouldn't happen, so
+	// for now we'll live with it.
+	cmd := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "PREROUTING", "1", "!", "-i", "tailscale0", "-j", "DNAT", "--to-destination", dstStr)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("executing iptables failed: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
 		return err
@@ -529,10 +561,20 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefi
 
 // settings is all the configuration for containerboot.
 type settings struct {
-	AuthKey            string
+	AuthKey  string
-	Hostname           string
+	Hostname string
-	Routes             string
+	Routes   string
-	ProxyTo            string
+
+	// IngressProxyTo is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	IngressProxyTo string
+
+	// EgressProxyTo is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. If empty, no
+	// proxying is done. This is typically a Tailscale IP.
+	EgressProxyTo string
+
 	DaemonExtraArgs    string
 	ExtraArgs          string
 	InKubernetes       bool

cmd/k8s-operator/operator.go

@@ -274,6 +274,8 @@ const (
 	AnnotationExpose   = "tailscale.com/expose"
 	AnnotationTags     = "tailscale.com/tags"
 	AnnotationHostname = "tailscale.com/hostname"
+
+	AnnotationTargetIP = "tailscale.com/target-ip"
 )
 
 // ServiceReconciler is a simple ControllerManagedBy example implementation.
@@ -321,7 +323,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) {
+	if !svc.DeletionTimestamp.IsZero() || (!a.shouldExpose(svc) && !a.hasTargetAnnotation(svc)) {
 		logger.Debugf("service is being deleted or should not be exposed, cleaning up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
@@ -402,6 +404,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	return nil
 }
 
+// TODO(maisem): XXXXXXXXXXXXXXXXXXX update docs
 // maybeProvision ensures that svc is exposed over tailscale, taking any actions
 // necessary to reach that state.
 //
@@ -444,6 +447,19 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to reconcile statefulset: %w", err)
 	}
 
+	if a.hasTargetAnnotation(svc) {
+		headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc"
+		if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
+			svc.Spec.ExternalName = headlessSvcName
+			svc.Spec.Selector = nil
+			svc.Spec.Type = corev1.ServiceTypeExternalName
+			if err := a.Update(ctx, svc); err != nil {
+				return fmt.Errorf("failed to update service: %w", err)
+			}
+		}
+		return nil
+	}
+
 	if !a.hasLoadBalancerClass(svc) {
 		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
 		return nil
@@ -482,7 +498,7 @@ func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
 		return false
 	}
 
-	return a.hasLoadBalancerClass(svc) || a.hasAnnotation(svc)
+	return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
 }
 
 func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
@@ -492,9 +508,12 @@ func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
 		*svc.Spec.LoadBalancerClass == "tailscale"
 }
 
-func (a *ServiceReconciler) hasAnnotation(svc *corev1.Service) bool {
+func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
-	return svc != nil &&
+	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
-		svc.Annotations[AnnotationExpose] == "true"
+}
+
+func (a *ServiceReconciler) hasTargetAnnotation(svc *corev1.Service) bool {
+	return svc != nil && svc.Annotations[AnnotationTargetIP] != ""
 }
 
 func (a *ServiceReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (*corev1.Service, error) {
@@ -612,11 +631,22 @@ func (a *ServiceReconciler) reconcileSTS(ctx context.Context, logger *zap.Sugare
 	}
 	container := &ss.Spec.Template.Spec.Containers[0]
 	container.Image = a.proxyImage
+	if ip := parentSvc.Annotations[AnnotationTargetIP]; ip != "" {
+		container.Env = append(container.Env,
+			corev1.EnvVar{
+				Name:  "TS_EGRESS_IP",
+				Value: ip,
+			},
+		)
+	} else {
+		container.Env = append(container.Env,
+			corev1.EnvVar{
+				Name:  "TS_DEST_IP",
+				Value: parentSvc.Spec.ClusterIP,
+			},
+		)
+	}
 	container.Env = append(container.Env,
-		corev1.EnvVar{
-			Name:  "TS_DEST_IP",
-			Value: parentSvc.Spec.ClusterIP,
-		},
 		corev1.EnvVar{
 			Name:  "TS_KUBE_SECRET",
 			Value: authKeySecret,