net/dnsfallback: run recursive resolver and compare results

When performing a fallback DNS query, run the recursive resolver in a
separate goroutine and compare the results returned by the recursive
resolver with the results we get from "regular" bootstrap DNS. This will
allow us to gather data about whether the recursive DNS resolver works
better, worse, or about the same as "regular" bootstrap DNS.

Updates #5853

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: Ifa0b0cc9eeb0dccd6f7a3d91675fe44b3b34bd48

cmd/tailscale/depaware.txt

@@ -24,6 +24,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L 💣 github.com/mdlayher/netlink                                  from github.com/jsimonetti/rtnetlink+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+        github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli+
         github.com/peterbourgon/ff/v3                                from github.com/peterbourgon/ff/v3/ffcli
         github.com/peterbourgon/ff/v3/ffcli                          from tailscale.com/cmd/tailscale/cli
@@ -68,6 +69,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/metrics                                        from tailscale.com/derp
+        tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp+
         tailscale.com/net/dnsfallback                                from tailscale.com/control/controlhttp
         tailscale.com/net/flowtrack                                  from tailscale.com/wgengine/filter+
@@ -145,7 +147,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from golang.org/x/exp/slices
+        golang.org/x/exp/constraints                                 from golang.org/x/exp/slices+
         golang.org/x/exp/slices                                      from tailscale.com/net/tsaddr+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
@@ -207,7 +209,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         embed                                                        from tailscale.com/cmd/tailscale/cli+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
-        encoding/base32                                              from tailscale.com/tka
+        encoding/base32                                              from tailscale.com/tka+
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+

cmd/tailscaled/depaware.txt

@@ -111,6 +111,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
+        github.com/miekg/dns                                         from tailscale.com/net/dns/recursive
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/safesocket
    L    github.com/pierrec/lz4/v4                                    from github.com/u-root/uio/uio
    L    github.com/pierrec/lz4/v4/internal/lz4block                  from github.com/pierrec/lz4/v4+
@@ -242,6 +243,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dns/publicdns                              from tailscale.com/net/dns/resolver+
+        tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
         tailscale.com/net/dns/resolvconffile                         from tailscale.com/net/dns+
         tailscale.com/net/dns/resolver                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
@@ -428,7 +430,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         embed                                                        from tailscale.com+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
-        encoding/base32                                              from tailscale.com/tka
+        encoding/base32                                              from tailscale.com/tka+
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+

net/dnsfallback/dnsfallback.go

@@ -22,22 +22,72 @@ import (
 	"sync/atomic"
 	"time"
 
+	"golang.org/x/exp/slices"
 	"tailscale.com/atomicfile"
+	"tailscale.com/envknob"
+	"tailscale.com/net/dns/recursive"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/slicesx"
 )
 
+var disableRecursiveResolver = envknob.RegisterBool("TS_DNSFALLBACK_DISABLE_RECURSIVE_RESOLVER")
+
 // MakeLookupFunc creates a function that can be used to resolve hostnames
 // (e.g. as a LookupIPFallback from dnscache.Resolver).
 // The netMon parameter is optional; if non-nil it's used to do faster interface lookups.
 func MakeLookupFunc(logf logger.Logf, netMon *netmon.Monitor) func(ctx context.Context, host string) ([]netip.Addr, error) {
 	return func(ctx context.Context, host string) ([]netip.Addr, error) {
-		return lookup(ctx, host, logf, netMon)
+		if disableRecursiveResolver() {
+			return lookup(ctx, host, logf, netMon)
+		}
+
+		addrsCh := make(chan []netip.Addr, 1)
+
+		// Run the recursive resolver in the background so we can
+		// compare the results.
+		go func() {
+			logf := logger.WithPrefix(logf, "recursive: ")
+			resolver := recursive.Resolver{
+				Dialer: netns.NewDialer(logf, netMon),
+				Logf:   logf,
+			}
+			addrs, minTTL, err := resolver.Resolve(ctx, host)
+			if err != nil {
+				logf("error using recursive resolver: %v", err)
+				metricRecursiveErrors.Add(1)
+				return
+			}
+			slices.SortFunc(addrs, func(a, b netip.Addr) bool { return a.Less(b) })
+
+			// Wait for a response from the main function
+			oldAddrs := <-addrsCh
+			slices.SortFunc(oldAddrs, func(a, b netip.Addr) bool { return a.Less(b) })
+
+			matches := slices.Equal(addrs, oldAddrs)
+
+			logf("bootstrap DNS comparison: matches=%v oldAddrs=%v addrs=%v minTTL=%v", matches, oldAddrs, addrs, minTTL)
+
+			if matches {
+				metricRecursiveMatches.Add(1)
+			} else {
+				metricRecursiveMismatches.Add(1)
+			}
+		}()
+
+		addrs, err := lookup(ctx, host, logf, netMon)
+		if err != nil {
+			addrsCh <- nil
+			return nil, err
+		}
+
+		addrsCh <- slices.Clone(addrs)
+		return addrs, nil
 	}
 }
 
@@ -254,3 +304,9 @@ func SetCachePath(path string, logf logger.Logf) {
 	cachedDERPMap.Store(dm)
 	logf("[v2] dnsfallback: SetCachePath loaded cached DERP map")
 }
+
+var (
+	metricRecursiveMatches    = clientmetric.NewCounter("dnsfallback_recursive_matches")
+	metricRecursiveMismatches = clientmetric.NewCounter("dnsfallback_recursive_mismatches")
+	metricRecursiveErrors     = clientmetric.NewCounter("dnsfallback_recursive_errors")
+)