From efb84ca60de3cd87df4b66acf7faf8fd08af3907 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 23 Sep 2021 09:20:14 -0700 Subject: [PATCH] ipn/localapi, cmd/tailscale: add CPU & memory profile support, debug command This was already possible on Linux if you ran tailscaled with --debug (which runs net/http/pprof), but it requires the user have the Go toolchain around. Also, it wasn't possible on macOS, as there's no way to run the IPNExtension with a debug server (it doesn't run tailscaled). And on Windows it's super tedious: beyond what users want to do or what we want to explain. Instead, put it in "tailscale debug" so it works and works the same on all platforms. Then we can ask users to run it when we're debugging something and they can email us the output files. Signed-off-by: Brad Fitzpatrick --- client/tailscale/tailscale.go | 12 ++++++++++++ cmd/tailscale/cli/debug.go | 28 ++++++++++++++++++++++++++++ cmd/tailscaled/depaware.txt | 2 +- ipn/localapi/localapi.go | 20 ++++++++++++++++++++ ipn/localapi/profile.go | 30 ++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 ipn/localapi/profile.go diff --git a/client/tailscale/tailscale.go b/client/tailscale/tailscale.go index d8b1a20fa..aa7e3762a 100644 --- a/client/tailscale/tailscale.go +++ b/client/tailscale/tailscale.go @@ -154,6 +154,18 @@ func Goroutines(ctx context.Context) ([]byte, error) { return get200(ctx, "/localapi/v0/goroutines") } +// Profile returns a pprof profile of the Tailscale daemon. +func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) { + var secArg string + if sec < 0 || sec > 300 { + return nil, errors.New("duration out of range") + } + if sec != 0 || pprofType == "profile" { + secArg = fmt.Sprint(sec) + } + return get200(ctx, fmt.Sprintf("/localapi/v0/profile?name=%s&seconds=%v", url.QueryEscape(pprofType), secArg)) +} + // BugReport logs and returns a log marker that can be shared by the user with support. func BugReport(ctx context.Context, note string) (string, error) { body, err := send(ctx, "POST", "/localapi/v0/bugreport?note="+url.QueryEscape(note), 200, nil) diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index f5cc32ef6..2cb927841 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -36,6 +36,9 @@ var debugCmd = &ffcli.Command{ fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode") fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled") fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME") + fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file") + fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file") + fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty") return fs })(), } @@ -49,6 +52,9 @@ var debugArgs struct { file string prefs bool pretty bool + cpuSec int + cpuFile string + memFile string } func runDebug(ctx context.Context, args []string) error { @@ -68,6 +74,28 @@ func runDebug(ctx context.Context, args []string) error { fmt.Printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket()) return nil } + if out := debugArgs.cpuFile; out != "" { + log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec) + if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil { + return err + } else { + if err := os.WriteFile(out, v, 0600); err != nil { + return err + } + log.Printf("CPU profile written to %s", out) + } + } + if out := debugArgs.memFile; out != "" { + log.Printf("Capturing memory profile ...") + if v, err := tailscale.Profile(ctx, "heap", 0); err != nil { + return err + } else { + if err := os.WriteFile(out, v, 0600); err != nil { + return err + } + log.Printf("Memory profile written to %s", out) + } + } if debugArgs.prefs { prefs, err := tailscale.GetPrefs(ctx) if err != nil { diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index c65a374fa..108e8e3a9 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -279,7 +279,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de net/http/httptrace from github.com/tcnksm/go-httpstat+ net/http/httputil from tailscale.com/ipn/localapi net/http/internal from net/http+ - net/http/pprof from tailscale.com/cmd/tailscaled + net/http/pprof from tailscale.com/cmd/tailscaled+ net/textproto from golang.org/x/net/http/httpguts+ net/url from crypto/x509+ os from crypto/rand+ diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index e424dc5d7..e00dd3a00 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -94,6 +94,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.serveWhoIs(w, r) case "/localapi/v0/goroutines": h.serveGoroutines(w, r) + case "/localapi/v0/profile": + h.serveProfile(w, r) case "/localapi/v0/status": h.serveStatus(w, r) case "/localapi/v0/logout": @@ -181,6 +183,24 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) { w.Write(buf) } +// serveProfileFunc is the implementation of Handler.serveProfile, after auth, +// for platforms where we want to link it in. +var serveProfileFunc func(http.ResponseWriter, *http.Request) + +func (h *Handler) serveProfile(w http.ResponseWriter, r *http.Request) { + // Require write access out of paranoia that the profile dump + // might contain something sensitive. + if !h.PermitWrite { + http.Error(w, "profile access denied", http.StatusForbidden) + return + } + if serveProfileFunc == nil { + http.Error(w, "not implemented on this platform", http.StatusServiceUnavailable) + return + } + serveProfileFunc(w, r) +} + func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) { if !h.PermitRead { http.Error(w, "IP forwarding check access denied", http.StatusForbidden) diff --git a/ipn/localapi/profile.go b/ipn/localapi/profile.go new file mode 100644 index 000000000..7780f7126 --- /dev/null +++ b/ipn/localapi/profile.go @@ -0,0 +1,30 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !ios && !android +// +build !ios,!android + +// We don't include it on mobile where we're more memory constrained and +// there's no CLI to get at the results anyway. + +package localapi + +import ( + "net/http" + "net/http/pprof" +) + +func init() { + serveProfileFunc = serveProfile +} + +func serveProfile(w http.ResponseWriter, r *http.Request) { + name := r.FormValue("name") + switch name { + case "profile": + pprof.Profile(w, r) + default: + pprof.Handler(name).ServeHTTP(w, r) + } +}