tstest: ignore certain goroutines in ResourceCheck

On Unix platforms, it's possible that the net package will launch
goroutines that we cannot terminate and could live for an indeterminate
time; the stack looks like this:

  1 @ 0x43ae6e 0x81f72b 0x822392 0x82189d 0x8224a7 0x4a5d41
  #	0x81f72a	net._C2func_getaddrinfo+0x8a	_cgo_gotypes.go:94
  #	0x822391	net.cgoLookupIPCNAME.func1+0xb1	/go/1.19.2/x64/src/net/cgo_unix.go:160
  #	0x82189c	net.cgoLookupIPCNAME+0x27c	/go/1.19.2/x64/src/net/cgo_unix.go:160
  #	0x8224a6	net.cgoIPLookup+0x66		/go/1.19.2/x64/src/net/cgo_unix.go:217

In tests that do DNS lookups, it's possible that these goroutines will
result in a test flake. Rather than try to carefully shut those down,
just ignore them entirely when counting goroutines.

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I25907e29d1a6b43a95002e1b55208cb965b9bfa4

shell.nix

@@ -51,5 +51,7 @@ in
 	    pkgs.gotools pkgs.gopls
 	    tailscale-go
 	    pkgs.graphviz
+	    pkgs.protobuf
+	    pkgs.protoc-gen-go
 	  ];
 	}

tstest/profilepb/generate.go

@@ -0,0 +1,3 @@
+//go:generate protoc --go_out=paths=source_relative:. --go_opt=Mprofile.proto=tailscale.com/tstest/profilepb profile.proto
+
+package profilepb

tstest/profilepb/profile.proto

@@ -0,0 +1,212 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Profile is a common stacktrace profile format.
+//
+// Measurements represented with this format should follow the
+// following conventions:
+//
+// - Consumers should treat unset optional fields as if they had been
+//   set with their default value.
+//
+// - When possible, measurements should be stored in "unsampled" form
+//   that is most useful to humans.  There should be enough
+//   information present to determine the original sampled values.
+//
+// - On-disk, the serialized proto must be gzip-compressed.
+//
+// - The profile is represented as a set of samples, where each sample
+//   references a sequence of locations, and where each location belongs
+//   to a mapping.
+// - There is a N->1 relationship from sample.location_id entries to
+//   locations. For every sample.location_id entry there must be a
+//   unique Location with that id.
+// - There is an optional N->1 relationship from locations to
+//   mappings. For every nonzero Location.mapping_id there must be a
+//   unique Mapping with that id.
+
+syntax = "proto3";
+
+package perftools.profiles;
+
+option java_package = "com.google.perftools.profiles";
+option java_outer_classname = "ProfileProto";
+
+message Profile {
+  // A description of the samples associated with each Sample.value.
+  // For a cpu profile this might be:
+  //   [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]]
+  // For a heap profile, this might be:
+  //   [["allocations","count"], ["space","bytes"]],
+  // If one of the values represents the number of events represented
+  // by the sample, by convention it should be at index 0 and use
+  // sample_type.unit == "count".
+  repeated ValueType sample_type = 1;
+  // The set of samples recorded in this profile.
+  repeated Sample sample = 2;
+  // Mapping from address ranges to the image/binary/library mapped
+  // into that address range.  mapping[0] will be the main binary.
+  repeated Mapping mapping = 3;
+  // Useful program location
+  repeated Location location = 4;
+  // Functions referenced by locations
+  repeated Function function = 5;
+  // A common table for strings referenced by various messages.
+  // string_table[0] must always be "".
+  repeated string string_table = 6;
+  // frames with Function.function_name fully matching the following
+  // regexp will be dropped from the samples, along with their successors.
+  int64 drop_frames = 7;   // Index into string table.
+  // frames with Function.function_name fully matching the following
+  // regexp will be kept, even if it matches drop_frames.
+  int64 keep_frames = 8;  // Index into string table.
+
+  // The following fields are informational, do not affect
+  // interpretation of results.
+
+  // Time of collection (UTC) represented as nanoseconds past the epoch.
+  int64 time_nanos = 9;
+  // Duration of the profile, if a duration makes sense.
+  int64 duration_nanos = 10;
+  // The kind of events between sampled ocurrences.
+  // e.g [ "cpu","cycles" ] or [ "heap","bytes" ]
+  ValueType period_type = 11;
+  // The number of events between sampled occurrences.
+  int64 period = 12;
+  // Freeform text associated to the profile.
+  repeated int64 comment = 13; // Indices into string table.
+  // Index into the string table of the type of the preferred sample
+  // value. If unset, clients should default to the last sample value.
+  int64 default_sample_type = 14;
+}
+
+// ValueType describes the semantics and measurement units of a value.
+message ValueType {
+  int64 type = 1; // Index into string table.
+  int64 unit = 2; // Index into string table.
+}
+
+// Each Sample records values encountered in some program
+// context. The program context is typically a stack trace, perhaps
+// augmented with auxiliary information like the thread-id, some
+// indicator of a higher level request being handled etc.
+message Sample {
+  // The ids recorded here correspond to a Profile.location.id.
+  // The leaf is at location_id[0].
+  repeated uint64 location_id = 1;
+  // The type and unit of each value is defined by the corresponding
+  // entry in Profile.sample_type. All samples must have the same
+  // number of values, the same as the length of Profile.sample_type.
+  // When aggregating multiple samples into a single sample, the
+  // result has a list of values that is the element-wise sum of the
+  // lists of the originals.
+  repeated int64 value = 2;
+  // label includes additional context for this sample. It can include
+  // things like a thread id, allocation size, etc
+  repeated Label label = 3;
+}
+
+message Label {
+  int64 key = 1;   // Index into string table
+
+  // At most one of the following must be present
+  int64 str = 2;   // Index into string table
+  int64 num = 3;
+
+  // Should only be present when num is present.
+  // Specifies the units of num.
+  // Use arbitrary string (for example, "requests") as a custom count unit.
+  // If no unit is specified, consumer may apply heuristic to deduce the unit.
+  // Consumers may also  interpret units like "bytes" and "kilobytes" as memory
+  // units and units like "seconds" and "nanoseconds" as time units,
+  // and apply appropriate unit conversions to these.
+  int64 num_unit = 4;  // Index into string table
+}
+
+message Mapping {
+  // Unique nonzero id for the mapping.
+  uint64 id = 1;
+  // Address at which the binary (or DLL) is loaded into memory.
+  uint64 memory_start = 2;
+  // The limit of the address range occupied by this mapping.
+  uint64 memory_limit = 3;
+  // Offset in the binary that corresponds to the first mapped address.
+  uint64 file_offset = 4;
+  // The object this entry is loaded from.  This can be a filename on
+  // disk for the main binary and shared libraries, or virtual
+  // abstractions like "[vdso]".
+  int64 filename = 5;  // Index into string table
+  // A string that uniquely identifies a particular program version
+  // with high probability. E.g., for binaries generated by GNU tools,
+  // it could be the contents of the .note.gnu.build-id field.
+  int64 build_id = 6;  // Index into string table
+
+  // The following fields indicate the resolution of symbolic info.
+  bool has_functions = 7;
+  bool has_filenames = 8;
+  bool has_line_numbers = 9;
+  bool has_inline_frames = 10;
+}
+
+// Describes function and line table debug information.
+message Location {
+  // Unique nonzero id for the location.  A profile could use
+  // instruction addresses or any integer sequence as ids.
+  uint64 id = 1;
+  // The id of the corresponding profile.Mapping for this location.
+  // It can be unset if the mapping is unknown or not applicable for
+  // this profile type.
+  uint64 mapping_id = 2;
+  // The instruction address for this location, if available.  It
+  // should be within [Mapping.memory_start...Mapping.memory_limit]
+  // for the corresponding mapping. A non-leaf address may be in the
+  // middle of a call instruction. It is up to display tools to find
+  // the beginning of the instruction if necessary.
+  uint64 address = 3;
+  // Multiple line indicates this location has inlined functions,
+  // where the last entry represents the caller into which the
+  // preceding entries were inlined.
+  //
+  // E.g., if memcpy() is inlined into printf:
+  //    line[0].function_name == "memcpy"
+  //    line[1].function_name == "printf"
+  repeated Line line = 4;
+  // Provides an indication that multiple symbols map to this location's
+  // address, for example due to identical code folding by the linker. In that
+  // case the line information above represents one of the multiple
+  // symbols. This field must be recomputed when the symbolization state of the
+  // profile changes.
+  bool is_folded = 5;
+}
+
+message Line {
+  // The id of the corresponding profile.Function for this line.
+  uint64 function_id = 1;
+  // Line number in source code.
+  int64 line = 2;
+}
+
+message Function {
+  // Unique nonzero id for the function.
+  uint64 id = 1;
+  // Name of the function, in human-readable form if available.
+  int64 name = 2; // Index into string table
+  // Name of the function, as identified by the system.
+  // For instance, it can be a C++ mangled name.
+  int64 system_name = 3; // Index into string table
+  // Source file containing the function.
+  int64 filename = 4; // Index into string table
+  // Line number in source file.
+  int64 start_line = 5;
+}

tstest/resource.go

@@ -6,12 +6,16 @@ package tstest
 
 import (
 	"bytes"
+	"compress/gzip"
+	"io"
 	"runtime"
 	"runtime/pprof"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/proto"
+	"tailscale.com/tstest/profilepb"
 )
 
 func ResourceCheck(tb testing.TB) {
@@ -29,12 +33,13 @@ func ResourceCheck(tb testing.TB) {
 			}
 			time.Sleep(5 * time.Millisecond)
 		}
-		endN, endStacks := goroutines()
-		if endN <= startN {
+		filteredGoroutines := numGoroutines(tb, pprof.Lookup("goroutine"))
+		if filteredGoroutines <= startN {
 			return
 		}
+		endN, endStacks := goroutines()
 		tb.Logf("goroutine diff:\n%v\n", cmp.Diff(startStacks, endStacks))
-		tb.Fatalf("goroutine count: expected %d, got %d\n", startN, endN)
+		tb.Fatalf("goroutine count: expected %d, got %d (%d filtered)\n", startN, endN, filteredGoroutines)
 	})
 }
 
@@ -44,3 +49,69 @@ func goroutines() (int, []byte) {
 	p.WriteTo(b, 1)
 	return p.Count(), b.Bytes()
 }
+
+var ignoredGoroutineStarts = map[string]bool{
+	"net.cgoIPLookup": true,
+}
+
+func numGoroutines(tb testing.TB, p *pprof.Profile) int {
+	b := new(bytes.Buffer)
+	p.WriteTo(b, 0) // gzip-compressed protobuf format
+
+	zr, err := gzip.NewReader(b)
+	if err != nil {
+		tb.Logf("error creating gzip.Reader: %v", err)
+		return -1
+	}
+	pb, err := io.ReadAll(zr)
+	if err != nil {
+		tb.Logf("error decompressing profile: %v", err)
+		return -1
+	}
+
+	var prof profilepb.Profile
+	if err := proto.Unmarshal(pb, &prof); err != nil {
+		tb.Logf("error parsing profile: %v", err)
+		return -1
+	}
+
+	var (
+		functions = make(map[uint64]*profilepb.Function)
+		locations = make(map[uint64]*profilepb.Location)
+	)
+	for _, f := range prof.Function {
+		functions[f.Id] = f
+	}
+	for _, m := range prof.Location {
+		locations[m.Id] = m
+	}
+
+	var num int64
+	for _, sample := range prof.Sample {
+		skip := false
+		for _, locid := range sample.LocationId {
+			loc := locations[locid]
+			for _, line := range loc.Line {
+				fn := functions[line.FunctionId]
+				fname := prof.StringTable[fn.Name]
+				if ignoredGoroutineStarts[fname] {
+					tb.Logf("skipping goroutine: %s", fname)
+					skip = true
+					break
+				}
+			}
+			if skip {
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+
+		for _, val := range sample.Value {
+			num += val
+		}
+	}
+
+	return int(num)
+}