wip

Signed-off-by: Claire Wang <claire@tailscale.com>

.github/workflows/test.yml

@@ -90,9 +90,16 @@ jobs:
     - name: build test wrapper
       run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
     - name: test all
-      run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
+      run: PATH=$PWD/tool:$PATH /tmp/testwrapper -json test_attempts.json ./... ${{matrix.buildflags}}
       env:
         GOARCH: ${{ matrix.goarch }}
+    - name: upload test output
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-test-attempts.json 
+        path: test_attempts.json
+    - name: remove upload test output 
+      run: rm test_attempts.json
     - name: bench all
       run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ 
       env:

cmd/testwrapper/testwrapper.go

@@ -28,18 +28,21 @@ import (
 
 const maxAttempts = 3
 
+// testAttempt keeps track of the test name, outcome, logs, and if the test is flakey.
+// After running the tests, each testAttempt is written to a json file.
 type testAttempt struct {
-	name          testName
+	Name          testName `json:",inline"`
-	outcome       string // "pass", "fail", "skip"
+	Outcome       string   `json:",omitempty"` // "pass", "fail", "skip"
 	logs          bytes.Buffer
-	isMarkedFlaky bool // set if the test is marked as flaky
+	IsMarkedFlaky bool `json:",omitempty"` // set if the test is marked as flaky
-
+	AttemptCount  int  `json:",omitempty"`
-	pkgFinished bool
+	PkgFinished   bool `json:",omitempty"`
 }
 
+// testName keeps track of the test name and its package.
 type testName struct {
-	pkg  string // "tailscale.com/types/key"
+	Pkg  string `json:",omitempty"` // "tailscale.com/types/key"
-	name string // "TestFoo"
+	Name string `json:",omitempty"` // "TestFoo"
 }
 
 type packageTests struct {
@@ -59,13 +62,17 @@ type goTestOutput struct {
 	Output  string
 }
 
+type options struct {
+	w io.Writer // optional writer if the -w flag is set.
+}
+
 var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
 
 // runTests runs the tests in pt and sends the results on ch. It sends a
-// testAttempt for each test and a final testAttempt per pkg with pkgFinished
+// testAttempt for each test and a final testAttempt per pkg with PkgFinished
 // set to true.
 // It calls close(ch) when it's done.
-func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt) {
+func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt, opt options) {
 	defer close(ch)
 	args := []string{"test", "-json", pt.pattern}
 	args = append(args, otherArgs...)
@@ -110,21 +117,21 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
 			switch goOutput.Action {
 			case "fail", "pass", "skip":
 				ch <- &testAttempt{
-					name: testName{
+					Name: testName{
-						pkg: goOutput.Package,
+						Pkg: goOutput.Package,
 					},
-					outcome:     goOutput.Action,
+					Outcome:     goOutput.Action,
-					pkgFinished: true,
+					PkgFinished: true,
 				}
 			}
 			continue
 		}
 		name := testName{
-			pkg:  goOutput.Package,
+			Pkg:  goOutput.Package,
-			name: goOutput.Test,
+			Name: goOutput.Test,
 		}
 		if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
-			name.name = test
+			name.Name = test
 			if goOutput.Action == "output" {
 				resultMap[name].logs.WriteString(goOutput.Output)
 			}
@@ -135,33 +142,43 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
 			// ignore
 		case "run":
 			resultMap[name] = &testAttempt{
-				name: name,
+				Name:         name,
+				AttemptCount: attempt,
 			}
 		case "skip", "pass", "fail":
-			resultMap[name].outcome = goOutput.Action
+			resultMap[name].Outcome = goOutput.Action
 			ch <- resultMap[name]
 		case "output":
 			if strings.TrimSpace(goOutput.Output) == flakytest.FlakyTestLogMessage {
-				resultMap[name].isMarkedFlaky = true
+				resultMap[name].IsMarkedFlaky = true
 			} else {
 				resultMap[name].logs.WriteString(goOutput.Output)
 			}
 		}
 	}
+	if opt.w != nil {
+		for _, result := range resultMap {
+			testAttemptJson, _ := json.Marshal(result)
+			_, err = opt.w.Write(testAttemptJson)
+			if err != nil {
+				log.Printf("error appending to test attempt json file: %v", err)
+			}
+		}
+	}
 	<-done
 }
 
 func main() {
 	ctx := context.Background()
 
-	// We only need to parse the -v flag to figure out whether to print the logs
+	// We need to parse the -v flag to figure out whether to print the logs
-	// for a test. We don't need to parse any other flags, so we just use the
+	// for a test.
-	// flag package to parse the -v flag and then pass the rest of the args
+	// The -json flag is to indicate whether we want to write the json test results to a provided file.
-	// through to 'go test'.
 	// We run `go test -json` which returns the same information as `go test -v`,
 	// but in a machine-readable format. So this flag is only for testwrapper's
 	// output.
 	v := flag.Bool("v", false, "verbose")
+	jsonFile := flag.String("json", "", "if set, the file to store all test results in")
 
 	flag.Usage = func() {
 		fmt.Println("usage: testwrapper [testwrapper-flags] [pattern] [build/test flags & test binary flags]")
@@ -172,6 +189,7 @@ func main() {
 		fmt.Println("examples:")
 		fmt.Println("\ttestwrapper -v ./... -count=1")
 		fmt.Println("\ttestwrapper ./pkg/foo -run TestBar -count=1")
+		fmt.Println("\ttestwrapper -json=test_attempts.json ./pkg/foo -run TestBar -count=1")
 		fmt.Println()
 		fmt.Println("Unlike 'go test', testwrapper requires a package pattern as the first positional argument and only supports a single pattern.")
 	}
@@ -217,7 +235,15 @@ func main() {
 		}
 		fmt.Printf("%s\t%s\n", outcome, pkg)
 	}
-
+	opts := &options{}
+	if *jsonFile != "" {
+		f, err := os.Create(*jsonFile)
+		if err != nil {
+			log.Printf("error creating test attempt json file: %v", err)
+		}
+		defer f.Close()
+		opts.w = f
+	}
 	for len(toRun) > 0 {
 		var thisRun *nextRun
 		thisRun, toRun = toRun[0], toRun[1:]
@@ -234,20 +260,20 @@ func main() {
 		toRetry := make(map[string][]string) // pkg -> tests to retry
 		for _, pt := range thisRun.tests {
 			ch := make(chan *testAttempt)
-			go runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
+			go runTests(ctx, thisRun.attempt, pt, otherArgs, ch, *opts)
 			for tr := range ch {
-				if tr.pkgFinished {
+				if tr.PkgFinished {
-					printPkgOutcome(tr.name.pkg, tr.outcome, thisRun.attempt)
+					printPkgOutcome(tr.Name.Pkg, tr.Outcome, thisRun.attempt)
 					continue
 				}
-				if *v || tr.outcome == "fail" {
+				if *v || tr.Outcome == "fail" {
 					io.Copy(os.Stdout, &tr.logs)
 				}
-				if tr.outcome != "fail" {
+				if tr.Outcome != "fail" {
 					continue
 				}
-				if tr.isMarkedFlaky {
+				if tr.IsMarkedFlaky {
-					toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
+					toRetry[tr.Name.Pkg] = append(toRetry[tr.Name.Pkg], tr.Name.Name)
 				} else {
 					failed = true
 				}