Merge 67fd260317 into 92fb80d55f
Claire Wang
GitHub
commit
3c228fe5a9
|
|
@ -90,9 +90,16 @@ jobs:
|
|||
- name: build test wrapper
|
||||
run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
|
||||
- name: test all
|
||||
run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
|
||||
run: PATH=$PWD/tool:$PATH /tmp/testwrapper -json test_attempts.json ./... ${{matrix.buildflags}}
|
||||
env:
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
- name: upload test output
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-test-attempts.json
|
||||
path: test_attempts.json
|
||||
- name: remove upload test output
|
||||
run: rm test_attempts.json
|
||||
- name: bench all
|
||||
run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -28,18 +28,21 @@ import (
|
|||
|
||||
const maxAttempts = 3
|
||||
|
||||
// testAttempt keeps track of the test name, outcome, logs, and if the test is flakey.
|
||||
// After running the tests, each testAttempt is written to a json file.
|
||||
type testAttempt struct {
|
||||
name testName
|
||||
outcome string // "pass", "fail", "skip"
|
||||
Name testName `json:",inline"`
|
||||
Outcome string `json:",omitempty"` // "pass", "fail", "skip"
|
||||
logs bytes.Buffer
|
||||
isMarkedFlaky bool // set if the test is marked as flaky
|
||||
|
||||
pkgFinished bool
|
||||
IsMarkedFlaky bool `json:",omitempty"` // set if the test is marked as flaky
|
||||
AttemptCount int `json:",omitempty"`
|
||||
PkgFinished bool `json:",omitempty"`
|
||||
}
|
||||
|
||||
// testName keeps track of the test name and its package.
|
||||
type testName struct {
|
||||
pkg string // "tailscale.com/types/key"
|
||||
name string // "TestFoo"
|
||||
Pkg string `json:",omitempty"` // "tailscale.com/types/key"
|
||||
Name string `json:",omitempty"` // "TestFoo"
|
||||
}
|
||||
|
||||
type packageTests struct {
|
||||
|
|
@ -59,13 +62,17 @@ type goTestOutput struct {
|
|||
Output string
|
||||
}
|
||||
|
||||
type options struct {
|
||||
w io.Writer // optional writer if the -w flag is set.
|
||||
}
|
||||
|
||||
var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
|
||||
|
||||
// runTests runs the tests in pt and sends the results on ch. It sends a
|
||||
// testAttempt for each test and a final testAttempt per pkg with pkgFinished
|
||||
// testAttempt for each test and a final testAttempt per pkg with PkgFinished
|
||||
// set to true.
|
||||
// It calls close(ch) when it's done.
|
||||
func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt) {
|
||||
func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt, opt options) {
|
||||
defer close(ch)
|
||||
args := []string{"test", "-json", pt.pattern}
|
||||
args = append(args, otherArgs...)
|
||||
|
|
@ -110,21 +117,21 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
|
|||
switch goOutput.Action {
|
||||
case "fail", "pass", "skip":
|
||||
ch <- &testAttempt{
|
||||
name: testName{
|
||||
pkg: goOutput.Package,
|
||||
Name: testName{
|
||||
Pkg: goOutput.Package,
|
||||
},
|
||||
outcome: goOutput.Action,
|
||||
pkgFinished: true,
|
||||
Outcome: goOutput.Action,
|
||||
PkgFinished: true,
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
name := testName{
|
||||
pkg: goOutput.Package,
|
||||
name: goOutput.Test,
|
||||
Pkg: goOutput.Package,
|
||||
Name: goOutput.Test,
|
||||
}
|
||||
if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
|
||||
name.name = test
|
||||
name.Name = test
|
||||
if goOutput.Action == "output" {
|
||||
resultMap[name].logs.WriteString(goOutput.Output)
|
||||
}
|
||||
|
|
@ -135,33 +142,43 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
|
|||
// ignore
|
||||
case "run":
|
||||
resultMap[name] = &testAttempt{
|
||||
name: name,
|
||||
Name: name,
|
||||
AttemptCount: attempt,
|
||||
}
|
||||
case "skip", "pass", "fail":
|
||||
resultMap[name].outcome = goOutput.Action
|
||||
resultMap[name].Outcome = goOutput.Action
|
||||
ch <- resultMap[name]
|
||||
case "output":
|
||||
if strings.TrimSpace(goOutput.Output) == flakytest.FlakyTestLogMessage {
|
||||
resultMap[name].isMarkedFlaky = true
|
||||
resultMap[name].IsMarkedFlaky = true
|
||||
} else {
|
||||
resultMap[name].logs.WriteString(goOutput.Output)
|
||||
}
|
||||
}
|
||||
}
|
||||
if opt.w != nil {
|
||||
for _, result := range resultMap {
|
||||
testAttemptJson, _ := json.Marshal(result)
|
||||
_, err = opt.w.Write(testAttemptJson)
|
||||
if err != nil {
|
||||
log.Printf("error appending to test attempt json file: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
<-done
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
// We only need to parse the -v flag to figure out whether to print the logs
|
||||
// for a test. We don't need to parse any other flags, so we just use the
|
||||
// flag package to parse the -v flag and then pass the rest of the args
|
||||
// through to 'go test'.
|
||||
// We need to parse the -v flag to figure out whether to print the logs
|
||||
// for a test.
|
||||
// The -json flag is to indicate whether we want to write the json test results to a provided file.
|
||||
// We run `go test -json` which returns the same information as `go test -v`,
|
||||
// but in a machine-readable format. So this flag is only for testwrapper's
|
||||
// output.
|
||||
v := flag.Bool("v", false, "verbose")
|
||||
jsonFile := flag.String("json", "", "if set, the file to store all test results in")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Println("usage: testwrapper [testwrapper-flags] [pattern] [build/test flags & test binary flags]")
|
||||
|
|
@ -172,6 +189,7 @@ func main() {
|
|||
fmt.Println("examples:")
|
||||
fmt.Println("\ttestwrapper -v ./... -count=1")
|
||||
fmt.Println("\ttestwrapper ./pkg/foo -run TestBar -count=1")
|
||||
fmt.Println("\ttestwrapper -json=test_attempts.json ./pkg/foo -run TestBar -count=1")
|
||||
fmt.Println()
|
||||
fmt.Println("Unlike 'go test', testwrapper requires a package pattern as the first positional argument and only supports a single pattern.")
|
||||
}
|
||||
|
|
@ -217,7 +235,15 @@ func main() {
|
|||
}
|
||||
fmt.Printf("%s\t%s\n", outcome, pkg)
|
||||
}
|
||||
|
||||
opts := &options{}
|
||||
if *jsonFile != "" {
|
||||
f, err := os.Create(*jsonFile)
|
||||
if err != nil {
|
||||
log.Printf("error creating test attempt json file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
opts.w = f
|
||||
}
|
||||
for len(toRun) > 0 {
|
||||
var thisRun *nextRun
|
||||
thisRun, toRun = toRun[0], toRun[1:]
|
||||
|
|
@ -234,20 +260,20 @@ func main() {
|
|||
toRetry := make(map[string][]string) // pkg -> tests to retry
|
||||
for _, pt := range thisRun.tests {
|
||||
ch := make(chan *testAttempt)
|
||||
go runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
|
||||
go runTests(ctx, thisRun.attempt, pt, otherArgs, ch, *opts)
|
||||
for tr := range ch {
|
||||
if tr.pkgFinished {
|
||||
printPkgOutcome(tr.name.pkg, tr.outcome, thisRun.attempt)
|
||||
if tr.PkgFinished {
|
||||
printPkgOutcome(tr.Name.Pkg, tr.Outcome, thisRun.attempt)
|
||||
continue
|
||||
}
|
||||
if *v || tr.outcome == "fail" {
|
||||
if *v || tr.Outcome == "fail" {
|
||||
io.Copy(os.Stdout, &tr.logs)
|
||||
}
|
||||
if tr.outcome != "fail" {
|
||||
if tr.Outcome != "fail" {
|
||||
continue
|
||||
}
|
||||
if tr.isMarkedFlaky {
|
||||
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
|
||||
if tr.IsMarkedFlaky {
|
||||
toRetry[tr.Name.Pkg] = append(toRetry[tr.Name.Pkg], tr.Name.Name)
|
||||
} else {
|
||||
failed = true
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue