diff --git a/cmd/main.go b/cmd/main.go index f55064ed..069f41c1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -161,6 +161,7 @@ Formats: testname print a line for each test and package testdox print a sentence for each test using gotestdox github-actions testname format with github actions log grouping + tap TAP format (Test Anything Protocol) standard-quiet standard go test format standard-verbose standard go test -v format @@ -336,7 +337,10 @@ func run(opts *options) error { } func finishRun(opts *options, exec *testjson.Execution, exitErr error) error { - testjson.PrintSummary(opts.stdout, exec, opts.hideSummary.value) + // TAP format handles its own output, skip the standard summary + if opts.format != "tap" { + testjson.PrintSummary(opts.stdout, exec, opts.hideSummary.value) + } if err := writeJUnitFile(opts, exec); err != nil { return fmt.Errorf("failed to write junit file: %w", err) diff --git a/cmd/testdata/gotestsum-help-text b/cmd/testdata/gotestsum-help-text index c2283b43..3f4f54dc 100644 --- a/cmd/testdata/gotestsum-help-text +++ b/cmd/testdata/gotestsum-help-text @@ -41,6 +41,7 @@ Formats: testname print a line for each test and package testdox print a sentence for each test using gotestdox github-actions testname format with github actions log grouping + tap TAP format (Test Anything Protocol) standard-quiet standard go test format standard-verbose standard go test -v format diff --git a/testjson/format.go b/testjson/format.go index 64b2f261..1e891032 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -449,6 +449,8 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E return pkgNameWithFailuresFormat(out, formatOpts) case "github-actions", "github-action": return githubActionsFormat(out) + case "tap": + return tapFormat(out) default: return nil } @@ -513,3 +515,116 @@ func githubActionsFormat(out io.Writer) EventFormatter { return buf.Flush() }) } + +// tapFormat returns a TAP (Test Anything Protocol) format EventFormatter. +func tapFormat(out io.Writer) EventFormatter { + return &tapFormatter{ + out: out, + testNum: 0, + output: make(map[string][]string), + started: false, + } +} + +type tapFormatter struct { + out io.Writer + testNum int + output map[string][]string + started bool +} + +func (t *tapFormatter) Format(event TestEvent, exec *Execution) error { + // Write TAP version header on first output + if !t.started { + t.started = true + fmt.Fprintln(t.out, "TAP version 13") + fmt.Fprintln(t.out, "1..N") + } + + // Handle package events (build failures, etc.) + if event.PackageEvent() { + return t.formatPackageEvent(event, exec) + } + + // Buffer output for test cases + if event.Action == ActionOutput { + key := event.Package + "::" + event.Test + t.output[key] = append(t.output[key], event.Output) + return nil + } + + // Handle test completion + if event.Action.IsTerminal() { + t.formatTestEnd(event, exec) + } + + return nil +} + +func (t *tapFormatter) formatPackageEvent(event TestEvent, exec *Execution) error { + // Handle build failures or package-level failures + if event.Action == ActionFail { + pkg := exec.Package(event.Package) + if pkg.TestMainFailed() || len(pkg.Failed) == 0 { + // Build failure or TestMain failure - emit Bail out! + fmt.Fprintf(t.out, "Bail out! %s\n", event.Package) + // Write package output as diagnostics + var buf strings.Builder + pkg.WriteOutputTo(&buf, 0) + for _, line := range strings.Split(buf.String(), "\n") { + if line != "" { + fmt.Fprintf(t.out, "# %s\n", line) + } + } + } + } + return nil +} + +func (t *tapFormatter) formatTestEnd(event TestEvent, exec *Execution) error { + t.testNum++ + key := event.Package + "::" + event.Test + + // Build test line + var status string + switch event.Action { + case ActionPass: + status = "ok" + case ActionFail: + status = "not ok" + case ActionSkip: + status = "ok" + } + + // Format: "ok N - package.TestName" or "not ok N - package.TestName" + desc := fmt.Sprintf("%s.%s", event.Package, event.Test) + line := fmt.Sprintf("%s %d - %s", status, t.testNum, desc) + + // Add SKIP directive + if event.Action == ActionSkip { + line += " # SKIP" + } + + // Add elapsed time as comment + if event.Elapsed > 0 { + line += fmt.Sprintf(" # time=%.3fs", event.Elapsed) + } + + fmt.Fprintln(t.out, line) + + // Emit buffered output as diagnostics only for failed tests + if event.Action == ActionFail { + if output, ok := t.output[key]; ok { + for _, out := range output { + for _, l := range strings.Split(out, "\n") { + if l != "" { + fmt.Fprintf(t.out, "# %s\n", l) + } + } + } + } + } + delete(t.output, key) + + return nil +} diff --git a/testjson/tapformat_test.go b/testjson/tapformat_test.go new file mode 100644 index 00000000..8475bf12 --- /dev/null +++ b/testjson/tapformat_test.go @@ -0,0 +1,160 @@ +package testjson + +import ( + "bytes" + "testing" + + "gotest.tools/v3/assert" + "gotest.tools/v3/golden" +) + +func TestTapFormat_VersionLine(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + err := formatter.Format(TestEvent{ + Action: ActionPass, + Package: "github.com/example/pkg", + Test: "TestOne", + Elapsed: 0.01, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("TAP version 13")), "expected TAP version line") +} + +func TestTapFormat_TestLine(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + err := formatter.Format(TestEvent{ + Action: ActionPass, + Package: "github.com/example/pkg", + Test: "TestExample", + Elapsed: 0.05, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("ok 1 - github.com/example/pkg.TestExample")), "expected test line") + assert.Assert(t, bytes.Contains([]byte(got), []byte("time=")), "expected time comment") +} + +func TestTapFormat_SkipLine(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + err := formatter.Format(TestEvent{ + Action: ActionSkip, + Package: "github.com/example/pkg", + Test: "TestSkipped", + Elapsed: 0.001, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("ok 1 - github.com/example/pkg.TestSkipped")), "expected ok line for skip") + assert.Assert(t, bytes.Contains([]byte(got), []byte("# SKIP")), "expected SKIP directive") +} + +func TestTapFormat_FailLine(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + err := formatter.Format(TestEvent{ + Action: ActionFail, + Package: "github.com/example/pkg", + Test: "TestFailed", + Elapsed: 0.1, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("not ok 1 - github.com/example/pkg.TestFailed")), "expected not ok line for fail") +} + +func TestTapFormat_WithOutput(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + + // First, send output event + err := formatter.Format(TestEvent{ + Package: "github.com/example/pkg", + Test: "TestWithOutput", + Action: ActionOutput, + Output: "debug message\n", + }, exec) + assert.NilError(t, err) + + // Then, send fail event + err = formatter.Format(TestEvent{ + Package: "github.com/example/pkg", + Test: "TestWithOutput", + Action: ActionFail, + Elapsed: 0.05, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("# debug message")), "expected output as diagnostic") +} + +func TestTapFormat_MultipleTests(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + exec := newExecution() + + // Test 1 + err := formatter.Format(TestEvent{ + Action: ActionPass, + Package: "github.com/example/pkg", + Test: "TestOne", + Elapsed: 0.01, + }, exec) + assert.NilError(t, err) + + // Test 2 + err = formatter.Format(TestEvent{ + Action: ActionPass, + Package: "github.com/example/pkg", + Test: "TestTwo", + Elapsed: 0.02, + }, exec) + assert.NilError(t, err) + + // Test 3 + err = formatter.Format(TestEvent{ + Action: ActionFail, + Package: "github.com/example/pkg", + Test: "TestThree", + Elapsed: 0.03, + }, exec) + assert.NilError(t, err) + + got := out.String() + assert.Assert(t, bytes.Contains([]byte(got), []byte("ok 1 - github.com/example/pkg.TestOne")), "expected test 1") + assert.Assert(t, bytes.Contains([]byte(got), []byte("ok 2 - github.com/example/pkg.TestTwo")), "expected test 2") + assert.Assert(t, bytes.Contains([]byte(got), []byte("not ok 3 - github.com/example/pkg.TestThree")), "expected test 3") +} + +func TestTapFormat_Golden(t *testing.T) { + out := new(bytes.Buffer) + formatter := tapFormat(out) + + shim := newFakeHandler(formatter, "input/go-test-json-tap-sample") + exec, err := ScanTestOutput(shim.Config(t)) + assert.NilError(t, err) + + golden.Assert(t, out.String(), "tapformat-golden.tap") + assert.Equal(t, len(exec.Failed()), 1) + assert.Equal(t, len(exec.Skipped()), 1) + assert.Assert(t, exec.Total() >= 3) +} diff --git a/testjson/testdata/input/go-test-json-tap-sample.err b/testjson/testdata/input/go-test-json-tap-sample.err new file mode 100644 index 00000000..e69de29b diff --git a/testjson/testdata/input/go-test-json-tap-sample.out b/testjson/testdata/input/go-test-json-tap-sample.out new file mode 100644 index 00000000..c192a047 --- /dev/null +++ b/testjson/testdata/input/go-test-json-tap-sample.out @@ -0,0 +1,21 @@ +{"Time":"2022-06-19T13:44:44.859682146-04:00","Action":"run","Package":"github.com/example/pkg","Test":"TestPass"} +{"Time":"2022-06-19T13:44:44.859690198-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestPass","Output":"=== RUN TestPass\n"} +{"Time":"2022-06-19T13:44:44.859696077-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestPass","Output":"--- PASS: TestPass (0.00s)\n"} +{"Time":"2022-06-19T13:44:44.859699224-04:00","Action":"pass","Package":"github.com/example/pkg","Test":"TestPass","Elapsed":0.01} +{"Time":"2022-06-19T13:44:44.859701901-04:00","Action":"run","Package":"github.com/example/pkg","Test":"TestFail"} +{"Time":"2022-06-19T13:44:44.859704199-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestFail","Output":"=== RUN TestFail\n"} +{"Time":"2022-06-19T13:44:44.859706845-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestFail","Output":" example_test.go:42: assertion failed\n"} +{"Time":"2022-06-19T13:44:44.859709737-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestFail","Output":"--- FAIL: TestFail (0.00s)\n"} +{"Time":"2022-06-19T13:44:44.859712195-04:00","Action":"fail","Package":"github.com/example/pkg","Test":"TestFail","Elapsed":0.02} +{"Time":"2022-06-19T13:44:44.859714391-04:00","Action":"run","Package":"github.com/example/pkg","Test":"TestSkip"} +{"Time":"2022-06-19T13:44:44.859716575-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestSkip","Output":"=== RUN TestSkip\n"} +{"Time":"2022-06-19T13:44:44.859719038-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestSkip","Output":"--- SKIP: TestSkip (0.00s)\n"} +{"Time":"2022-06-19T13:44:44.859721732-04:00","Action":"skip","Package":"github.com/example/pkg","Test":"TestSkip","Elapsed":0.001} +{"Time":"2022-06-19T13:44:44.859724262-04:00","Action":"run","Package":"github.com/example/pkg","Test":"TestPassWithOutput"} +{"Time":"2022-06-19T13:44:44.859726506-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestPassWithOutput","Output":"=== RUN TestPassWithOutput\n"} +{"Time":"2022-06-19T13:44:44.859728601-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestPassWithOutput","Output":"some debug output\n"} +{"Time":"2022-06-19T13:44:44.859738124-04:00","Action":"output","Package":"github.com/example/pkg","Test":"TestPassWithOutput","Output":"--- PASS: TestPassWithOutput (0.00s)\n"} +{"Time":"2022-06-19T13:44:44.859741082-04:00","Action":"pass","Package":"github.com/example/pkg","Test":"TestPassWithOutput","Elapsed":0.03} +{"Time":"2022-06-19T13:44:44.859860205-04:00","Action":"output","Package":"github.com/example/pkg","Output":"PASS\n"} +{"Time":"2022-06-19T13:44:44.859862788-04:00","Action":"output","Package":"github.com/example/pkg","Output":"ok \tgithub.com/example/pkg\t0.05s\n"} +{"Time":"2022-06-19T13:44:44.85986508-04:00","Action":"pass","Package":"github.com/example/pkg","Elapsed":0.05} diff --git a/testjson/testdata/tapformat-golden.tap b/testjson/testdata/tapformat-golden.tap new file mode 100644 index 00000000..d40d1ca2 --- /dev/null +++ b/testjson/testdata/tapformat-golden.tap @@ -0,0 +1,9 @@ +TAP version 13 +1..N +ok 1 - github.com/example/pkg.TestPass # time=0.010s +not ok 2 - github.com/example/pkg.TestFail # time=0.020s +# === RUN TestFail +# example_test.go:42: assertion failed +# --- FAIL: TestFail (0.00s) +ok 3 - github.com/example/pkg.TestSkip # SKIP # time=0.001s +ok 4 - github.com/example/pkg.TestPassWithOutput # time=0.030s