diff --git a/internal/job/starter.go b/internal/job/starter.go
index d29a27bc9..0097d8a83 100644
--- a/internal/job/starter.go
+++ b/internal/job/starter.go
@@ -10,6 +10,9 @@ type StartOptions struct {
// DisplayName is used for local logging purposes only (e.g. console).
DisplayName string `json:"-"`
+ // PreviousJobIDs contains the previous jobs IDs in the context of retries.
+ PreviousJobIDs []string `json:"-"`
+
// Timeout is used for local/per-suite timeout.
Timeout time.Duration `json:"-"`
diff --git a/internal/junit/reporter.go b/internal/junit/reporter.go
index 005ce347f..4d3acd0b7 100644
--- a/internal/junit/reporter.go
+++ b/internal/junit/reporter.go
@@ -2,6 +2,7 @@ package junit
import (
"encoding/xml"
+ "errors"
"fmt"
"os"
"strconv"
@@ -9,6 +10,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/saucelabs/saucectl/internal/report"
+ "golang.org/x/exp/maps"
)
// Reporter is a junit implementation for report.Reporter.
@@ -25,6 +27,99 @@ func (r *Reporter) Add(t report.TestResult) {
r.TestResults = append(r.TestResults, t)
}
+func parseJunitFiles(junits []report.Artifact) ([]TestSuites, error) {
+ var parsed []TestSuites
+ var errs []error
+ for _, ju := range junits {
+ if ju.Error != nil {
+ errs = append(errs, fmt.Errorf("failed to retrieve junit file: %w", ju.Error))
+ continue
+ }
+ ts, err := Parse(ju.Body)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("failed to parse junit file: %w", err))
+ continue
+ }
+ parsed = append(parsed, ts)
+ }
+ if len(errs) > 0 {
+ return parsed, fmt.Errorf("%d errors occured while evaluating junit files: %w", len(errs), errors.Join(errs...))
+ }
+ return parsed, nil
+}
+
+// reduceSuite updates "old" with values from "new".
+func reduceSuite(old TestSuite, new TestSuite) TestSuite {
+ testMap := map[string]int{}
+ for idx, tc := range old.TestCases {
+ key := fmt.Sprintf(`%s.%s`, tc.ClassName, tc.Name)
+ testMap[key] = idx
+ }
+
+ for _, tc := range new.TestCases {
+ key := fmt.Sprintf(`%s.%s`, tc.ClassName, tc.Name)
+ var idx int
+ var ok bool
+ if idx, ok = testMap[key]; !ok {
+ log.Warn().Str("test", key).Msg("Sanity check failed when merging related junit test suites. New test encountered without prior history.")
+ continue
+ }
+ old.TestCases[idx] = tc
+ }
+ old.Tests = len(old.TestCases)
+ old.Errors = countErrors(old.TestCases)
+ old.Skipped = countSkipped(old.TestCases)
+ return old
+}
+
+func reduceJunitFiles(junits []TestSuites) TestSuites {
+ suites := map[string]TestSuite{}
+
+ for _, junit := range junits {
+ for _, suite := range junit.TestSuites {
+ if _, ok := suites[suite.Name]; !ok {
+ suites[suite.Name] = suite
+ continue
+ }
+ suites[suite.Name] = reduceSuite(suites[suite.Name], suite)
+ }
+ }
+
+ output := TestSuites{}
+
+ output.TestSuites = append(output.TestSuites, maps.Values(suites)...)
+ return output
+}
+
+func countErrors(tcs []TestCase) int {
+ count := 0
+ for _, tc := range tcs {
+ if tc.Status == "error" {
+ count++
+ }
+ }
+ return count
+}
+func countSkipped(tcs []TestCase) int {
+ count := 0
+ for _, tc := range tcs {
+ if tc.Status == "skipped" {
+ count++
+ }
+ }
+ return count
+}
+
+func filterJunitArtifacts(artifacts []report.Artifact) []report.Artifact {
+ var junits []report.Artifact
+ for _, v := range artifacts {
+ if v.AssetType == report.JUnitArtifact {
+ junits = append(junits, v)
+ }
+ }
+ return junits
+}
+
// Render renders out a test summary junit report to the destination of Reporter.Filename.
func (r *Reporter) Render() {
r.lock.Lock()
@@ -36,33 +131,24 @@ func (r *Reporter) Render() {
Name: v.Name,
Time: strconv.Itoa(int(v.Duration.Seconds())),
}
-
t.Properties = append(t.Properties, extractProperties(v)...)
- for _, a := range v.Artifacts {
- if a.AssetType != report.JUnitArtifact {
- continue
- }
-
- if a.Error != nil {
- t.Errors++
- log.Warn().Err(a.Error).Str("suite", v.Name).Msg("Failed to download junit report. Summary may be incorrect!")
- continue
- }
+ mainJunits := filterJunitArtifacts(v.Artifacts)
+ junitFiles := v.ParentJUnits
+ junitFiles = append(junitFiles, mainJunits...)
- jsuites, err := Parse(a.Body)
- if err != nil {
- t.Errors++
- log.Warn().Err(err).Str("suite", v.Name).Msg("Failed to parse junit report. Summary may be incorrect!")
- continue
- }
+ jsuites, err := parseJunitFiles(junitFiles)
+ if err != nil {
+ log.Warn().Err(err).Str("suite", v.Name).Msg("Failed to parse some junit report. Summary may be incorrect!")
+ continue
+ }
+ reduced := reduceJunitFiles(jsuites)
- for _, ts := range jsuites.TestSuites {
- t.Tests += ts.Tests
- t.Failures += ts.Failures
- t.Errors += ts.Errors
- t.TestCases = append(t.TestCases, ts.TestCases...)
- }
+ for _, ts := range reduced.TestSuites {
+ t.Tests += ts.Tests
+ t.Failures += ts.Failures
+ t.Errors += ts.Errors
+ t.TestCases = append(t.TestCases, ts.TestCases...)
}
tt.Tests += t.Tests
diff --git a/internal/junit/reporter_test.go b/internal/junit/reporter_test.go
index 334f16b58..2fb9dd87a 100644
--- a/internal/junit/reporter_test.go
+++ b/internal/junit/reporter_test.go
@@ -7,6 +7,7 @@ import (
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
"github.com/saucelabs/saucectl/internal/job"
"github.com/saucelabs/saucectl/internal/report"
)
@@ -122,3 +123,342 @@ func TestReporter_Render(t *testing.T) {
})
}
}
+
+func TestReduceJunitFiles(t *testing.T) {
+ input := []TestSuites{
+ {
+ TestSuites: []TestSuite{
+ {
+ Tests: 24,
+ Errors: 2,
+ Time: "47.917",
+ Package: "com.example.android.testing.espresso.BasicSample",
+ TestCases: []TestCase{
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test10Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test10Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test11Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test11Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "error",
+ Error: "androidx.test.espresso.base.AssertionErrorHandler$AssertionFailedWithCauseError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat dalvik.system.VMStack.getThreadStackTrace(Native Method)\n\tat java.lang.Thread.getStackTrace(Thread.java:1841)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:35)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:26)\n\tat androidx.test.espresso.base.DefaultFailureHandler$TypedFailureHandler.handle(DefaultFailureHandler.java:167)\n\tat androidx.test.espresso.base.DefaultFailureHandler.handle(DefaultFailureHandler.java:128)\n\tat androidx.test.espresso.ViewInteraction.waitForAndHandleInteractionResults(ViewInteraction.java:388)\n\tat androidx.test.espresso.ViewInteraction.check(ViewInteraction.java:367)\n\tat com.example.android.testing.espresso.BasicSample.Test12Test.changeText_sameActivity(Test12Test.java:74)\n\t... 33 trimmed\nCaused by: junit.framework.AssertionFailedError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat androidx.test.espresso.matcher.ViewMatchers.assertThat(ViewMatchers.java:611)\n\tat androidx.test.espresso.assertion.ViewAssertions$MatchesViewAssertion.check(ViewAssertions.java:97)\n\tat androidx.test.espresso.ViewInteraction$SingleExecutionViewAssertion.check(ViewInteraction.java:489)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:347)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:320)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat android.os.Handler.handleCallback(Handler.java:942)\n\tat android.os.Handler.dispatchMessage(Handler.java:99)\n\tat android.os.Looper.loopOnce(Looper.java:201)\n\tat android.os.Looper.loop(Looper.java:288)\n\tat android.app.ActivityThread.main(ActivityThread.java:7872)\n\tat java.lang.reflect.Method.invoke(Native Method)\n\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:548)\n\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:936)",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test1Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test1Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test2Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test2Test",
+ Status: "error",
+ Error: "androidx.test.espresso.base.AssertionErrorHandler$AssertionFailedWithCauseError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat dalvik.system.VMStack.getThreadStackTrace(Native Method)\n\tat java.lang.Thread.getStackTrace(Thread.java:1841)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:35)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:26)\n\tat androidx.test.espresso.base.DefaultFailureHandler$TypedFailureHandler.handle(DefaultFailureHandler.java:167)\n\tat androidx.test.espresso.base.DefaultFailureHandler.handle(DefaultFailureHandler.java:128)\n\tat androidx.test.espresso.ViewInteraction.waitForAndHandleInteractionResults(ViewInteraction.java:388)\n\tat androidx.test.espresso.ViewInteraction.check(ViewInteraction.java:367)\n\tat com.example.android.testing.espresso.BasicSample.Test12Test.changeText_sameActivity(Test12Test.java:74)\n\t... 33 trimmed\nCaused by: junit.framework.AssertionFailedError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat androidx.test.espresso.matcher.ViewMatchers.assertThat(ViewMatchers.java:611)\n\tat androidx.test.espresso.assertion.ViewAssertions$MatchesViewAssertion.check(ViewAssertions.java:97)\n\tat androidx.test.espresso.ViewInteraction$SingleExecutionViewAssertion.check(ViewInteraction.java:489)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:347)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:320)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat android.os.Handler.handleCallback(Handler.java:942)\n\tat android.os.Handler.dispatchMessage(Handler.java:99)\n\tat android.os.Looper.loopOnce(Looper.java:201)\n\tat android.os.Looper.loop(Looper.java:288)\n\tat android.app.ActivityThread.main(ActivityThread.java:7872)\n\tat java.lang.reflect.Method.invoke(Native Method)\n\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:548)\n\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:936)",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test3Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test3Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test4Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test4Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test5Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test5Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test6Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test6Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test7Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test7Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test8Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test8Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test9Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test9Test",
+ Status: "success",
+ },
+ },
+ SystemOut: "",
+ },
+ },
+ },
+ {
+ TestSuites: []TestSuite{
+ {
+ Tests: 2,
+ Errors: 1,
+ Time: "11.007",
+ Package: "com.example.android.testing.espresso.BasicSample",
+ TestCases: []TestCase{
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test2Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "error",
+ Error: "androidx.test.espresso.base.AssertionErrorHandler$AssertionFailedWithCauseError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat dalvik.system.VMStack.getThreadStackTrace(Native Method)\n\tat java.lang.Thread.getStackTrace(Thread.java:1841)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:35)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:26)\n\tat androidx.test.espresso.base.DefaultFailureHandler$TypedFailureHandler.handle(DefaultFailureHandler.java:167)\n\tat androidx.test.espresso.base.DefaultFailureHandler.handle(DefaultFailureHandler.java:128)\n\tat androidx.test.espresso.ViewInteraction.waitForAndHandleInteractionResults(ViewInteraction.java:388)\n\tat androidx.test.espresso.ViewInteraction.check(ViewInteraction.java:367)\n\tat com.example.android.testing.espresso.BasicSample.Test12Test.changeText_sameActivity(Test12Test.java:74)\n\t... 33 trimmed\nCaused by: junit.framework.AssertionFailedError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat androidx.test.espresso.matcher.ViewMatchers.assertThat(ViewMatchers.java:611)\n\tat androidx.test.espresso.assertion.ViewAssertions$MatchesViewAssertion.check(ViewAssertions.java:97)\n\tat androidx.test.espresso.ViewInteraction$SingleExecutionViewAssertion.check(ViewInteraction.java:489)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:347)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:320)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat android.os.Handler.handleCallback(Handler.java:942)\n\tat android.os.Handler.dispatchMessage(Handler.java:99)\n\tat android.os.Looper.loopOnce(Looper.java:201)\n\tat android.os.Looper.loop(Looper.java:288)\n\tat android.app.ActivityThread.main(ActivityThread.java:7872)\n\tat java.lang.reflect.Method.invoke(Native Method)\n\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:548)\n\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:936)",
+ },
+ },
+ SystemOut: "",
+ },
+ },
+ },
+ {
+ TestSuites: []TestSuite{
+ {
+ Tests: 1,
+ Errors: 1,
+ Time: "6.004",
+ Package: "com.example.android.testing.espresso.BasicSample",
+ TestCases: []TestCase{
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "error",
+ Error: "androidx.test.espresso.base.AssertionErrorHandler$AssertionFailedWithCauseError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat dalvik.system.VMStack.getThreadStackTrace(Native Method)\n\tat java.lang.Thread.getStackTrace(Thread.java:1841)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:35)\n\tat androidx.test.espresso.base.AssertionErrorHandler.handleSafely(AssertionErrorHandler.java:26)\n\tat androidx.test.espresso.base.DefaultFailureHandler$TypedFailureHandler.handle(DefaultFailureHandler.java:167)\n\tat androidx.test.espresso.base.DefaultFailureHandler.handle(DefaultFailureHandler.java:128)\n\tat androidx.test.espresso.ViewInteraction.waitForAndHandleInteractionResults(ViewInteraction.java:388)\n\tat androidx.test.espresso.ViewInteraction.check(ViewInteraction.java:367)\n\tat com.example.android.testing.espresso.BasicSample.Test12Test.changeText_sameActivity(Test12Test.java:74)\n\t... 33 trimmed\nCaused by: junit.framework.AssertionFailedError: 'an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"' doesn't match the selected view.\nExpected: an instance of android.widget.TextView and view.getText() with or without transformation to match: is \"Espresso\"\n Got: view.getText() was \"INVALID TYPING\"\nView Details: TextView{id=2130903044, res-name=textToBeChanged, visibility=VISIBLE, width=328, height=59, has-focus=false, has-focusable=false, has-window-focus=true, is-clickable=false, is-enabled=true, is-focused=false, is-focusable=false, is-layout-requested=false, is-selected=false, layout-params=android.widget.LinearLayout$LayoutParams@YYYYYY, tag=null, root-is-layout-requested=false, has-input-connection=false, x=220.0, y=96.0, text=INVALID TYPING, input-type=0, ime-target=false, has-links=false}\n\n\tat androidx.test.espresso.matcher.ViewMatchers.assertThat(ViewMatchers.java:611)\n\tat androidx.test.espresso.assertion.ViewAssertions$MatchesViewAssertion.check(ViewAssertions.java:97)\n\tat androidx.test.espresso.ViewInteraction$SingleExecutionViewAssertion.check(ViewInteraction.java:489)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:347)\n\tat androidx.test.espresso.ViewInteraction$2.call(ViewInteraction.java:320)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat android.os.Handler.handleCallback(Handler.java:942)\n\tat android.os.Handler.dispatchMessage(Handler.java:99)\n\tat android.os.Looper.loopOnce(Looper.java:201)\n\tat android.os.Looper.loop(Looper.java:288)\n\tat android.app.ActivityThread.main(ActivityThread.java:7872)\n\tat java.lang.reflect.Method.invoke(Native Method)\n\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:548)\n\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:936)",
+ },
+ },
+ SystemOut: "",
+ },
+ },
+ },
+ {
+ TestSuites: []TestSuite{
+ {
+ Tests: 1,
+ Errors: 0,
+ Time: "5.535",
+ Package: "com.example.android.testing.espresso.BasicSample",
+ TestCases: []TestCase{
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "success",
+ },
+ },
+ SystemOut: "",
+ },
+ },
+ },
+ }
+ want := TestSuites{
+ TestSuites: []TestSuite{
+ {
+ Tests: 24,
+ Errors: 0,
+ Time: "47.917",
+ Package: "com.example.android.testing.espresso.BasicSample",
+ TestCases: []TestCase{
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test10Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test10Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test11Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test11Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test12Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test1Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test1Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test2Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test2Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test3Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test3Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test4Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test4Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test5Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test5Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test6Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test6Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test7Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test7Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test8Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test8Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_newActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test9Test",
+ Status: "success",
+ },
+ {
+ Name: "changeText_sameActivity",
+ ClassName: "com.example.android.testing.espresso.BasicSample.Test9Test",
+ Status: "success",
+ },
+ },
+ SystemOut: "",
+ },
+ },
+ }
+
+ got := reduceJunitFiles(input)
+ if !cmp.Equal(want, got) {
+ t.Error(cmp.Diff(want, got))
+ }
+}
diff --git a/internal/report/report.go b/internal/report/report.go
index e77d094ce..80b003140 100644
--- a/internal/report/report.go
+++ b/internal/report/report.go
@@ -19,6 +19,7 @@ type TestResult struct {
RDC bool `json:"-"`
TimedOut bool `json:"-"`
PassThreshold bool `json:"passThreshold"`
+ ParentJUnits []Artifact `json:"-"`
}
// ArtifactType represents the type of assets (e.g. a junit report). Semantically similar to Content-Type.
diff --git a/internal/saucecloud/cloud.go b/internal/saucecloud/cloud.go
index e3dccb450..5ec06244e 100644
--- a/internal/saucecloud/cloud.go
+++ b/internal/saucecloud/cloud.go
@@ -77,6 +77,7 @@ type result struct {
endTime time.Time
attempts int
retries int
+ previous []string
details insights.Details
}
@@ -145,6 +146,7 @@ func (r *CloudRunner) collectResults(artifactCfg config.ArtifactDownload, result
}
var artifacts []report.Artifact
+ var parentJUnits []report.Artifact
if junitRequired {
jb, err := r.JobService.GetJobAssetFileContent(
@@ -157,6 +159,19 @@ func (r *CloudRunner) collectResults(artifactCfg config.ArtifactDownload, result
Body: jb,
Error: err,
})
+ for _, id := range res.previous {
+ jb, err := r.JobService.GetJobAssetFileContent(
+ context.Background(),
+ id,
+ junit.JunitFileName,
+ res.job.IsRDC,
+ )
+ parentJUnits = append(parentJUnits, report.Artifact{
+ AssetType: report.JUnitArtifact,
+ Body: jb,
+ Error: err,
+ })
+ }
}
var url string
@@ -164,20 +179,21 @@ func (r *CloudRunner) collectResults(artifactCfg config.ArtifactDownload, result
url = fmt.Sprintf("%s/tests/%s", r.Region.AppBaseURL(), res.job.ID)
}
tr := report.TestResult{
- Name: res.name,
- Duration: res.duration,
- StartTime: res.startTime,
- EndTime: res.endTime,
- Status: res.job.TotalStatus(),
- Browser: browser,
- Platform: platform,
- DeviceName: res.job.BaseConfig.DeviceName,
- URL: url,
- Artifacts: artifacts,
- Origin: "sauce",
- Attempts: res.attempts,
- RDC: res.job.IsRDC,
- TimedOut: res.job.TimedOut,
+ Name: res.name,
+ Duration: res.duration,
+ StartTime: res.startTime,
+ EndTime: res.endTime,
+ Status: res.job.TotalStatus(),
+ Browser: browser,
+ Platform: platform,
+ DeviceName: res.job.BaseConfig.DeviceName,
+ URL: url,
+ Artifacts: artifacts,
+ Origin: "sauce",
+ Attempts: res.attempts,
+ RDC: res.job.IsRDC,
+ TimedOut: res.job.TimedOut,
+ ParentJUnits: parentJUnits,
}
files := r.downloadArtifacts(res.name, res.job, artifactCfg.When)
@@ -338,6 +354,7 @@ func (r *CloudRunner) runJobs(jobOpts chan job.StartOptions, results chan<- resu
}
opts.Attempt++
+ opts.PreviousJobIDs = append(opts.PreviousJobIDs, jobData.ID)
go r.Retrier.Retry(jobOpts, opts, jobData)
continue
}
@@ -371,6 +388,7 @@ func (r *CloudRunner) runJobs(jobOpts chan job.StartOptions, results chan<- resu
attempts: opts.Attempt + 1,
retries: opts.Retries,
details: details,
+ previous: opts.PreviousJobIDs,
}
}
}