From 37cf3135376545be5a860aa54d3bf5efd7235425 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:40:26 +0100 Subject: [PATCH 1/9] Use cursor-based pagination when getting runs from the API server Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsDownload_test.go | 7 +- pkg/runs/runsGet.go | 12 +- pkg/runs/runsGet_test.go | 217 ++++++++++++++++------------------ 3 files changed, 108 insertions(+), 128 deletions(-) diff --git a/pkg/runs/runsDownload_test.go b/pkg/runs/runsDownload_test.go index fd5123e4..da726600 100644 --- a/pkg/runs/runsDownload_test.go +++ b/pkg/runs/runsDownload_test.go @@ -12,7 +12,6 @@ import ( "net/http" "net/http/httptest" "os" - "strconv" "strings" "testing" "time" @@ -237,10 +236,7 @@ func WriteMockRasRunsResponse( writer.Header().Set("Content-Type", "application/json") values := req.URL.Query() - pageRequestedStr := values.Get("page") runNameQueryParameter := values.Get("runname") - pageRequested, _ := strconv.Atoi(pageRequestedStr) - assert.Equal(t, pageRequested, 1) assert.Equal(t, runNameQueryParameter, runName) @@ -254,9 +250,8 @@ func WriteMockRasRunsResponse( writer.Write([]byte(fmt.Sprintf(` { - "pageNumber": 1, + "nextCursor": "", "pageSize": 1, - "numPages": 1, "amountOfRuns": %d, "runs":[ %s ] }`, len(runResultStrings), combinedRunResultStrings))) diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index 48fe614c..851385c5 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -222,6 +222,7 @@ func GetRunsFromRestApi( var pageNumberWanted int32 = 1 gotAllResults := false var restApiVersion string + var pageCursor string = "" restApiVersion, err = embedded.GetGalasactlRestApiVersion() @@ -232,7 +233,7 @@ func GetRunsFromRestApi( var runData *galasaapi.RunResults var httpResponse *http.Response log.Printf("Requesting page '%d' ", pageNumberWanted) - apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion) + apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion).IncludeCursor("true") if fromAgeMins != 0 { apicall = apicall.From(fromTime) } @@ -251,7 +252,9 @@ func GetRunsFromRestApi( if shouldGetActive { apicall = apicall.Status(activeStatusNames) } - apicall = apicall.Page(pageNumberWanted) + if pageCursor != "" { + apicall = apicall.Cursor(pageCursor) + } apicall = apicall.Sort("to:desc") runData, httpResponse, err = apicall.Execute() @@ -270,10 +273,11 @@ func GetRunsFromRestApi( // Note: The ... syntax means 'all of the array', so they all get appended at once. results = append(results, runsOnThisPage...) - // Have we processed the last page ? - if pageNumberWanted == runData.GetNumPages() { + // If the page cursor is the same, then we've gone through all pages + if pageCursor == runData.GetNextCursor() || runData.GetNextCursor() == "" { gotAllResults = true } else { + pageCursor = runData.GetNextCursor() pageNumberWanted++ } } diff --git a/pkg/runs/runsGet_test.go b/pkg/runs/runsGet_test.go index 16b2728b..2937fb78 100644 --- a/pkg/runs/runsGet_test.go +++ b/pkg/runs/runsGet_test.go @@ -10,7 +10,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "strconv" "strings" "testing" @@ -84,9 +83,17 @@ const ( "contentType": "application/json" }] }` + + EMPTY_RUNS_RESPONSE = ` + { + "nextCursor": "", + "pageSize": 1, + "amountOfRuns": 0, + "runs":[] + }` ) -func NewRunsGetServletMock(t *testing.T, status int, runName string, runResultStrings ...string) *httptest.Server { +func NewRunsGetServletMock(t *testing.T, status int, nextPageCursors []string, runName string, runResultStrings ...string) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { clientVersion := r.Header.Get("ClientApiVersion") @@ -96,7 +103,8 @@ func NewRunsGetServletMock(t *testing.T, status int, runName string, runResultSt } else if strings.Contains(r.URL.Path, "/ras/resultnames") { ConfigureServerForResultNamesEndpoint(t, w, r, status) } else { - ConfigureServerForRasRunsEndpoint(t, w, r, runName, status, runResultStrings...) + ConfigureServerForRasRunsEndpoint(t, w, r, nextPageCursors[len(nextPageCursors) - 1], runName, status, runResultStrings...) + nextPageCursors = nextPageCursors[:len(nextPageCursors) - 1] } })) return server @@ -127,7 +135,15 @@ func ConfigureServerForDetailsEndpoint(t *testing.T, w http.ResponseWriter, r *h `, combinedRunResultStrings))) } -func ConfigureServerForRasRunsEndpoint(t *testing.T, w http.ResponseWriter, r *http.Request, runName string, status int, runResultStrings ...string) { +func ConfigureServerForRasRunsEndpoint( + t *testing.T, + w http.ResponseWriter, + r *http.Request, + nextPageCursor string, + runName string, + status int, + runResultStrings ...string, +) { if r.URL.Path != "/ras/runs" { t.Errorf("Expected to request '/ras/runs', got: %s", r.URL.Path) } @@ -138,10 +154,12 @@ func ConfigureServerForRasRunsEndpoint(t *testing.T, w http.ResponseWriter, r *h w.WriteHeader(status) values := r.URL.Query() - pageRequestedStr := values.Get("page") runNameQueryParameter := values.Get("runname") - pageRequested, _ := strconv.Atoi(pageRequestedStr) - assert.Equal(t, pageRequested, 1) + + cursorQueryParameter := values.Get("cursor") + if cursorQueryParameter != "" { + assert.Equal(t, nextPageCursor, cursorQueryParameter) + } assert.Equal(t, runNameQueryParameter, runName) combinedRunResultStrings := "" @@ -154,12 +172,11 @@ func ConfigureServerForRasRunsEndpoint(t *testing.T, w http.ResponseWriter, r *h w.Write([]byte(fmt.Sprintf(` { - "pageNumber": 1, + "nextCursor": "%s", "pageSize": 1, - "numPages": 1, "amountOfRuns": %d, "runs":[ %s ] - }`, len(runResultStrings), combinedRunResultStrings))) + }`, nextPageCursor, len(runResultStrings), combinedRunResultStrings))) } func ConfigureServerForResultNamesEndpoint(t *testing.T, w http.ResponseWriter, r *http.Request, status int) { @@ -208,12 +225,13 @@ func TestOutputFormatGarbageStringValidationGivesError(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedSummary(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } runName := "U456" age := "2d:24h" requestor := "" result := "" - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) shouldGetActive := false defer server.Close() @@ -247,11 +265,12 @@ func TestRunsGetOfRunNameWhichDoesNotExistProducesError(t *testing.T) { // Given ... age := "2d:24h" runName := "garbage" + nextPageCursor := []string{ "" } requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName) defer server.Close() mockConsole := utils.NewMockConsole() @@ -276,13 +295,14 @@ func TestRunsGetOfRunNameWhichDoesNotExistProducesError(t *testing.T) { func TestRunsGetWhereRunNameExistsTwiceProducesTwoRunResultLines(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456, RUN_U456_v2) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456, RUN_U456_v2) defer server.Close() mockConsole := utils.NewMockConsole() @@ -350,13 +370,14 @@ func TestOutputFormatDetailsValidatesOk(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedDetails(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "details" @@ -410,13 +431,14 @@ func TestGetFormatterNamesStringMultipleFormattersFormatsOk(t *testing.T) { func TestAPIInternalErrorIsHandledOk(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusInternalServerError, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusInternalServerError, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "details" @@ -439,13 +461,14 @@ func TestAPIInternalErrorIsHandledOk(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedRaw(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "raw" @@ -579,14 +602,7 @@ func TestRunsGetURLQueryWithFromAndToDate(t *testing.T) { assert.NotEqualValues(t, query.Get("to"), "") w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -619,14 +635,7 @@ func TestRunsGetURLQueryJustFromAge(t *testing.T) { assert.EqualValues(t, query.Get("to"), "") w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -659,14 +668,7 @@ func TestRunsGetURLQueryWithNoRunNameAndNoFromAgeReturnsError(t *testing.T) { assert.EqualValues(t, query.Get("runname"), "") w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -700,14 +702,7 @@ func TestRunsGetURLQueryWithOlderToAgeThanFromAgeReturnsError(t *testing.T) { assert.EqualValues(t, query.Get("runname"), "U456") w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -741,14 +736,7 @@ func TestRunsGetURLQueryWithBadlyFormedFromAndToParameterReturnsError(t *testing assert.EqualValues(t, query.Get("runname"), "U456") w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -937,14 +925,7 @@ func TestRunsGetURLQueryWithRequestorNotSuppliedReturnsOK(t *testing.T) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -979,14 +960,7 @@ func TestRunsGetURLQueryWithRequestorSuppliedReturnsOK(t *testing.T) { assert.EqualValues(t, query.Get("requestor"), requestor) w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1021,14 +995,7 @@ func TestRunsGetURLQueryWithNumericRequestorSuppliedReturnsOK(t *testing.T) { assert.Contains(t, r.URL.RawQuery, "requestor="+url.QueryEscape(requestor)) w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1063,14 +1030,7 @@ func TestRunsGetURLQueryWithDashInRequestorSuppliedReturnsOK(t *testing.T) { assert.Contains(t, r.URL.RawQuery, "requestor="+url.QueryEscape(requestor)) w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1105,14 +1065,7 @@ func TestRunsGetURLQueryWithAmpersandRequestorSuppliedReturnsOK(t *testing.T) { assert.Contains(t, r.URL.RawQuery, "requestor="+url.QueryEscape(requestor)) w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1148,14 +1101,7 @@ func TestRunsGetURLQueryWithSpecialCharactersRequestorSuppliedReturnsOK(t *testi w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1175,13 +1121,14 @@ func TestRunsGetURLQueryWithSpecialCharactersRequestorSuppliedReturnsOK(t *testi func TestRunsGetURLQueryWithResultSuppliedReturnsOK(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1202,13 +1149,14 @@ func TestRunsGetURLQueryWithResultSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithMultipleResultSuppliedReturnsOK(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed,envfail" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1230,13 +1178,14 @@ func TestRunsGetURLQueryWithMultipleResultSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithResultNotSuppliedReturnsOK(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1255,13 +1204,14 @@ func TestRunsGetURLQueryWithResultNotSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithInvalidResultSuppliedReturnsError(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "garbage" shouldGetActive := false - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1282,13 +1232,14 @@ func TestRunsGetURLQueryWithInvalidResultSuppliedReturnsError(t *testing.T) { func TestActiveAndResultAreMutuallyExclusiveShouldReturnError(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed" shouldGetActive := true - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1308,13 +1259,14 @@ func TestActiveAndResultAreMutuallyExclusiveShouldReturnError(t *testing.T) { func TestActiveParameterReturnsOk(t *testing.T) { // Given ... + nextPageCursor := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := true - server := NewRunsGetServletMock(t, http.StatusOK, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) defer server.Close() outputFormat := "summary" @@ -1351,14 +1303,7 @@ func TestRunsGetActiveRunsBuildsQueryCorrectly(t *testing.T) { assert.NotContains(t, r.URL.RawQuery, "status="+url.QueryEscape("finished")) w.Header().Set("Content-Type", "application/json") w.WriteHeader(200) - w.Write([]byte(` - { - "pageNumber": 1, - "pageSize": 1, - "numPages": 1, - "amountOfRuns": 0, - "runs":[] - }`)) + w.Write([]byte(EMPTY_RUNS_RESPONSE)) })) defer server.Close() @@ -1375,3 +1320,39 @@ func TestRunsGetActiveRunsBuildsQueryCorrectly(t *testing.T) { // Then ... assert.Nil(t, err) } + + +func TestRunsGetWithNextCursorGetsNextPageOfRuns(t *testing.T) { + + // Given ... + // When the last page is reached, the cursor doesn't change since all runs have been returned + nextPageCursors := []string{ "last-page", "last-page" } + age := "" + runName := "U456" + requestor := "" + result := "" + shouldGetActive := false + + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, runName, RUN_U456) + defer server.Close() + + outputFormat := "raw" + mockConsole := utils.NewMockConsole() + + apiServerUrl := server.URL + apiClient := api.InitialiseAPI(apiServerUrl) + mockTimeService := utils.NewMockTimeService() + + // When... + err := GetRuns(runName, age, requestor, result, shouldGetActive, outputFormat, mockTimeService, mockConsole, apiServerUrl, apiClient) + + // Then... + assert.Nil(t, err) + runsReturned := mockConsole.ReadText() + assert.Contains(t, runsReturned, runName) + + // The servlet should have been sent two requests, so expect two of the same run for this test + run := "U456|Finished|Passed|2023-05-10T06:00:13.043037Z|2023-05-10T06:00:36.159003Z|2023-05-10T06:02:53.823338Z|137664|myTestPackage.MyTestName|unitTesting|myBundleId|" + apiServerUrl + "/ras/runs/xxx876xxx/runlog\n" + expectedResults := run + run + assert.Equal(t, runsReturned, expectedResults) +} From 18310fc031561da188db3ff4c4d71603aec7d367 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:35:35 +0100 Subject: [PATCH 2/9] Remove includeCursor query parameter Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsDownload_test.go | 1 - pkg/runs/runsGet.go | 8 ++++---- pkg/runs/runsGet_test.go | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/runs/runsDownload_test.go b/pkg/runs/runsDownload_test.go index da726600..06d48490 100644 --- a/pkg/runs/runsDownload_test.go +++ b/pkg/runs/runsDownload_test.go @@ -250,7 +250,6 @@ func WriteMockRasRunsResponse( writer.Write([]byte(fmt.Sprintf(` { - "nextCursor": "", "pageSize": 1, "amountOfRuns": %d, "runs":[ %s ] diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index 851385c5..066ee400 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -222,7 +222,7 @@ func GetRunsFromRestApi( var pageNumberWanted int32 = 1 gotAllResults := false var restApiVersion string - var pageCursor string = "" + var pageCursor string restApiVersion, err = embedded.GetGalasactlRestApiVersion() @@ -233,7 +233,7 @@ func GetRunsFromRestApi( var runData *galasaapi.RunResults var httpResponse *http.Response log.Printf("Requesting page '%d' ", pageNumberWanted) - apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion).IncludeCursor("true") + apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion) if fromAgeMins != 0 { apicall = apicall.From(fromTime) } @@ -273,8 +273,8 @@ func GetRunsFromRestApi( // Note: The ... syntax means 'all of the array', so they all get appended at once. results = append(results, runsOnThisPage...) - // If the page cursor is the same, then we've gone through all pages - if pageCursor == runData.GetNextCursor() || runData.GetNextCursor() == "" { + // Have we processed the last page ? + if pageCursor == runData.GetNextCursor() || !runData.HasNextCursor() { gotAllResults = true } else { pageCursor = runData.GetNextCursor() diff --git a/pkg/runs/runsGet_test.go b/pkg/runs/runsGet_test.go index 2937fb78..1a9968c4 100644 --- a/pkg/runs/runsGet_test.go +++ b/pkg/runs/runsGet_test.go @@ -86,7 +86,6 @@ const ( EMPTY_RUNS_RESPONSE = ` { - "nextCursor": "", "pageSize": 1, "amountOfRuns": 0, "runs":[] From e93dad9c3378432028d4af12cad8f72926f196eb Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:54:36 +0100 Subject: [PATCH 3/9] Swap cursor condition order Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsGet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index 066ee400..1a14639b 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -274,7 +274,7 @@ func GetRunsFromRestApi( results = append(results, runsOnThisPage...) // Have we processed the last page ? - if pageCursor == runData.GetNextCursor() || !runData.HasNextCursor() { + if !runData.HasNextCursor() || pageCursor == runData.GetNextCursor() { gotAllResults = true } else { pageCursor = runData.GetNextCursor() From 60b2057b8ab5b712c1f2a860edf428b5e3019c3c Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:13:28 +0100 Subject: [PATCH 4/9] Turn on includeCursor parameter Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsGet.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index 1a14639b..ba8078ca 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -233,7 +233,7 @@ func GetRunsFromRestApi( var runData *galasaapi.RunResults var httpResponse *http.Response log.Printf("Requesting page '%d' ", pageNumberWanted) - apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion) + apicall := apiClient.ResultArchiveStoreAPIApi.GetRasSearchRuns(context).ClientApiVersion(restApiVersion).IncludeCursor("true") if fromAgeMins != 0 { apicall = apicall.From(fromTime) } @@ -272,6 +272,7 @@ func GetRunsFromRestApi( // Add all the runs into our set of results. // Note: The ... syntax means 'all of the array', so they all get appended at once. results = append(results, runsOnThisPage...) + log.Print("APPENDING ") // Have we processed the last page ? if !runData.HasNextCursor() || pageCursor == runData.GetNextCursor() { From 782c97a77e2bce178b7c082b3eb24bc723c10860 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:52:57 +0100 Subject: [PATCH 5/9] Remove log statement Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsGet.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index ba8078ca..ca2b1f7c 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -272,7 +272,6 @@ func GetRunsFromRestApi( // Add all the runs into our set of results. // Note: The ... syntax means 'all of the array', so they all get appended at once. results = append(results, runsOnThisPage...) - log.Print("APPENDING ") // Have we processed the last page ? if !runData.HasNextCursor() || pageCursor == runData.GetNextCursor() { From be5133aeb7fe719c2001233ef81daf94f921ed71 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:08:55 +0100 Subject: [PATCH 6/9] Empty commit to kick off build Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> From 354b1e89bf48d330b0b112c03d98ec18fe34ca58 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:55:48 +0100 Subject: [PATCH 7/9] Empty commit to kick off build Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> From b8d5479535f3be126572081c90971e77a3f5a168 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:33:28 +0100 Subject: [PATCH 8/9] Use 'from:desc' as the default sort for runs get Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> --- pkg/runs/runsGet.go | 4 +- pkg/runs/runsGet_test.go | 131 +++++++++++++++++++++++++++------------ 2 files changed, 95 insertions(+), 40 deletions(-) diff --git a/pkg/runs/runsGet.go b/pkg/runs/runsGet.go index ca2b1f7c..d6fcc745 100644 --- a/pkg/runs/runsGet.go +++ b/pkg/runs/runsGet.go @@ -255,7 +255,7 @@ func GetRunsFromRestApi( if pageCursor != "" { apicall = apicall.Cursor(pageCursor) } - apicall = apicall.Sort("to:desc") + apicall = apicall.Sort("from:desc") runData, httpResponse, err = apicall.Execute() if err != nil { @@ -274,7 +274,7 @@ func GetRunsFromRestApi( results = append(results, runsOnThisPage...) // Have we processed the last page ? - if !runData.HasNextCursor() || pageCursor == runData.GetNextCursor() { + if !runData.HasNextCursor() || len(runsOnThisPage) < int(runData.GetPageSize()) { gotAllResults = true } else { pageCursor = runData.GetNextCursor() diff --git a/pkg/runs/runsGet_test.go b/pkg/runs/runsGet_test.go index 1a9968c4..bc7b8638 100644 --- a/pkg/runs/runsGet_test.go +++ b/pkg/runs/runsGet_test.go @@ -92,7 +92,7 @@ const ( }` ) -func NewRunsGetServletMock(t *testing.T, status int, nextPageCursors []string, runName string, runResultStrings ...string) *httptest.Server { +func NewRunsGetServletMock(t *testing.T, status int, nextPageCursors []string, pages map[string][]string, pageSize int, runName string, runResultStrings ...string) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { clientVersion := r.Header.Get("ClientApiVersion") @@ -102,8 +102,13 @@ func NewRunsGetServletMock(t *testing.T, status int, nextPageCursors []string, r } else if strings.Contains(r.URL.Path, "/ras/resultnames") { ConfigureServerForResultNamesEndpoint(t, w, r, status) } else { - ConfigureServerForRasRunsEndpoint(t, w, r, nextPageCursors[len(nextPageCursors) - 1], runName, status, runResultStrings...) - nextPageCursors = nextPageCursors[:len(nextPageCursors) - 1] + nextCursor := "" + if len(nextPageCursors) > 0 { + // Advance the expected page cursors by one + nextCursor = nextPageCursors[0] + nextPageCursors = nextPageCursors[1:] + } + ConfigureServerForRasRunsEndpoint(t, w, r, pages, nextCursor, runName, pageSize, status) } })) return server @@ -138,10 +143,11 @@ func ConfigureServerForRasRunsEndpoint( t *testing.T, w http.ResponseWriter, r *http.Request, + pages map[string][]string, nextPageCursor string, runName string, + pageSize int, status int, - runResultStrings ...string, ) { if r.URL.Path != "/ras/runs" { t.Errorf("Expected to request '/ras/runs', got: %s", r.URL.Path) @@ -155,14 +161,22 @@ func ConfigureServerForRasRunsEndpoint( values := r.URL.Query() runNameQueryParameter := values.Get("runname") + var pageRunsJson []string + var keyExists bool cursorQueryParameter := values.Get("cursor") - if cursorQueryParameter != "" { - assert.Equal(t, nextPageCursor, cursorQueryParameter) - } + + // Keys of the pages map correspond to page cursors, including + // an empty string key for the first request to /ras/runs + pageRunsJson, keyExists = pages[cursorQueryParameter] + assert.True(t, keyExists) + + // Subsequent requests shouldn't be made to the same page, + // so delete the page since we've visited it + delete(pages, cursorQueryParameter) assert.Equal(t, runNameQueryParameter, runName) combinedRunResultStrings := "" - for index, runResult := range runResultStrings { + for index, runResult := range pageRunsJson { if index > 0 { combinedRunResultStrings += "," } @@ -172,10 +186,10 @@ func ConfigureServerForRasRunsEndpoint( w.Write([]byte(fmt.Sprintf(` { "nextCursor": "%s", - "pageSize": 1, + "pageSize": %d, "amountOfRuns": %d, "runs":[ %s ] - }`, nextPageCursor, len(runResultStrings), combinedRunResultStrings))) + }`, nextPageCursor, pageSize, len(pageRunsJson), combinedRunResultStrings))) } func ConfigureServerForResultNamesEndpoint(t *testing.T, w http.ResponseWriter, r *http.Request, status int) { @@ -224,13 +238,17 @@ func TestOutputFormatGarbageStringValidationGivesError(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedSummary(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } + runName := "U456" age := "2d:24h" requestor := "" result := "" + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) shouldGetActive := false defer server.Close() @@ -264,12 +282,15 @@ func TestRunsGetOfRunNameWhichDoesNotExistProducesError(t *testing.T) { // Given ... age := "2d:24h" runName := "garbage" - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() mockConsole := utils.NewMockConsole() @@ -294,14 +315,17 @@ func TestRunsGetOfRunNameWhichDoesNotExistProducesError(t *testing.T) { func TestRunsGetWhereRunNameExistsTwiceProducesTwoRunResultLines(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456, RUN_U456_v2 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456, RUN_U456_v2) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() mockConsole := utils.NewMockConsole() @@ -369,14 +393,17 @@ func TestOutputFormatDetailsValidatesOk(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedDetails(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName, RUN_U456) defer server.Close() outputFormat := "details" @@ -430,14 +457,17 @@ func TestGetFormatterNamesStringMultipleFormattersFormatsOk(t *testing.T) { func TestAPIInternalErrorIsHandledOk(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusInternalServerError, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusInternalServerError, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "details" @@ -460,14 +490,17 @@ func TestAPIInternalErrorIsHandledOk(t *testing.T) { func TestRunsGetOfRunNameWhichExistsProducesExpectedRaw(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "raw" @@ -1120,14 +1153,17 @@ func TestRunsGetURLQueryWithSpecialCharactersRequestorSuppliedReturnsOK(t *testi func TestRunsGetURLQueryWithResultSuppliedReturnsOK(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1148,14 +1184,17 @@ func TestRunsGetURLQueryWithResultSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithMultipleResultSuppliedReturnsOK(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed,envfail" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1177,14 +1216,17 @@ func TestRunsGetURLQueryWithMultipleResultSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithResultNotSuppliedReturnsOK(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1203,14 +1245,17 @@ func TestRunsGetURLQueryWithResultNotSuppliedReturnsOK(t *testing.T) { func TestRunsGetURLQueryWithInvalidResultSuppliedReturnsError(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "garbage" shouldGetActive := false + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1231,14 +1276,17 @@ func TestRunsGetURLQueryWithInvalidResultSuppliedReturnsError(t *testing.T) { func TestActiveAndResultAreMutuallyExclusiveShouldReturnError(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "Passed" shouldGetActive := true + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1258,14 +1306,17 @@ func TestActiveAndResultAreMutuallyExclusiveShouldReturnError(t *testing.T) { func TestActiveParameterReturnsOk(t *testing.T) { // Given ... - nextPageCursor := []string{ "" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + nextPageCursors := []string{ "" } age := "" runName := "U456" requestor := "" result := "" shouldGetActive := true + pageSize := 100 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursor, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "summary" @@ -1324,15 +1375,20 @@ func TestRunsGetActiveRunsBuildsQueryCorrectly(t *testing.T) { func TestRunsGetWithNextCursorGetsNextPageOfRuns(t *testing.T) { // Given ... - // When the last page is reached, the cursor doesn't change since all runs have been returned - nextPageCursors := []string{ "last-page", "last-page" } + pages := make(map[string][]string, 0) + pages[""] = []string{ RUN_U456 } + pages["page2"] = []string{ RUN_U456 } + pages["page3"] = []string{} + nextPageCursors := []string{ "page2", "page3" } + age := "" runName := "U456" requestor := "" result := "" shouldGetActive := false + pageSize := 1 - server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, runName, RUN_U456) + server := NewRunsGetServletMock(t, http.StatusOK, nextPageCursors, pages, pageSize, runName) defer server.Close() outputFormat := "raw" @@ -1350,7 +1406,6 @@ func TestRunsGetWithNextCursorGetsNextPageOfRuns(t *testing.T) { runsReturned := mockConsole.ReadText() assert.Contains(t, runsReturned, runName) - // The servlet should have been sent two requests, so expect two of the same run for this test run := "U456|Finished|Passed|2023-05-10T06:00:13.043037Z|2023-05-10T06:00:36.159003Z|2023-05-10T06:02:53.823338Z|137664|myTestPackage.MyTestName|unitTesting|myBundleId|" + apiServerUrl + "/ras/runs/xxx876xxx/runlog\n" expectedResults := run + run assert.Equal(t, runsReturned, expectedResults) From 2efff1f39e59d4e41cbedd98a7f11067f1d048d0 Mon Sep 17 00:00:00 2001 From: Eamonn Mansour <47121388+eamansour@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:09:23 +0100 Subject: [PATCH 9/9] Empty commit to kick off build Signed-off-by: Eamonn Mansour <47121388+eamansour@users.noreply.github.com>