diff --git a/RepoDb.PagingPrimitives/RepoDbExtensions.PagingPrimitives.csproj b/RepoDb.PagingPrimitives/RepoDbExtensions.PagingPrimitives.csproj index c8b96f8..cbe8207 100644 --- a/RepoDb.PagingPrimitives/RepoDbExtensions.PagingPrimitives.csproj +++ b/RepoDb.PagingPrimitives/RepoDbExtensions.PagingPrimitives.csproj @@ -3,9 +3,9 @@ netstandard2.0;netstandard2.1;net6.0; true - 1.1.5.5 - 1.1.5.5 - 1.1.5.5 + 1.1.5.6 + 1.1.5.6 + 1.1.5.6 BBernard / CajunCoding CajunCoding The primitives and helpers needed for RepoDbExtensions.SqlServer.PagingOperations pacakge; used for working with modern pagination approaches such as Cursor based paging, as well as Offset based pagination, using the RepoDb ORM with Sql Server. @@ -16,9 +16,10 @@ repodb, paging, pagination, cursor, offset, skip, take, sorting, graphql, graph-ql, hotchocolate, dapper, sqlkata Release Notes: - - Improve flexibility of base interface support for cursor navigation with non-generic ICursorPageNavigationInfo. + - Fix bug with in-memory cursor paging logic incorrectly indexing the results resulting in invalid page results. Prior Release Notes: + - Improve flexibility of base interface support for cursor navigation with non-generic ICursorPageNavigationInfo. - Simplify RetrieveTotalCount param name on ICursorParams. - Fix SQL building bug not using Raw SQL as specified; improved integration tests to validate this better. - Improved raw sql parameter name to be more consistent with RepDb naming conventions. diff --git a/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingExecuteQueryApiForRawSql.cs b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingExecuteQueryApiForRawSql.cs index 24bc8dc..be73cf4 100644 --- a/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingExecuteQueryApiForRawSql.cs +++ b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingExecuteQueryApiForRawSql.cs @@ -14,7 +14,7 @@ namespace RepoDb.SqlServer.PagingOperations.Tests public class PagingTestsUsingExecuteQueryApiForRawSql : BaseTest { [TestMethod] - public async Task TestCursorPagingWithRawSql() + public async Task TestCursorPagingWithRawSqlAsync() { using (var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false)) { @@ -75,7 +75,7 @@ public async Task TestCursorPagingWithRawSql() } [TestMethod] - public async Task TestOffsetPagingWithRawSql() + public async Task TestOffsetPagingWithRawSqlAsync() { using (var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false)) { diff --git a/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingInMemoryPaging.cs b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingInMemoryPaging.cs new file mode 100644 index 0000000..8d85ecb --- /dev/null +++ b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingInMemoryPaging.cs @@ -0,0 +1,165 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Data.SqlClient; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using RepoDb.PagingPrimitives.CursorPaging; +using RepoDb.PagingPrimitives.OffsetPaging; +using RepoDb.SqlServer.PagingOperations.InMemoryPaging; +using StarWars.Characters.DbModels; + + +namespace RepoDb.SqlServer.PagingOperations.Tests +{ + [TestClass] + public class PagingTestsUsingInMemoryPaging : BaseTest + { + [TestMethod] + public void TestCursorPagingWithInMemorySlicing() + { + var expectedTotalCount = 100; + var items = Enumerable.Range(1, expectedTotalCount).ToArray(); + + const int pageSize = 10; + int? totalCount = null; + int runningTotal = 0; + ICursorPageResults page = null; + var pageCounter = 0; + + do + { + page = items.SliceAsCursorPage( + first: pageSize, + after: page?.EndCursor, + before: null, + last: null, + includeTotalCount: totalCount is null + ); + + pageCounter++; + page.Should().NotBeNull(); + + if (pageCounter == 1) + { + page.HasNextPage.Should().BeTrue(); + page.HasPreviousPage.Should().BeFalse(); + } + else if (pageCounter == (int)Math.Ceiling((decimal)expectedTotalCount / pageSize)) + { + page.HasNextPage.Should().BeFalse(); + page.HasPreviousPage.Should().BeTrue(); + } + else + { + page.HasNextPage.Should().BeTrue(); + page.HasPreviousPage.Should().BeTrue(); + } + + var resultsList = page.CursorResults.ToList(); + resultsList.Should().HaveCount(pageSize); + + //Validate that we get Total Count only once, and on all following pages it is skipped and Null is returned as expected! + if (totalCount is null) + { + page.TotalCount.Should().Be(expectedTotalCount); + totalCount = page.TotalCount; + TestContext.WriteLine("*********************************************************"); + TestContext.WriteLine($"[{totalCount}] Total Results to be processed..."); + TestContext.WriteLine("*********************************************************"); + } + else + { + page.TotalCount.Should().BeNull(); + } + + runningTotal += resultsList.Count; + + TestContext.WriteLine(""); + TestContext.WriteLine($"[{resultsList.Count}] Page Results:"); + TestContext.WriteLine("----------------------------------------"); + foreach (var result in resultsList) + { + var value = result.Entity; + value.Should().BeInRange(1, expectedTotalCount); + TestContext.WriteLine($"[{result.Cursor}] ==> ({value})"); + } + + } while (page.HasNextPage); + + Assert.AreEqual(totalCount, runningTotal, "Total Count doesn't Match the final running total tally!"); + } + + [TestMethod] + public void TestOffsetPagingWithInMemorySlicing() + { + var expectedTotalCount = 100; + var items = Enumerable.Range(1, expectedTotalCount).ToArray(); + + const int pageSize = 10; + int? totalCount = null; + int runningTotal = 0; + IOffsetPageResults page = null; + var pageCounter = 0; + + do + { + page = items.SliceAsOffsetPage( + skip: page?.EndIndex, + take: pageSize, + includeTotalCount: totalCount is null + ); + + pageCounter++; + page.Should().NotBeNull(); + + if (pageCounter == 1) + { + page.HasNextPage.Should().BeTrue(); + page.HasPreviousPage.Should().BeFalse(); + } + else if (pageCounter == (int)Math.Ceiling((decimal)expectedTotalCount / pageSize)) + { + page.HasNextPage.Should().BeFalse(); + page.HasPreviousPage.Should().BeTrue(); + } + else + { + page.HasNextPage.Should().BeTrue(); + page.HasPreviousPage.Should().BeTrue(); + } + + var resultsList = page.Results.ToList(); + resultsList.Should().HaveCount(pageSize); + + //Validate that we get Total Count only once, and on all following pages it is skipped and Null is returned as expected! + if (totalCount is null) + { + page.TotalCount.Should().Be(expectedTotalCount); + totalCount = page.TotalCount; + TestContext.WriteLine("*********************************************************"); + TestContext.WriteLine($"[{totalCount}] Total Results to be processed..."); + TestContext.WriteLine("*********************************************************"); + } + else + { + page.TotalCount.Should().BeNull(); + } + + runningTotal += resultsList.Count; + + TestContext.WriteLine(""); + TestContext.WriteLine($"[{resultsList.Count}] Page Results:"); + TestContext.WriteLine("----------------------------------------"); + foreach (var result in resultsList) + { + result.Should().BeInRange(1, expectedTotalCount); + TestContext.WriteLine($"[{result}]"); + } + + } while (page.HasNextPage); + + Assert.AreEqual(totalCount, runningTotal, "Total Count doesn't Match the final running total tally!"); + } + } +} \ No newline at end of file diff --git a/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingQueryApiForObjectExpressions.cs b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingQueryApiForObjectExpressions.cs index 2e6a74e..786d0fb 100644 --- a/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingQueryApiForObjectExpressions.cs +++ b/RepoDb.SqlServer.PagingOperations.Tests/PagingTestsUsingQueryApiForObjectExpressions.cs @@ -14,7 +14,7 @@ namespace RepoDb.SqlServer.PagingOperations.Tests public class PagingTestsUsingQueryApiForObjectExpressions : BaseTest { [TestMethod] - public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApi() + public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApiAsync() { using var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false); @@ -47,7 +47,7 @@ public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApi() } [TestMethod] - public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApiWithWhereExpression() + public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApiWithWhereExpressionAsync() { using var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false); @@ -81,7 +81,7 @@ public async Task SimpleExampleOfCursorPagingThroughDatasetUsingQueryApiWithWher } [TestMethod] - public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApi() + public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApiAsync() { using var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false); @@ -113,7 +113,7 @@ public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApi() } [TestMethod] - public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApiWithWhereExpression() + public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApiWithWhereExpressionAsync() { using var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false); @@ -148,7 +148,7 @@ public async Task SimpleExampleOfOffsetPagingThroughDatasetUsingQueryApiWithWher } [TestMethod] - public async Task TestCursorPagingQueryApiSyntax() + public async Task TestCursorPagingQueryApiSyntaxAsync() { using (var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false)) { @@ -206,7 +206,7 @@ public async Task TestCursorPagingQueryApiSyntax() } [TestMethod] - public async Task TestOffsetPagingQueryApiSyntax() + public async Task TestOffsetPagingQueryApiSyntaxAsync() { using (var sqlConnection = await CreateSqlConnectionAsync().ConfigureAwait(false)) { diff --git a/RepoDb.SqlServer.PagingOperations.Tests/RepoDbExtensions.SqlServer.PagingOperations.Tests.csproj b/RepoDb.SqlServer.PagingOperations.Tests/RepoDbExtensions.SqlServer.PagingOperations.Tests.csproj index 0980ab4..1d8b56a 100644 --- a/RepoDb.SqlServer.PagingOperations.Tests/RepoDbExtensions.SqlServer.PagingOperations.Tests.csproj +++ b/RepoDb.SqlServer.PagingOperations.Tests/RepoDbExtensions.SqlServer.PagingOperations.Tests.csproj @@ -22,7 +22,8 @@ - + + diff --git a/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryCursorPagingExtensions.cs b/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryCursorPagingExtensions.cs index 085ede2..80c010e 100644 --- a/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryCursorPagingExtensions.cs +++ b/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryCursorPagingExtensions.cs @@ -12,14 +12,14 @@ public static class IEnumerableInMemoryCursorPagingExtensions /// Implement Linq in-memory slicing as described by Relay spec here: /// https://relay.dev/graphql/connections.htm#sec-Pagination-algorithm /// NOTE: This is primarily used for Unit Testing of in-memory data sets and is generally not recommended for production - /// use unless you always have 100% of all your data in-memory; this is because sorting must be done on a pre-filtered and/or + /// use unless you always have 100% of all your data in-memory. But there are valid use-cases such as if data is archived + /// in compressed format and is always retrieved into memory, etc. /// /// /// /// /// - public static ICursorPageResults SliceAsCursorPage(this IEnumerable items, string after, int? first, string before, int? last) - where T : class + public static ICursorPageResults SliceAsCursorPage(this IEnumerable items, string after, int? first, string before, int? last, bool includeTotalCount = true) { //Do nothing if there are no results... if (!items.Any()) @@ -39,7 +39,12 @@ public static ICursorPageResults SliceAsCursorPage(this IEnumerable ite //NOTE: We MUST materialize this after applying index values to prevent ongoing increments... IEnumerable> slice = items - .Select((item, index) => CursorResult.CreateIndexedCursor(item, CursorFactory.CreateCursor(index), index)) + .Select((item, index) => + { + //NOTE: We must increment index+1 to ensure our indexes are 1 based to simplify the rest of the algorithm... + var cursorIndex = index + 1; + return CursorResult.CreateIndexedCursor(item, CursorFactory.CreateCursor(cursorIndex), cursorIndex); + }) .ToList(); int totalCount = slice.Count(); @@ -76,7 +81,7 @@ public static ICursorPageResults SliceAsCursorPage(this IEnumerable ite var cursorPageSlice = new CursorPageResults( results, - totalCount, + includeTotalCount ? totalCount : (int?)null, hasPreviousPage: firstCursor?.CursorIndex > 1, hasNextPage: lastCursor?.CursorIndex < totalCount ); diff --git a/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryOffsetPagingExtensions.cs b/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryOffsetPagingExtensions.cs index ec312ef..08e4a19 100644 --- a/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryOffsetPagingExtensions.cs +++ b/RepoDb.SqlServer.PagingOperations/InMemoryPaging/IEnumerableInMemoryOffsetPagingExtensions.cs @@ -11,7 +11,8 @@ public static class IEnumerableInMemoryOffsetPagingExtensions /// Implement Linq in-memory slicing as described by Relay spec here: /// https://relay.dev/graphql/connections.htm#sec-Pagination-algorithm /// NOTE: This is primarily used for Unit Testing of in-memory data sets and is generally not recommended for production - /// use unless you always have 100% of all your data in-memory; this is because sorting must be done on a pre-filtered and/or + /// use unless you always have 100% of all your data in-memory. But there are valid use-cases such as if data is archived + /// in compressed format and is always retrieved into memory, etc. /// /// /// @@ -20,7 +21,6 @@ public static class IEnumerableInMemoryOffsetPagingExtensions /// /// public static IOffsetPageResults SliceAsOffsetPage(this IEnumerable items, int? skip, int? take, bool includeTotalCount = true) - where T : class { //Do nothing if there are no results... if (items?.Any() != true) @@ -40,7 +40,7 @@ public static IOffsetPageResults SliceAsOffsetPage(this IEnumerable ite if (hasNextPage) pagedResults.Remove(pagedResults.Last()); //NOTE: We only materialize the FULL Count if actually requested to do so... - var totalCount = includeTotalCount ? (int?)items.Count() : null; + var totalCount = includeTotalCount ? items.Count() : (int?)null; //Wrap all results into a Offset Page Slice result with Total Count... return new OffsetPageResults( diff --git a/RepoDb.SqlServer.PagingOperations/RepoDbExtensions.SqlServer.PagingOperations.csproj b/RepoDb.SqlServer.PagingOperations/RepoDbExtensions.SqlServer.PagingOperations.csproj index 2b87a6c..a1ce914 100644 --- a/RepoDb.SqlServer.PagingOperations/RepoDbExtensions.SqlServer.PagingOperations.csproj +++ b/RepoDb.SqlServer.PagingOperations/RepoDbExtensions.SqlServer.PagingOperations.csproj @@ -2,9 +2,9 @@ netstandard2.0;netstandard2.1;net6.0; - 1.1.5.5 - 1.1.5.5 - 1.1.5.5 + 1.1.5.6 + 1.1.5.6 + 1.1.5.6 BBernard / CajunCoding CajunCoding A set of extensions for working with modern pagination approaches such as Cursor based paging, as well as Offset based pagination, using the RepoDb ORM with Sql Server. @@ -15,9 +15,11 @@ repodb, paging, pagination, cursor, offset, skip, take, sorting, graphql, graph-ql, hotchocolate, dapper, sqlkata Release Notes: - - Improve flexibility of base interface support for cursor navigation with non-generic ICursorPageNavigationInfo. + - Fix bug with in-memory cursor paging logic incorrectly indexing the results resulting in invalid page results. + - Removed unnecessary class constraint for both cursor and offset in-memory paging extensions making them more flexible. Prior Release Notes: + - Improve flexibility of base interface support for cursor navigation with non-generic ICursorPageNavigationInfo. - Simplify RetrieveTotalCount param name on ICursorParams. - Fix SQL building bug not using Raw SQL as specified; improved integration tests to validate this better. - Improved raw sql parameter name to be more consistent with RepDb naming conventions.