From 98b3634d1e63e5986848512e70f74d0f2e26ebba Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Wed, 25 Sep 2024 15:39:31 +0100 Subject: [PATCH 01/15] feat: Implement custom SQL caching --- .../Dfe.PlanTech.Application.csproj | 2 + .../Extensions/QueryableExtensions.cs | 52 +++++++++++++++++++ .../CmsDbContext.cs | 10 ++-- .../Repositories/RecommendationsRepository.cs | 15 +++--- .../Controllers/CacheController.cs | 6 +-- src/Dfe.PlanTech.Web/Dfe.PlanTech.Web.csproj | 1 - src/Dfe.PlanTech.Web/ProgramExtensions.cs | 9 ---- .../Controllers/CacheControllerTests.cs | 40 +++++++------- 8 files changed, 91 insertions(+), 44 deletions(-) create mode 100644 src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs diff --git a/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj b/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj index 1ff542c0e..9efb7a631 100644 --- a/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj +++ b/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj @@ -10,6 +10,8 @@ + + diff --git a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs new file mode 100644 index 000000000..c35a35a28 --- /dev/null +++ b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs @@ -0,0 +1,52 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; + +namespace Dfe.PlanTech.Application.Extensions; + +/// +/// Extension methods for caching database commands by QueryString +/// These do not have any cache invalidation and should only be used on CMSDbContext queries, not PlanTechDbContext +/// +public static class QueryableExtensions +{ + private static MemoryCache _cache = new(new MemoryCacheOptions()); + private const int CacheDurationMinutes = 30; + + private static string GetCacheKey(IQueryable query) + { + var queryString = query.ToQueryString(); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(queryString)); + return Convert.ToBase64String(hash); + } + + private static async Task GetOrCreateAsyncWithCache( + IQueryable queryable, + Func, CancellationToken, Task> queryFunc, + CancellationToken cancellationToken = default) + { + var key = GetCacheKey(queryable); + return await _cache.GetOrCreateAsync(key, cacheEntry => + { + cacheEntry.AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(CacheDurationMinutes); + return queryFunc(queryable, cancellationToken); + }) ?? await queryFunc(queryable, cancellationToken); + } + + public static Task> ToListAsyncWithCache(this IQueryable queryable, CancellationToken cancellationToken = default) + { + return GetOrCreateAsyncWithCache(queryable, (q, ctoken) => q.ToListAsync(ctoken), cancellationToken); + } + + public static Task FirstOrDefaultAsyncWithCache(this IQueryable queryable, CancellationToken cancellationToken = default) + { + return GetOrCreateAsyncWithCache(queryable, (q, ctoken) => q.FirstOrDefaultAsync(ctoken), cancellationToken); + } + + public static void ClearCmsCache() + { + _cache.Dispose(); + _cache = new MemoryCache(new MemoryCacheOptions()); + } +} diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index 41867e45e..c4b5a0928 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -1,6 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; using Dfe.PlanTech.Application.Persistence.Interfaces; +using Dfe.PlanTech.Application.Extensions; using Dfe.PlanTech.Domain.Content.Models; using Dfe.PlanTech.Domain.Content.Models.Buttons; using Dfe.PlanTech.Domain.Exceptions; @@ -215,15 +216,16 @@ private Expression> ShouldShowEntity() => entity => (_contentfulOptions.UsePreview || entity.Published) && !entity.Archived && !entity.Deleted; public Task GetPageBySlug(string slug, CancellationToken cancellationToken = default) - => Pages.Include(page => page.BeforeTitleContent) + => Pages.Where(page => page.Slug == slug) + .Include(page => page.BeforeTitleContent) .Include(page => page.Content) .Include(page => page.Title) .AsSplitQuery() - .FirstOrDefaultAsync(page => page.Slug == slug, cancellationToken); + .FirstOrDefaultAsyncWithCache(cancellationToken); public Task> ToListAsync(IQueryable queryable, CancellationToken cancellationToken = default) => - queryable.ToListAsync(cancellationToken: cancellationToken); + queryable.ToListAsyncWithCache(cancellationToken: cancellationToken); public Task FirstOrDefaultAsync(IQueryable queryable, CancellationToken cancellationToken = default) - => queryable.FirstOrDefaultAsync(cancellationToken); + => queryable.FirstOrDefaultAsyncWithCache(cancellationToken); } diff --git a/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs b/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs index 57c8f3cd0..a5d1274bd 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs @@ -1,3 +1,4 @@ +using Dfe.PlanTech.Application.Extensions; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Questionnaire.Interfaces; using Dfe.PlanTech.Domain.Questionnaire.Models; @@ -28,7 +29,7 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger intro.SubtopicRecommendations.Any(subtopicRec => subtopicRec.Id == recommendation.Id)) .Include(intro => intro.Header) - .ToListAsync(cancellationToken); + .ToListAsyncWithCache(cancellationToken); var chunks = await _db.RecommendationChunks.Where(chunk => chunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId)) .Select(chunk => new RecommendationChunkDbEntity() @@ -49,7 +50,7 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger chunk.Order) - .ToListAsync(cancellationToken); + .ToListAsyncWithCache(cancellationToken); var introContent = await _db.RecommendationIntroContents.Where(introContent => introContent.RecommendationIntro != null && introContent.RecommendationIntro.SubtopicRecommendations.Any(rec => rec.Id == recommendation.Id)) @@ -60,7 +61,7 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger chunkContent.RecommendationChunk != null && chunkContent.RecommendationChunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId)) @@ -71,14 +72,14 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger rt.SubtopicRecommendationId == recommendation.Id) - .ToListAsync(cancellationToken); + .ToListAsyncWithCache(cancellationToken); return new SubtopicRecommendationDbEntity() { @@ -117,7 +118,7 @@ await _db.RichTextContentWithSubtopicRecommendationIds => _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId) .Select(subtopicRecommendation => subtopicRecommendation.Intros.FirstOrDefault(intro => intro.Maturity == maturity)) .Select(intro => intro != null ? new RecommendationsViewDto(intro.Slug, intro.Header.Text) : null) - .FirstOrDefaultAsync(cancellationToken: cancellationToken); + .FirstOrDefaultAsyncWithCache(cancellationToken: cancellationToken); /// /// Check for invalid join rows, and log any errored rows. diff --git a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs index 7fbcc794a..200df998f 100644 --- a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs +++ b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs @@ -1,6 +1,6 @@ using Dfe.PlanTech.Web.Helpers; -using EFCoreSecondLevelCacheInterceptor; using Microsoft.AspNetCore.Mvc; +using Dfe.PlanTech.Application.Extensions; namespace Dfe.PlanTech.Web.Controllers; @@ -10,11 +10,11 @@ public class CacheController(ILogger cacheLogger) : BaseControl { [HttpPost("clear")] [ValidateApiKey] - public IActionResult ClearCache([FromServices] IEFCacheServiceProvider cacheServiceProvider) + public IActionResult ClearCache() { try { - cacheServiceProvider.ClearAllCachedEntries(); + QueryableExtensions.ClearCmsCache(); logger.LogInformation("Database cache has been cleared"); return Ok(true); } diff --git a/src/Dfe.PlanTech.Web/Dfe.PlanTech.Web.csproj b/src/Dfe.PlanTech.Web/Dfe.PlanTech.Web.csproj index 4f04058b7..fc0da49df 100644 --- a/src/Dfe.PlanTech.Web/Dfe.PlanTech.Web.csproj +++ b/src/Dfe.PlanTech.Web/Dfe.PlanTech.Web.csproj @@ -26,7 +26,6 @@ - diff --git a/src/Dfe.PlanTech.Web/ProgramExtensions.cs b/src/Dfe.PlanTech.Web/ProgramExtensions.cs index 56060c29c..ace5ff142 100644 --- a/src/Dfe.PlanTech.Web/ProgramExtensions.cs +++ b/src/Dfe.PlanTech.Web/ProgramExtensions.cs @@ -33,7 +33,6 @@ using Dfe.PlanTech.Web.Helpers; using Dfe.PlanTech.Web.Middleware; using Dfe.PlanTech.Web.Routing; -using EFCoreSecondLevelCacheInterceptor; using Microsoft.AspNetCore.Authorization; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; @@ -141,16 +140,8 @@ public static IServiceCollection AddDatabase(this IServiceCollection services, I .CommandTimeout((int)TimeSpan.FromSeconds(30).TotalSeconds) .EnableRetryOnFailure(); }) - .AddInterceptors(serviceProvider.GetRequiredService()) ); - services.AddEFSecondLevelCache(options => - { - options.UseMemoryCacheProvider().ConfigureLogging(false).UseCacheKeyPrefix("EF_"); - options.CacheAllQueries(CacheExpirationMode.Absolute, TimeSpan.FromMinutes(30)); - options.UseDbCallsIfCachingProviderIsDown(TimeSpan.FromMinutes(1)); - }); - services.AddDbContext(databaseOptionsAction); ConfigureCookies(services, configuration); diff --git a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs index fb2562e20..837017483 100644 --- a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs +++ b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs @@ -1,6 +1,8 @@ -using Dfe.PlanTech.Web.Controllers; -using EFCoreSecondLevelCacheInterceptor; +using Dfe.PlanTech.Application.Extensions; +using Dfe.PlanTech.Domain.Content.Models; +using Dfe.PlanTech.Web.Controllers; using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; @@ -9,40 +11,38 @@ namespace Dfe.PlanTech.Web.UnitTests.Controllers; public class CacheControllerTests { - private readonly IEFCacheServiceProvider _cacheServiceProvider = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); private readonly CacheController _cacheController; + private readonly IQueryable _mockQueryable = Substitute.For>(); + private int _queryCallCount; + public CacheControllerTests() { _cacheController = new CacheController(_logger); + + _mockQueryable + .When(query => query.ToListAsync()) + .Do(_ => _queryCallCount++ ); } [Fact] - public void ClearCache_Should_Return_True_On_Success() + public void ClearCache_Should_Empty_Cache() { - var clearCacheResult = _cacheController.ClearCache(_cacheServiceProvider); + _mockQueryable.ToListAsyncWithCache(); + _mockQueryable.ToListAsyncWithCache(); + + Assert.Equal(1, _queryCallCount); + + var clearCacheResult = _cacheController.ClearCache(); Assert.NotNull(clearCacheResult); var result = clearCacheResult as ObjectResult; Assert.NotNull(result); Assert.Equal(200, result.StatusCode); - _cacheServiceProvider.Received(1).ClearAllCachedEntries(); - } - - [Fact] - public void ClearCache_Should_Return_False_On_Failure() - { - _cacheServiceProvider - .When(call => call.ClearAllCachedEntries()) - .Do(_ => throw new Exception("unexpected error")); - - var clearCacheResult = _cacheController.ClearCache(_cacheServiceProvider); - Assert.NotNull(clearCacheResult); + _mockQueryable.ToListAsyncWithCache(); - var result = clearCacheResult as ObjectResult; - Assert.NotNull(result); - Assert.Equal(500, result.StatusCode); + Assert.Equal(2, _queryCallCount); } } From 733a65cf4adc25d9a57e909c8df3cf49dc894340 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Wed, 25 Sep 2024 15:54:05 +0100 Subject: [PATCH 02/15] chore: use relative expiration --- .../Extensions/QueryableExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs index c35a35a28..39b27c8e9 100644 --- a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs +++ b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs @@ -27,9 +27,9 @@ private static async Task GetOrCreateAsyncWithCache( CancellationToken cancellationToken = default) { var key = GetCacheKey(queryable); - return await _cache.GetOrCreateAsync(key, cacheEntry => + return await _cache.GetOrCreateAsync(key, entry => { - cacheEntry.AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(CacheDurationMinutes); + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(CacheDurationMinutes); return queryFunc(queryable, cancellationToken); }) ?? await queryFunc(queryable, cancellationToken); } From b19e66f57301d2b11b7eec1c97c1df7fb95d798b Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Wed, 25 Sep 2024 16:12:08 +0100 Subject: [PATCH 03/15] docs: caching docs --- docs/Contentful-Caching-Process.md | 12 ++++++------ docs/Conventions.md | 11 ++++++++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/Contentful-Caching-Process.md b/docs/Contentful-Caching-Process.md index 19cee4739..e5cb2ad65 100644 --- a/docs/Contentful-Caching-Process.md +++ b/docs/Contentful-Caching-Process.md @@ -41,7 +41,7 @@ - We normalise the incoming entry JSON. This involves essentially just copying the children within the "fields" object to a new JSON object, along with the "id" field from the "sys" object. - The end result of this is a JSON that should match the actual Contentful classes we use one-to-one. - We then deserialise this JSON to the appropriate database class for the content type -- We retrieve the existing entity from our database, if it exists, using the id. +- We retrieve the existing entity from our database, if it exists, using the id. - If it does exist, we use reflection to copy over the values from the _incoming_ mapped entity, to the found _existing_ entity. - Certain properties are ignored during this, for a variety of reasons (e.g. they might be metadata, meaning they wouldn't exist on in the incoming data, which could cause errors or incorrect data copied over) - This is done by using our custom [DontCopyValueAttribute](./src/Dfe.PlanTech.Domain/DontCopyValueAttribute.cs) on each property we do not wish to copy, and checking for its existance per property @@ -81,11 +81,11 @@ There are _some_ custom mappings done using LINQ `select` projections, due to va There are several steps to it: 1. We retrieve the `Page` matching the Slug from the table, and Include all `BeforeTitleContent` and `Content`. Note: there are various AutoIncludes defined in the [/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs](/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs) for the majority of the content tables, to ensure all data is pulled in. - + 2. Due to various issues with certain navigations, we execute various other queries and then merge the data together. These are completed using various `IGetPageChildrenQuery` objects that are injected into the `GetPageQuery` in the constructor using DI. - If there is any content which has a `RichTextContent` property (using the `IHasText` interface to check), we execute a query to retrieve all the `RichTextContent` for the Page. The `RichTextMark` and `RichTextData` tables are joined automatically. This is handled by the [GetRichTextsQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetRichTextsQuery.cs) class. - + - If there is any `ButtonWithEntryReference` content, then we retrieve the `LinkToEntry` property from it manually. This is to ensure we do not have a cartesian explosion (i.e. retrieving that entire page, + content, etc.), and to minimise the data we retrieve from the database (we only retrieve the slug and Id fields for the `LinkToEntry` property). This is handled by the [GetButtonWithEntryReferencesQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetButtonWithEntryReferencesQuery.cs) class. - If there are any `Category` content, we retrieve the `Sections` for them manually. This is because we also need to retrieve the `Question`s and `Recommendation`s for each Section, but only certain pieces of information. To prevent execessive data being retrieved, we only query the necessary fields. This is handled by the [GetCategorySectionsQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetCategorySectionsQuery.cs) class. @@ -94,6 +94,6 @@ There are several steps to it: ## Caching -- Caching is handled by the open-source [EFCoreSecondLevelCacheInterceptor](https://github.com/VahidN/EFCoreSecondLevelCacheInterceptor) C# package. -- It is enabled only in the [web project](./src/Dfe.PlanTech.Web), and is enabled in the services configuration in [ProgramExtensions.cs](./src/Dfe.PlanTech.Web/ProgramExtensions.cs). We currently have no functionality setup to amend the configuration (e.g. caching length) via any sort of environment variables, but this should be added when possible. -- The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. \ No newline at end of file +- Caching is handled by an in memory cache defined in the QueryableExtensions class in [Dfe.PlanTech.Application](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). +- It does not have any form of cache invalidation as it is only intended for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. +- The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. diff --git a/docs/Conventions.md b/docs/Conventions.md index 6113a2a63..c079e567b 100644 --- a/docs/Conventions.md +++ b/docs/Conventions.md @@ -29,4 +29,13 @@ ```html
This element will be hidden by default, but made visible via JS.
``` - - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml) \ No newline at end of file + - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml) + +## EF Query Conventions +- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in in [Dfe.PlanTech.Application](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). +- Any queries using the `CmsDbContext` should be cached: + - either by using the `db.ToListAsync` or `db.FirstOrDefaultAsync` methods from `CmsDbContext` + - or by using the `ToListAsyncWithCache` and `FirstOrDefaultAsyncWithCache` extensions from `QueryableExtensions` + - the cache can be cleared with the `ClearCmsCache` method which clears everything +- These extensions should not be used for any `PlanTechDbContext` queries as these tables are frequently updated +- More about contentful caching is explained in [Contentful-Caching-Process](/docs/Contentful-Caching-Process.md) From cc5e5163909a317265b61c9bc4ff01c4dbcf3606 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:19:13 +0000 Subject: [PATCH 04/15] chore: Linted code for plan-technology-for-your-school.sln solution --- src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs | 2 +- src/Dfe.PlanTech.Web/Controllers/CacheController.cs | 2 +- .../Controllers/CacheControllerTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index c4b5a0928..d6013bf34 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -1,7 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; -using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Application.Extensions; +using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Content.Models; using Dfe.PlanTech.Domain.Content.Models.Buttons; using Dfe.PlanTech.Domain.Exceptions; diff --git a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs index 200df998f..0792fbba7 100644 --- a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs +++ b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs @@ -1,6 +1,6 @@ +using Dfe.PlanTech.Application.Extensions; using Dfe.PlanTech.Web.Helpers; using Microsoft.AspNetCore.Mvc; -using Dfe.PlanTech.Application.Extensions; namespace Dfe.PlanTech.Web.Controllers; diff --git a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs index 837017483..634dc1f04 100644 --- a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs +++ b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs @@ -23,7 +23,7 @@ public CacheControllerTests() _mockQueryable .When(query => query.ToListAsync()) - .Do(_ => _queryCallCount++ ); + .Do(_ => _queryCallCount++); } [Fact] From d221d6cf49a927af489fd2966f8dd654206d905d Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Wed, 25 Sep 2024 16:29:08 +0100 Subject: [PATCH 05/15] docs: fix readme links --- docs/Contentful-Caching-Process.md | 2 +- docs/Conventions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Contentful-Caching-Process.md b/docs/Contentful-Caching-Process.md index e5cb2ad65..9f4472d2d 100644 --- a/docs/Contentful-Caching-Process.md +++ b/docs/Contentful-Caching-Process.md @@ -94,6 +94,6 @@ There are several steps to it: ## Caching -- Caching is handled by an in memory cache defined in the QueryableExtensions class in [Dfe.PlanTech.Application](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). +- Caching is handled by an in memory cache defined in the QueryableExtensions class in [Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs) - It does not have any form of cache invalidation as it is only intended for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. - The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. diff --git a/docs/Conventions.md b/docs/Conventions.md index c079e567b..52010f8d0 100644 --- a/docs/Conventions.md +++ b/docs/Conventions.md @@ -32,7 +32,7 @@ - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml) ## EF Query Conventions -- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in in [Dfe.PlanTech.Application](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). +- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in [Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). - Any queries using the `CmsDbContext` should be cached: - either by using the `db.ToListAsync` or `db.FirstOrDefaultAsync` methods from `CmsDbContext` - or by using the `ToListAsyncWithCache` and `FirstOrDefaultAsyncWithCache` extensions from `QueryableExtensions` From e96091d8e508ca634940bd2bc4bb6835071ecd59 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Fri, 27 Sep 2024 16:49:21 +0100 Subject: [PATCH 06/15] fix: remove ef from application --- .../Caching/Interfaces/IQueryCacher.cs | 12 +++++ .../Dfe.PlanTech.Application.csproj | 1 - .../Extensions/QueryableExtensions.cs | 37 +++----------- .../CmsDbContext.cs | 48 ++++++++++++++----- .../Controllers/CacheController.cs | 6 +-- .../Controllers/CacheControllerTests.cs | 39 +++++++-------- 6 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 src/Dfe.PlanTech.Application/Caching/Interfaces/IQueryCacher.cs diff --git a/src/Dfe.PlanTech.Application/Caching/Interfaces/IQueryCacher.cs b/src/Dfe.PlanTech.Application/Caching/Interfaces/IQueryCacher.cs new file mode 100644 index 000000000..59d9e3b55 --- /dev/null +++ b/src/Dfe.PlanTech.Application/Caching/Interfaces/IQueryCacher.cs @@ -0,0 +1,12 @@ +namespace Dfe.PlanTech.Application.Caching.Interfaces; + +public interface IQueryCacher +{ + public Task GetOrCreateAsyncWithCache( + string key, + IQueryable queryable, + Func, CancellationToken, Task> queryFunc, + CancellationToken cancellationToken = default); + + public void ClearCache(); +} diff --git a/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj b/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj index 9efb7a631..5ac3b1345 100644 --- a/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj +++ b/src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs index 39b27c8e9..5f53c979f 100644 --- a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs +++ b/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs @@ -1,32 +1,19 @@ -using System.Security.Cryptography; -using System.Text; -using Microsoft.EntityFrameworkCore; +using Dfe.PlanTech.Application.Caching.Interfaces; using Microsoft.Extensions.Caching.Memory; -namespace Dfe.PlanTech.Application.Extensions; +namespace Dfe.PlanTech.Application.Caching.Models; -/// -/// Extension methods for caching database commands by QueryString -/// These do not have any cache invalidation and should only be used on CMSDbContext queries, not PlanTechDbContext -/// -public static class QueryableExtensions +public class QueryCacher: IQueryCacher { - private static MemoryCache _cache = new(new MemoryCacheOptions()); + private MemoryCache _cache = new(new MemoryCacheOptions()); private const int CacheDurationMinutes = 30; - private static string GetCacheKey(IQueryable query) - { - var queryString = query.ToQueryString(); - var hash = SHA256.HashData(Encoding.UTF8.GetBytes(queryString)); - return Convert.ToBase64String(hash); - } - - private static async Task GetOrCreateAsyncWithCache( + public async Task GetOrCreateAsyncWithCache( + string key, IQueryable queryable, Func, CancellationToken, Task> queryFunc, CancellationToken cancellationToken = default) { - var key = GetCacheKey(queryable); return await _cache.GetOrCreateAsync(key, entry => { entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(CacheDurationMinutes); @@ -34,17 +21,7 @@ private static async Task GetOrCreateAsyncWithCache( }) ?? await queryFunc(queryable, cancellationToken); } - public static Task> ToListAsyncWithCache(this IQueryable queryable, CancellationToken cancellationToken = default) - { - return GetOrCreateAsyncWithCache(queryable, (q, ctoken) => q.ToListAsync(ctoken), cancellationToken); - } - - public static Task FirstOrDefaultAsyncWithCache(this IQueryable queryable, CancellationToken cancellationToken = default) - { - return GetOrCreateAsyncWithCache(queryable, (q, ctoken) => q.FirstOrDefaultAsync(ctoken), cancellationToken); - } - - public static void ClearCmsCache() + public void ClearCache() { _cache.Dispose(); _cache = new MemoryCache(new MemoryCacheOptions()); diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index d6013bf34..bcae8ab6f 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -1,6 +1,9 @@ using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; -using Dfe.PlanTech.Application.Extensions; +using System.Security.Cryptography; +using System.Text; +using Dfe.PlanTech.Application.Caching.Interfaces; +using Dfe.PlanTech.Application.Caching.Models; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Content.Models; using Dfe.PlanTech.Domain.Content.Models.Buttons; @@ -115,15 +118,18 @@ public class CmsDbContext : DbContext, ICmsDbContext #endregion private readonly ContentfulOptions _contentfulOptions; + private readonly QueryCacher _queryCacher; public CmsDbContext() { _contentfulOptions = new ContentfulOptions(false); + _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); } public CmsDbContext(DbContextOptions options) : base(options) { _contentfulOptions = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(ContentfulOptions)}"); + _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); } protected override void OnModelCreating(ModelBuilder modelBuilder) @@ -215,17 +221,33 @@ public virtual Task SetComponentPublishedAndDeletedStatuses(ContentComponen private Expression> ShouldShowEntity() => entity => (_contentfulOptions.UsePreview || entity.Published) && !entity.Archived && !entity.Deleted; + private static string GetCacheKey(IQueryable query) + { + var queryString = query.ToQueryString(); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(queryString)); + return Convert.ToBase64String(hash); + } + public Task GetPageBySlug(string slug, CancellationToken cancellationToken = default) - => Pages.Where(page => page.Slug == slug) - .Include(page => page.BeforeTitleContent) - .Include(page => page.Content) - .Include(page => page.Title) - .AsSplitQuery() - .FirstOrDefaultAsyncWithCache(cancellationToken); - - public Task> ToListAsync(IQueryable queryable, CancellationToken cancellationToken = default) => - queryable.ToListAsyncWithCache(cancellationToken: cancellationToken); - - public Task FirstOrDefaultAsync(IQueryable queryable, CancellationToken cancellationToken = default) - => queryable.FirstOrDefaultAsyncWithCache(cancellationToken); + => FirstOrDefaultAsync( + Pages.Where(page => page.Slug == slug) + .Include(page => page.BeforeTitleContent) + .Include(page => page.Content) + .Include(page => page.Title) + .AsSplitQuery(), + cancellationToken); + + public async Task> ToListAsync(IQueryable queryable, CancellationToken cancellationToken = default) + { + var key = GetCacheKey(queryable); + return await _queryCacher.GetOrCreateAsyncWithCache(key, queryable, + (q, ctoken) => q.ToListAsync(ctoken), cancellationToken); + } + + public async Task FirstOrDefaultAsync(IQueryable queryable, CancellationToken cancellationToken = default) + { + var key = GetCacheKey(queryable); + return await _queryCacher.GetOrCreateAsyncWithCache(key, queryable, + (q, ctoken) => q.FirstOrDefaultAsync(ctoken), cancellationToken); + } } diff --git a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs index 0792fbba7..1efeb278f 100644 --- a/src/Dfe.PlanTech.Web/Controllers/CacheController.cs +++ b/src/Dfe.PlanTech.Web/Controllers/CacheController.cs @@ -1,4 +1,4 @@ -using Dfe.PlanTech.Application.Extensions; +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Web.Helpers; using Microsoft.AspNetCore.Mvc; @@ -6,7 +6,7 @@ namespace Dfe.PlanTech.Web.Controllers; [Route("cache")] [LogInvalidModelState] -public class CacheController(ILogger cacheLogger) : BaseController(cacheLogger) +public class CacheController([FromServices] IQueryCacher queryCacher, ILogger cacheLogger) : BaseController(cacheLogger) { [HttpPost("clear")] [ValidateApiKey] @@ -14,7 +14,7 @@ public IActionResult ClearCache() { try { - QueryableExtensions.ClearCmsCache(); + queryCacher.ClearCache(); logger.LogInformation("Database cache has been cleared"); return Ok(true); } diff --git a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs index 634dc1f04..d9b03a427 100644 --- a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs +++ b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs @@ -1,8 +1,6 @@ -using Dfe.PlanTech.Application.Extensions; -using Dfe.PlanTech.Domain.Content.Models; +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Web.Controllers; using Microsoft.AspNetCore.Mvc; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; @@ -11,38 +9,41 @@ namespace Dfe.PlanTech.Web.UnitTests.Controllers; public class CacheControllerTests { + private readonly IQueryCacher _queryCacher = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); private readonly CacheController _cacheController; - private readonly IQueryable _mockQueryable = Substitute.For>(); - private int _queryCallCount; - public CacheControllerTests() { - _cacheController = new CacheController(_logger); - - _mockQueryable - .When(query => query.ToListAsync()) - .Do(_ => _queryCallCount++); + _cacheController = new CacheController(_queryCacher, _logger); } [Fact] - public void ClearCache_Should_Empty_Cache() + public void ClearCache_Should_Return_True_On_Success() { - _mockQueryable.ToListAsyncWithCache(); - _mockQueryable.ToListAsyncWithCache(); - - Assert.Equal(1, _queryCallCount); - var clearCacheResult = _cacheController.ClearCache(); + Assert.NotNull(clearCacheResult); var result = clearCacheResult as ObjectResult; Assert.NotNull(result); Assert.Equal(200, result.StatusCode); - _mockQueryable.ToListAsyncWithCache(); + _queryCacher.Received(1).ClearCache(); + } + + [Fact] + public void ClearCache_Should_Return_False_On_Failure() + { + _queryCacher + .When(call => call.ClearCache()) + .Do(_ => throw new Exception("unexpected error")); - Assert.Equal(2, _queryCallCount); + var clearCacheResult = _cacheController.ClearCache(); + Assert.NotNull(clearCacheResult); + + var result = clearCacheResult as ObjectResult; + Assert.NotNull(result); + Assert.Equal(500, result.StatusCode); } } From 7cfacce109a9cc9e1caa6f532f6dea4d40b67e73 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Mon, 30 Sep 2024 11:55:31 +0100 Subject: [PATCH 07/15] tests: fix repository tests --- .../RecommendationsRepositoryTests.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/Repositories/RecommendationsRepositoryTests.cs b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/Repositories/RecommendationsRepositoryTests.cs index f0c0dd1a9..67aaf2beb 100644 --- a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/Repositories/RecommendationsRepositoryTests.cs +++ b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/Repositories/RecommendationsRepositoryTests.cs @@ -4,7 +4,10 @@ using Dfe.PlanTech.Domain.Questionnaire.Interfaces; using Dfe.PlanTech.Domain.Questionnaire.Models; using Dfe.PlanTech.Infrastructure.Data.Repositories; +using Dfe.PlanTech.Questionnaire.Models; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using MockQueryable.EntityFrameworkCore; using MockQueryable.NSubstitute; using NSubstitute; @@ -78,30 +81,46 @@ public RecommendationsRepositoryTests() .. _subtopicRecommendation.Section.Chunks.SelectMany(chunk => chunk.Answers) ]); - var mockContext = Substitute.For(); - var subtopicRecDbSetMock = _subtopicRecommendations.BuildMock(); _db.SubtopicRecommendations.Returns(subtopicRecDbSetMock); + SetupMockQueryable(); var sectionsDbSetMock = _sections.BuildMock(); _db.Sections.Returns(sectionsDbSetMock); + SetupMockQueryable(); var introsMockSet = _intros.BuildMock(); _db.RecommendationIntros.Returns(introsMockSet); + SetupMockQueryable(); var chunksMockSet = _chunks.BuildMock(); _db.RecommendationChunks.Returns(chunksMockSet); + SetupMockQueryable(); var introContentMock = _introContent.BuildMock(); _db.RecommendationIntroContents.Returns(introContentMock); + SetupMockQueryable(); var chunkContentMock = _chunkContent.BuildMock(); _db.RecommendationChunkContents.Returns(chunkContentMock); + SetupMockQueryable(); var richTextsMock = _richTexts.BuildMock(); _db.RichTextContentWithSubtopicRecommendationIds.Returns(richTextsMock); + SetupMockQueryable(); _repository = new RecommendationsRepository(_db, _logger); + + _db.FirstOrDefaultAsync(Arg.Any>()) + .Returns(args => ((TestAsyncEnumerableEfCore)args[0]).FirstOrDefaultAsync()); + } + + private void SetupMockQueryable() + { + _db.ToListAsync(Arg.Any>()) + .Returns(args => ((IQueryable)args[0]).ToListAsync()); + _db.FirstOrDefaultAsync(Arg.Any>()) + .Returns(args => ((IQueryable)args[0]).FirstOrDefaultAsync()); } private static SubtopicRecommendationDbEntity CreateSubtopicRecommendationDbEntity() From 7a9aa603164bd2d2b21810a1f4da383c596531c8 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Mon, 30 Sep 2024 12:19:55 +0100 Subject: [PATCH 08/15] fix: cherry picked changes and documentation --- docs/Contentful-Caching-Process.md | 4 +- docs/Conventions.md | 16 ++- .../Models/QueryCacher.cs} | 2 +- .../Repositories/RecommendationsRepository.cs | 126 ++++++++++-------- src/Dfe.PlanTech.Web/ProgramExtensions.cs | 1 + 5 files changed, 81 insertions(+), 68 deletions(-) rename src/Dfe.PlanTech.Application/{Extensions/QueryableExtensions.cs => Caching/Models/QueryCacher.cs} (95%) diff --git a/docs/Contentful-Caching-Process.md b/docs/Contentful-Caching-Process.md index 9f4472d2d..f44b90ac6 100644 --- a/docs/Contentful-Caching-Process.md +++ b/docs/Contentful-Caching-Process.md @@ -94,6 +94,6 @@ There are several steps to it: ## Caching -- Caching is handled by an in memory cache defined in the QueryableExtensions class in [Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs) -- It does not have any form of cache invalidation as it is only intended for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. +- Caching is handled by an in memory cache defined in the QueryCacher class in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs) +- The only cache invalidation is invalidation of the whole cache as it intended only for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. - The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. diff --git a/docs/Conventions.md b/docs/Conventions.md index 52010f8d0..3d1a3bd11 100644 --- a/docs/Conventions.md +++ b/docs/Conventions.md @@ -32,10 +32,14 @@ - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml) ## EF Query Conventions -- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in [Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs](/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs). -- Any queries using the `CmsDbContext` should be cached: - - either by using the `db.ToListAsync` or `db.FirstOrDefaultAsync` methods from `CmsDbContext` - - or by using the `ToListAsyncWithCache` and `FirstOrDefaultAsyncWithCache` extensions from `QueryableExtensions` - - the cache can be cleared with the `ClearCmsCache` method which clears everything -- These extensions should not be used for any `PlanTechDbContext` queries as these tables are frequently updated +- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs). +- Any queries using the `CmsDbContext` should be cached by using the `db.ToListAsync` or `db.FirstOrDefaultAsync` methods from `CmsDbContext` rather than extension methods + - For example + ```csharp + // don't do this + db.RecommendationChunks.Where(condition).FirstOrDefaultAsync(cancellationToken); + // do this instead + db.FirstOrDefaultAsync(db.RecommendationChunks.Where(condition), cancellationToken); + ``` +- the cache can be cleared with the `ClearCmsCache` method which clears everything - More about contentful caching is explained in [Contentful-Caching-Process](/docs/Contentful-Caching-Process.md) diff --git a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs b/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs similarity index 95% rename from src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs rename to src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs index 5f53c979f..2737063df 100644 --- a/src/Dfe.PlanTech.Application/Extensions/QueryableExtensions.cs +++ b/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs @@ -3,7 +3,7 @@ namespace Dfe.PlanTech.Application.Caching.Models; -public class QueryCacher: IQueryCacher +public class QueryCacher : IQueryCacher { private MemoryCache _cache = new(new MemoryCacheOptions()); private const int CacheDurationMinutes = 30; diff --git a/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs b/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs index a5d1274bd..9c8ad8b1a 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs @@ -1,4 +1,3 @@ -using Dfe.PlanTech.Application.Extensions; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Questionnaire.Interfaces; using Dfe.PlanTech.Domain.Questionnaire.Models; @@ -14,72 +13,79 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger GetCompleteRecommendationsForSubtopic(string subtopicId, CancellationToken cancellationToken) { - var recommendation = await _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId) - .Select(subtopicRecommendation => new SubtopicRecommendationDbEntity() - { - Subtopic = new SectionDbEntity() - { - Name = subtopicRecommendation.Subtopic.Name - }, - Section = new RecommendationSectionDbEntity() - { - Id = subtopicRecommendation.Section.Id, - Answers = subtopicRecommendation!.Section.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(), - }, - SectionId = subtopicRecommendation!.SectionId, - Id = subtopicRecommendation.Id - }) - .FirstOrDefaultAsyncWithCache(cancellationToken); + var recommendation = await _db.FirstOrDefaultAsync( + _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId) + .Select(subtopicRecommendation => new SubtopicRecommendationDbEntity() + { + Subtopic = new SectionDbEntity() + { + Name = subtopicRecommendation.Subtopic.Name + }, + Section = new RecommendationSectionDbEntity() + { + Id = subtopicRecommendation.Section.Id, + Answers = subtopicRecommendation!.Section.Answers + .Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(), + }, + SectionId = subtopicRecommendation!.SectionId, + Id = subtopicRecommendation.Id + }), cancellationToken); if (recommendation == null) { return null; } - var intros = await _db.RecommendationIntros.Where(intro => intro.SubtopicRecommendations.Any(subtopicRec => subtopicRec.Id == recommendation.Id)) - .Include(intro => intro.Header) - .ToListAsyncWithCache(cancellationToken); + var intros = await _db.ToListAsync( + _db.RecommendationIntros + .Where(intro => intro.SubtopicRecommendations.Any(subtopicRec => subtopicRec.Id == recommendation.Id)) + .Include(intro => intro.Header), cancellationToken); - var chunks = await _db.RecommendationChunks.Where(chunk => chunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId)) - .Select(chunk => new RecommendationChunkDbEntity() - { - Header = chunk.Header, - Answers = chunk.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(), - Id = chunk.Id, - Order = chunk.Order, - CSLink = chunk.CSLink - }) - .OrderBy(chunk => chunk.Order) - .ToListAsyncWithCache(cancellationToken); + var chunks = await _db.ToListAsync( + _db.RecommendationChunks + .Where(chunk => chunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId)) + .Select(chunk => new RecommendationChunkDbEntity() + { + Header = chunk.Header, + Answers = chunk.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(), + Id = chunk.Id, + Order = chunk.Order, + CSLink = chunk.CSLink + }) + .OrderBy(chunk => chunk.Order), cancellationToken); - var introContent = await _db.RecommendationIntroContents.Where(introContent => introContent.RecommendationIntro != null && - introContent.RecommendationIntro.SubtopicRecommendations.Any(rec => rec.Id == recommendation.Id)) - .Select(introContent => new RecommendationIntroContentDbEntity() - { - RecommendationIntroId = introContent.RecommendationIntroId, - ContentComponent = introContent.ContentComponent, - ContentComponentId = introContent.ContentComponentId, - Id = introContent.Id - }) - .ToListAsyncWithCache(cancellationToken); + var introContent = await _db.ToListAsync( + _db.RecommendationIntroContents.Where(introContent => introContent.RecommendationIntro != null && + introContent.RecommendationIntro + .SubtopicRecommendations + .Any(rec => rec.Id == recommendation.Id)) + .Select(introContent => new RecommendationIntroContentDbEntity() + { + RecommendationIntroId = introContent.RecommendationIntroId, + ContentComponent = introContent.ContentComponent, + ContentComponentId = introContent.ContentComponentId, + Id = introContent.Id + }), cancellationToken); - var chunkContent = await _db.RecommendationChunkContents.Where(chunkContent => chunkContent.RecommendationChunk != null && - chunkContent.RecommendationChunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId)) - .Select(chunkContent => new RecommendationChunkContentDbEntity() - { - RecommendationChunkId = chunkContent.RecommendationChunkId, - ContentComponent = chunkContent.ContentComponent, - ContentComponentId = chunkContent.ContentComponentId, - Id = chunkContent.Id - }) - .ToListAsyncWithCache(cancellationToken); + var chunkContent = await _db.ToListAsync( + _db.RecommendationChunkContents.Where(chunkContent => + chunkContent.RecommendationChunk != null && + chunkContent.RecommendationChunk.RecommendationSections.Any(section => + section.Id == recommendation.SectionId)) + .Select(chunkContent => new RecommendationChunkContentDbEntity() + { + RecommendationChunkId = chunkContent.RecommendationChunkId, + ContentComponent = chunkContent.ContentComponent, + ContentComponentId = chunkContent.ContentComponentId, + Id = chunkContent.Id + }), cancellationToken); LogInvalidJoinRows(introContent); LogInvalidJoinRows(chunkContent); - await _db.RichTextContentWithSubtopicRecommendationIds - .Where(rt => rt.SubtopicRecommendationId == recommendation.Id) - .ToListAsyncWithCache(cancellationToken); + await _db.ToListAsync( + _db.RichTextContentWithSubtopicRecommendationIds + .Where(rt => rt.SubtopicRecommendationId == recommendation.Id), cancellationToken); return new SubtopicRecommendationDbEntity() { @@ -114,11 +120,13 @@ await _db.RichTextContentWithSubtopicRecommendationIds }; } - public Task GetRecommenationsViewDtoForSubtopicAndMaturity(string subtopicId, string maturity, CancellationToken cancellationToken) - => _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId) - .Select(subtopicRecommendation => subtopicRecommendation.Intros.FirstOrDefault(intro => intro.Maturity == maturity)) - .Select(intro => intro != null ? new RecommendationsViewDto(intro.Slug, intro.Header.Text) : null) - .FirstOrDefaultAsyncWithCache(cancellationToken: cancellationToken); + public Task GetRecommenationsViewDtoForSubtopicAndMaturity(string subtopicId, + string maturity, CancellationToken cancellationToken) + => _db.FirstOrDefaultAsync( + _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId) + .Select(subtopicRecommendation => subtopicRecommendation.Intros.FirstOrDefault(intro => intro.Maturity == maturity)) + .Select(intro => intro != null ? new RecommendationsViewDto(intro.Slug, intro.Header.Text) : null), + cancellationToken); /// /// Check for invalid join rows, and log any errored rows. diff --git a/src/Dfe.PlanTech.Web/ProgramExtensions.cs b/src/Dfe.PlanTech.Web/ProgramExtensions.cs index 9d9b7b77c..171cc5a90 100644 --- a/src/Dfe.PlanTech.Web/ProgramExtensions.cs +++ b/src/Dfe.PlanTech.Web/ProgramExtensions.cs @@ -125,6 +125,7 @@ public static IServiceCollection AddCaching(this IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddSingleton(); return services; } From 504cd973fa3a2747d811bd8b98967023451b80a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:21:23 +0000 Subject: [PATCH 09/15] chore: Linted code for plan-technology-for-your-school.sln solution --- src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index bcae8ab6f..cc008b767 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -2,7 +2,6 @@ using System.Linq.Expressions; using System.Security.Cryptography; using System.Text; -using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Application.Caching.Models; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Content.Models; From 77fecc9ec6f7b87dccaacbd3cd91965a42af5455 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Mon, 30 Sep 2024 14:15:57 +0100 Subject: [PATCH 10/15] move cacher setup --- docs/Conventions.md | 2 +- src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs | 8 ++++---- src/Dfe.PlanTech.Web/ProgramExtensions.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Conventions.md b/docs/Conventions.md index 3d1a3bd11..214ebdb7d 100644 --- a/docs/Conventions.md +++ b/docs/Conventions.md @@ -41,5 +41,5 @@ // do this instead db.FirstOrDefaultAsync(db.RecommendationChunks.Where(condition), cancellationToken); ``` -- the cache can be cleared with the `ClearCmsCache` method which clears everything +- the cache can be cleared with the `ClearCache` method which clears everything - More about contentful caching is explained in [Contentful-Caching-Process](/docs/Contentful-Caching-Process.md) diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index cc008b767..633ccf958 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -119,16 +119,16 @@ public class CmsDbContext : DbContext, ICmsDbContext private readonly ContentfulOptions _contentfulOptions; private readonly QueryCacher _queryCacher; - public CmsDbContext() + public CmsDbContext(QueryCacher queryCacher) { _contentfulOptions = new ContentfulOptions(false); - _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); + _queryCacher = queryCacher ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); } - public CmsDbContext(DbContextOptions options) : base(options) + public CmsDbContext(QueryCacher queryCacher, DbContextOptions options) : base(options) { _contentfulOptions = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(ContentfulOptions)}"); - _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); + _queryCacher = queryCacher ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); } protected override void OnModelCreating(ModelBuilder modelBuilder) diff --git a/src/Dfe.PlanTech.Web/ProgramExtensions.cs b/src/Dfe.PlanTech.Web/ProgramExtensions.cs index 171cc5a90..3d8c42a30 100644 --- a/src/Dfe.PlanTech.Web/ProgramExtensions.cs +++ b/src/Dfe.PlanTech.Web/ProgramExtensions.cs @@ -125,7 +125,6 @@ public static IServiceCollection AddCaching(this IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddSingleton(); return services; } @@ -133,6 +132,7 @@ public static IServiceCollection AddCaching(this IServiceCollection services) public static IServiceCollection AddDatabase(this IServiceCollection services, IConfiguration configuration) { void databaseOptionsAction(DbContextOptionsBuilder options) => options.UseSqlServer(configuration.GetConnectionString("Database")); + services.AddSingleton(); services.AddDbContextPool((serviceProvider, optionsBuilder) => optionsBuilder From b6eecc3898b0755021a0b0c43c4ae3204b2c27b2 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Mon, 30 Sep 2024 14:35:06 +0100 Subject: [PATCH 11/15] fix: fix initialisation of querycacher --- src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index 633ccf958..4548d2b4d 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -2,7 +2,7 @@ using System.Linq.Expressions; using System.Security.Cryptography; using System.Text; -using Dfe.PlanTech.Application.Caching.Models; +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Content.Models; using Dfe.PlanTech.Domain.Content.Models.Buttons; @@ -117,18 +117,18 @@ public class CmsDbContext : DbContext, ICmsDbContext #endregion private readonly ContentfulOptions _contentfulOptions; - private readonly QueryCacher _queryCacher; + private readonly IQueryCacher _queryCacher; - public CmsDbContext(QueryCacher queryCacher) + public CmsDbContext() { _contentfulOptions = new ContentfulOptions(false); - _queryCacher = queryCacher ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); + _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(IQueryCacher)}"); } - public CmsDbContext(QueryCacher queryCacher, DbContextOptions options) : base(options) + public CmsDbContext(DbContextOptions options) : base(options) { _contentfulOptions = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(ContentfulOptions)}"); - _queryCacher = queryCacher ?? throw new MissingServiceException($"Could not find service {nameof(QueryCacher)}"); + _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(IQueryCacher)}"); } protected override void OnModelCreating(ModelBuilder modelBuilder) From 2005bf68082b4a476ddb941a2c489f6c0f1d8978 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Tue, 1 Oct 2024 14:46:08 +0100 Subject: [PATCH 12/15] fix: resolve merge issues --- src/Dfe.PlanTech.Web/Caching/CacheClearer.cs | 7 +-- src/Dfe.PlanTech.Web/ProgramExtensions.cs | 1 + .../DatabaseHelperTests.cs | 6 ++- .../Caching/CacheClearerTests.cs | 8 +-- .../Controllers/CacheControllerTests.cs | 49 ------------------- 5 files changed, 14 insertions(+), 57 deletions(-) delete mode 100644 tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs diff --git a/src/Dfe.PlanTech.Web/Caching/CacheClearer.cs b/src/Dfe.PlanTech.Web/Caching/CacheClearer.cs index b029d8dc6..487054a73 100644 --- a/src/Dfe.PlanTech.Web/Caching/CacheClearer.cs +++ b/src/Dfe.PlanTech.Web/Caching/CacheClearer.cs @@ -1,9 +1,10 @@ +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Domain.Caching.Interfaces; -using EFCoreSecondLevelCacheInterceptor; +using Microsoft.AspNetCore.Mvc; namespace Dfe.PlanTech.Web.Caching; -public class CacheClearer(IEFCacheServiceProvider cacheServiceProvider, ILogger logger) : ICacheClearer +public class CacheClearer([FromServices] IQueryCacher queryCacher, ILogger logger) : ICacheClearer { /// /// Makes a call to the plan tech web app that invalidates the database cache. @@ -12,7 +13,7 @@ public bool ClearCache() { try { - cacheServiceProvider.ClearAllCachedEntries(); + queryCacher.ClearCache(); logger.LogInformation("Database cache has been cleared"); return true; } diff --git a/src/Dfe.PlanTech.Web/ProgramExtensions.cs b/src/Dfe.PlanTech.Web/ProgramExtensions.cs index 211f888a6..4d96a719f 100644 --- a/src/Dfe.PlanTech.Web/ProgramExtensions.cs +++ b/src/Dfe.PlanTech.Web/ProgramExtensions.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using Dfe.ContentSupport.Web.Extensions; +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Application.Caching.Models; using Dfe.PlanTech.Application.Content.Queries; using Dfe.PlanTech.Application.Cookie.Service; diff --git a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs index a34e657f4..5a8db3904 100644 --- a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs +++ b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs @@ -1,4 +1,5 @@ using System.Reflection; +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Persistence.Models; using Dfe.PlanTech.Domain.Questionnaire.Models; @@ -13,14 +14,17 @@ namespace Dfe.PlanTech.Infrastructure.Data.UnitTests; public class DatabaseHelperTests { private readonly IServiceProvider _serviceProvider = Substitute.For(); - private readonly CmsDbContext _mockDb = Substitute.For(); + private readonly IQueryCacher _queryCacher = Substitute.For(); + private readonly CmsDbContext _mockDb; private readonly DatabaseHelper _databaseHelper; private readonly string[] _nonNullablePropertyNames = ["Id", "Archived", "Published", "Deleted", "Slug", "Text"]; public DatabaseHelperTests() { + _serviceProvider.GetService(typeof(IQueryCacher)).Returns(_queryCacher); _serviceProvider.GetService(typeof(CmsDbContext)).Returns(_mockDb); _serviceProvider.GetService(typeof(ICmsDbContext)).Returns(_mockDb); + _mockDb = Substitute.For(); AddRealDbProperties(); _databaseHelper = new DatabaseHelper(_serviceProvider); diff --git a/tests/Dfe.PlanTech.Web.UnitTests/Caching/CacheClearerTests.cs b/tests/Dfe.PlanTech.Web.UnitTests/Caching/CacheClearerTests.cs index ebabb6ebc..e971e5cc0 100644 --- a/tests/Dfe.PlanTech.Web.UnitTests/Caching/CacheClearerTests.cs +++ b/tests/Dfe.PlanTech.Web.UnitTests/Caching/CacheClearerTests.cs @@ -1,5 +1,5 @@ +using Dfe.PlanTech.Application.Caching.Interfaces; using Dfe.PlanTech.Web.Caching; -using EFCoreSecondLevelCacheInterceptor; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; @@ -9,12 +9,12 @@ namespace Dfe.PlanTech.Web.UnitTests.Caching; public class CacheClearerTests { private readonly CacheClearer _cacheHandler; - private readonly IEFCacheServiceProvider _cacheServiceProvider = Substitute.For(); + private readonly IQueryCacher _queryCacher = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); public CacheClearerTests() { - _cacheHandler = new CacheClearer(_cacheServiceProvider, _logger); + _cacheHandler = new CacheClearer(_queryCacher, _logger); } [Fact] @@ -33,7 +33,7 @@ public void CacheHandler_Should_ClearCache() public void CacheHandler_Should_LogErrors() { var exception = new Exception("Exception thrown"); - _cacheServiceProvider.When(cp => cp.ClearAllCachedEntries()).Throw(exception); + _queryCacher.When(cp => cp.ClearCache()).Throw(exception); var result = _cacheHandler.ClearCache(); Assert.False(result); diff --git a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs b/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs deleted file mode 100644 index d9b03a427..000000000 --- a/tests/Dfe.PlanTech.Web.UnitTests/Controllers/CacheControllerTests.cs +++ /dev/null @@ -1,49 +0,0 @@ -using Dfe.PlanTech.Application.Caching.Interfaces; -using Dfe.PlanTech.Web.Controllers; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using NSubstitute; -using Xunit; - -namespace Dfe.PlanTech.Web.UnitTests.Controllers; - -public class CacheControllerTests -{ - private readonly IQueryCacher _queryCacher = Substitute.For(); - private readonly ILogger _logger = Substitute.For>(); - private readonly CacheController _cacheController; - - public CacheControllerTests() - { - _cacheController = new CacheController(_queryCacher, _logger); - } - - [Fact] - public void ClearCache_Should_Return_True_On_Success() - { - var clearCacheResult = _cacheController.ClearCache(); - - Assert.NotNull(clearCacheResult); - - var result = clearCacheResult as ObjectResult; - Assert.NotNull(result); - Assert.Equal(200, result.StatusCode); - - _queryCacher.Received(1).ClearCache(); - } - - [Fact] - public void ClearCache_Should_Return_False_On_Failure() - { - _queryCacher - .When(call => call.ClearCache()) - .Do(_ => throw new Exception("unexpected error")); - - var clearCacheResult = _cacheController.ClearCache(); - Assert.NotNull(clearCacheResult); - - var result = clearCacheResult as ObjectResult; - Assert.NotNull(result); - Assert.Equal(500, result.StatusCode); - } -} From 80830f5e584b1b9f5200c25aa55bfba143737c81 Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Tue, 1 Oct 2024 14:49:52 +0100 Subject: [PATCH 13/15] docs: resolved duplicated docs --- docs/Contentful-Caching-Process.md | 99 ------------------------------ docs/cms/README.md | 7 +-- 2 files changed, 3 insertions(+), 103 deletions(-) delete mode 100644 docs/Contentful-Caching-Process.md diff --git a/docs/Contentful-Caching-Process.md b/docs/Contentful-Caching-Process.md deleted file mode 100644 index f44b90ac6..000000000 --- a/docs/Contentful-Caching-Process.md +++ /dev/null @@ -1,99 +0,0 @@ -# Contentful Caching via Database Process - -## Overview - -- Webhook posts to Azure function -- Azure function writes to queue -- Another Azure function reads from queue + writes to database -- Data is read from DB where applicable, if any failure then an attempt to load from Contentful is performed - -## Contentful -> DB Process - -![Contentful to database architecture][/docs/diagrams/cms-to-db-process-flow.png] - -- Code is contained in the [/src/Dfe.PlanTech.AzureFunctions/](/src/Dfe.PlanTech.AzureFunctions/) project. - -### Webhook -> Queue - -- We have a webhook on Contentful setup for entries -- The webhook is setup to trigger on all events (Create, Save, Autosave, Archive, Unarchive, Publish, Unpublish, Delete) for an entry -- The webhook points to an API hosted as an Azure Function [/src/Dfe.PlanTech.AzureFunctions/ContentfulWebHook.cs](/src/Dfe.PlanTech.AzureFunctions/ContentfulWebHook.cs) -- The API receives the JSON payload from the webhook, and writes it to an Azure Servicebus queue. - - There is no validation of the JSON payload at this point. - -- The Azure Function for this is secured by the inbuilt Azure Function HTTP authentication - -### Queue -> DB - -- We have another Azure Function which is triggered by messages on the Azure Servicebus (/src/Dfe.PlanTech.AzureFunctions/QueueReceiver.cs) - - These messages are batched where applicable, using the default batch settings - -- The Azure Function reads the message from the queue and, for each message, - 1. Strips out unnecessary information from the JSON payload, converting the JSON to just be what the entry value is - 2. Adds necessary relationship data for an entry, if any, into the JSON - 3. Desrialises the JSON to the _database entity model_ - 4. Updates the entry status columns where appropriate (i.e. archived/published/deleted) - 5. Upserts the entry in the database - 6. Makes a call to the Plan Tech service to invalidate the cache - -#### Mapping - -- We normalise the incoming entry JSON. This involves essentially just copying the children within the "fields" object to a new JSON object, along with the "id" field from the "sys" object. - - The end result of this is a JSON that should match the actual Contentful classes we use one-to-one. -- We then deserialise this JSON to the appropriate database class for the content type -- We retrieve the existing entity from our database, if it exists, using the id. - - If it does exist, we use reflection to copy over the values from the _incoming_ mapped entity, to the found _existing_ entity. - - Certain properties are ignored during this, for a variety of reasons (e.g. they might be metadata, meaning they wouldn't exist on in the incoming data, which could cause errors or incorrect data copied over) - - This is done by using our custom [DontCopyValueAttribute](./src/Dfe.PlanTech.Domain/DontCopyValueAttribute.cs) on each property we do not wish to copy, and checking for its existance per property -- Any relationship fields are mapped in the individual mapping classes per content type. If the relationship foreign key is on the _related_ entity, then we: - - Create an object of that content type - - Attach it to the EF Core context - - Make the changes to the relationship field - - This ensures that the changes to the relationship field are tracked by EF Core, without having to query the database for the existence of the row in the database -- We then save the changes in EF Core. - -## DB Architecture - -### Schema - -![CMS DB Schema](/docs/diagrams/published/PTFYS%20CMS%20Schema.png) - -### Functions - -## Reading from database - -We use EF Core as our ORM for reading/writing to the database. The DbContext used is [/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs](/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs). - -### Mapping - -AutoMapper is used for the majority of the mapping, as the DB models + Contentful models are 1-1 mapped for the vast majority of the fields/properties. The mapping profile for this is located in [/src/Dfe.PlanTech.Application/Mappings/MappingProfile.cs](/src/Dfe.PlanTech.Application/Mappings/MappingProfile.cs). - -There are _some_ custom mappings done using LINQ `select` projections, due to various issues such as cyclical navigations with certain tables, or simply to limit what data is returned. - -### Read navigation links - -[/src/Dfe.PlanTech.Application/Content/Queries/GetNavigationQuery.cs] is where we retrieve navigation links for the footer. It is a simple query, merely returning the entire database table. - -### Read page - -[/src/Dfe.PlanTech.Application/Content/Queries/GetPageQuery.cs](/src/Dfe.PlanTech.Application/Content/Queries/GetPageQuery.cs) is responsible for retrieving Page data. - -There are several steps to it: - -1. We retrieve the `Page` matching the Slug from the table, and Include all `BeforeTitleContent` and `Content`. Note: there are various AutoIncludes defined in the [/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs](/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs) for the majority of the content tables, to ensure all data is pulled in. - -2. Due to various issues with certain navigations, we execute various other queries and then merge the data together. These are completed using various `IGetPageChildrenQuery` objects that are injected into the `GetPageQuery` in the constructor using DI. - - - If there is any content which has a `RichTextContent` property (using the `IHasText` interface to check), we execute a query to retrieve all the `RichTextContent` for the Page. The `RichTextMark` and `RichTextData` tables are joined automatically. This is handled by the [GetRichTextsQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetRichTextsQuery.cs) class. - - - If there is any `ButtonWithEntryReference` content, then we retrieve the `LinkToEntry` property from it manually. This is to ensure we do not have a cartesian explosion (i.e. retrieving that entire page, + content, etc.), and to minimise the data we retrieve from the database (we only retrieve the slug and Id fields for the `LinkToEntry` property). This is handled by the [GetButtonWithEntryReferencesQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetButtonWithEntryReferencesQuery.cs) class. - - - If there are any `Category` content, we retrieve the `Sections` for them manually. This is because we also need to retrieve the `Question`s and `Recommendation`s for each Section, but only certain pieces of information. To prevent execessive data being retrieved, we only query the necessary fields. This is handled by the [GetCategorySectionsQuery](/src/Dfe.PlanTech.Application/Content/Queries/GetCategorySectionsQuery.cs) class. - -3. We then use AutoMapper to map the database models to the Contentful models, as previously described. - -## Caching - -- Caching is handled by an in memory cache defined in the QueryCacher class in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs) -- The only cache invalidation is invalidation of the whole cache as it intended only for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. -- The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. diff --git a/docs/cms/README.md b/docs/cms/README.md index 81e5b39ed..74ab65636 100644 --- a/docs/cms/README.md +++ b/docs/cms/README.md @@ -24,7 +24,6 @@ Or our [database content](./db-content.md) documentation for information on how We cache content data that was retrieved by the DB in-memory in our web app. -Currently caching is handled by the open-source [EFCoreSecondLevelCacheInterceptor](https://github.com/VahidN/EFCoreSecondLevelCacheInterceptor) C# package. - -It is enabled only in the [web project](./src/Dfe.PlanTech.Web), and is enabled in the services configuration in [ProgramExtensions.cs](./src/Dfe.PlanTech.Web/ProgramExtensions.cs). We currently have no functionality setup to amend the configuration (e.g. caching length) via any sort of environment variables, but this should be added when possible. -The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function. +Currently caching is handled by an in memory cache defined in the QueryCacher class in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs) +The only cache invalidation is invalidation of the whole cache as it intended only for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information. +The cache is invalided anytime a cms message with a content update is successfully processed. From b827df0610cf597bcbd94f980997a25840323e2e Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Tue, 1 Oct 2024 16:08:01 +0100 Subject: [PATCH 14/15] tests: Fix dbhelper tests post caching merge --- src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs | 3 ++- .../DatabaseHelperTests.cs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs index d8c509f98..278413cbd 100644 --- a/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs +++ b/src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs @@ -3,6 +3,7 @@ using System.Security.Cryptography; using System.Text; using Dfe.PlanTech.Application.Caching.Interfaces; +using Dfe.PlanTech.Application.Caching.Models; using Dfe.PlanTech.Application.Persistence.Interfaces; using Dfe.PlanTech.Domain.Content.Models; using Dfe.PlanTech.Domain.Content.Models.Buttons; @@ -122,7 +123,7 @@ public class CmsDbContext : DbContext, ICmsDbContext public CmsDbContext() { _contentfulOptions = new ContentfulOptions(false); - _queryCacher = this.GetService() ?? throw new MissingServiceException($"Could not find service {nameof(IQueryCacher)}"); + _queryCacher = new QueryCacher(); } public CmsDbContext(DbContextOptions options) : base(options) diff --git a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs index 5a8db3904..998ca134e 100644 --- a/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs +++ b/tests/Dfe.PlanTech.Infrastructure.Data.UnitTests/DatabaseHelperTests.cs @@ -15,7 +15,7 @@ public class DatabaseHelperTests { private readonly IServiceProvider _serviceProvider = Substitute.For(); private readonly IQueryCacher _queryCacher = Substitute.For(); - private readonly CmsDbContext _mockDb; + private readonly CmsDbContext _mockDb = Substitute.For(); private readonly DatabaseHelper _databaseHelper; private readonly string[] _nonNullablePropertyNames = ["Id", "Archived", "Published", "Deleted", "Slug", "Text"]; @@ -24,7 +24,6 @@ public DatabaseHelperTests() _serviceProvider.GetService(typeof(IQueryCacher)).Returns(_queryCacher); _serviceProvider.GetService(typeof(CmsDbContext)).Returns(_mockDb); _serviceProvider.GetService(typeof(ICmsDbContext)).Returns(_mockDb); - _mockDb = Substitute.For(); AddRealDbProperties(); _databaseHelper = new DatabaseHelper(_serviceProvider); @@ -38,6 +37,7 @@ private void AddRealDbProperties() var services = new ServiceCollection(); services.AddSingleton(new ContentfulOptions()); + services.AddSingleton(_queryCacher); dbContextOptionsBuilder.UseApplicationServiceProvider(services.BuildServiceProvider()); var actualDbContext = new CmsDbContext(dbContextOptionsBuilder.Options); From 761bd230794c235e4912962b8432dbb4930b903b Mon Sep 17 00:00:00 2001 From: katie-gardner-AND Date: Tue, 1 Oct 2024 16:48:13 +0100 Subject: [PATCH 15/15] tests: Add tests for query cacher --- .../Caching/QueryCacherTests.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/Dfe.PlanTech.Application.UnitTests/Caching/QueryCacherTests.cs diff --git a/tests/Dfe.PlanTech.Application.UnitTests/Caching/QueryCacherTests.cs b/tests/Dfe.PlanTech.Application.UnitTests/Caching/QueryCacherTests.cs new file mode 100644 index 000000000..9027dd066 --- /dev/null +++ b/tests/Dfe.PlanTech.Application.UnitTests/Caching/QueryCacherTests.cs @@ -0,0 +1,48 @@ +using Dfe.PlanTech.Application.Caching.Models; +using Dfe.PlanTech.Domain.Content.Models; +using NSubstitute; + +namespace Dfe.PlanTech.Application.UnitTests.Caching; + +public class QueryCacherTests +{ + private readonly QueryCacher _queryCacher; + private readonly IQueryable _mockQueryable = Substitute.For>(); + private readonly Func, CancellationToken, Task> _mockQueryFunc; + private readonly ContentComponent _mockResult = Substitute.For(); + + public QueryCacherTests() + { + _queryCacher = new QueryCacher(); + _mockQueryFunc = Substitute.For, CancellationToken, Task>>(); + _mockQueryFunc.Invoke(_mockQueryable, Arg.Any()).Returns(Task.FromResult(_mockResult)); + _queryCacher.ClearCache(); + } + + [Fact] + public async Task Should_Cache_Query_Results_By_Key() + { + await _queryCacher.GetOrCreateAsyncWithCache("key", _mockQueryable, _mockQueryFunc); + await _mockQueryFunc.Received(1).Invoke(_mockQueryable, Arg.Any()); + + await _queryCacher.GetOrCreateAsyncWithCache("key", _mockQueryable, _mockQueryFunc); + await _mockQueryFunc.Received(1).Invoke(_mockQueryable, Arg.Any()); + + await _queryCacher.GetOrCreateAsyncWithCache("different_key", _mockQueryable, _mockQueryFunc); + await _mockQueryFunc.Received(2).Invoke(_mockQueryable, Arg.Any()); + } + + [Fact] + public async Task Should_Clear_All_With_ClearCache() + { + await _queryCacher.GetOrCreateAsyncWithCache("key", _mockQueryable, _mockQueryFunc); + await _queryCacher.GetOrCreateAsyncWithCache("different_key", _mockQueryable, _mockQueryFunc); + await _mockQueryFunc.Received(2).Invoke(_mockQueryable, CancellationToken.None); + + _queryCacher.ClearCache(); + + await _queryCacher.GetOrCreateAsyncWithCache("key", _mockQueryable, _mockQueryFunc); + await _queryCacher.GetOrCreateAsyncWithCache("different_key", _mockQueryable, _mockQueryFunc); + await _mockQueryFunc.Received(4).Invoke(_mockQueryable, CancellationToken.None); + } +}