Merge pull request #793 from DFE-Digital/feat/216205/implement-new-sq…

…l-caching

Feat: Implement new sql caching

docs/Conventions.md

@@ -29,4 +29,17 @@
     ```html
     <div class="js-only">This element will be hidden by default, but made visible via JS.</div>
     ```
-  - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml)
+  - The code that unhides these elements is located in [_BodyEnd.cshtml](src/Dfe.PlanTech.Web/Views/Shared/_BodyEnd.cshtml)
+
+## EF Query Conventions
+- We use a Memory Cache to cache content queries by the hash of their query string. This is setup in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs).
+- Any queries using the `CmsDbContext` should be cached by using the `db.ToListAsync` or `db.FirstOrDefaultAsync` methods from `CmsDbContext` rather than extension methods
+  - For example
+    ```csharp
+    // don't do this
+    db.RecommendationChunks.Where(condition).FirstOrDefaultAsync(cancellationToken);
+    // do this instead
+    db.FirstOrDefaultAsync(db.RecommendationChunks.Where(condition), cancellationToken);
+    ```
+- the cache can be cleared with the `ClearCache` method which clears everything
+- More about contentful caching is explained in [Contentful-Caching-Process](/docs/Contentful-Caching-Process.md)

docs/cms/README.md

@@ -24,7 +24,6 @@ Or our [database content](./db-content.md) documentation for information on how
 
 We cache content data that was retrieved by the DB in-memory in our web app.
 
-Currently caching is handled by the open-source [EFCoreSecondLevelCacheInterceptor](https://github.com/VahidN/EFCoreSecondLevelCacheInterceptor) C# package.
-
-It is enabled only in the [web project](./src/Dfe.PlanTech.Web), and is enabled in the services configuration in [ProgramExtensions.cs](./src/Dfe.PlanTech.Web/ProgramExtensions.cs). We currently have no functionality setup to amend the configuration (e.g. caching length) via any sort of environment variables, but this should be added when possible.
-The Cache can be invalidated by an API key protected endpoint in the website. This is called by the azure function whenever content is updated in the database. The API key is stored in the key vault and referenced by an environment variable for the function.
+Currently caching is handled by an in memory cache defined in the QueryCacher class in [Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs](/src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs)
+The only cache invalidation is invalidation of the whole cache as it intended only for use with the `CmsDbContext` and not any of the `dbo` tables which will be frequently updated. See [Conventions](/docs/Conventions.md) for more information.
+The cache is invalided anytime a cms message with a content update is successfully processed.

src/Dfe.PlanTech.Application/Caching/Models/QueryCacher.cs

@@ -0,0 +1,29 @@
+using Dfe.PlanTech.Application.Caching.Interfaces;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace Dfe.PlanTech.Application.Caching.Models;
+
+public class QueryCacher : IQueryCacher
+{
+    private MemoryCache _cache = new(new MemoryCacheOptions());
+    private const int CacheDurationMinutes = 30;
+
+    public async Task<TResult> GetOrCreateAsyncWithCache<T, TResult>(
+        string key,
+        IQueryable<T> queryable,
+        Func<IQueryable<T>, CancellationToken, Task<TResult>> queryFunc,
+        CancellationToken cancellationToken = default)
+    {
+        return await _cache.GetOrCreateAsync(key, entry =>
+        {
+            entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(CacheDurationMinutes);
+            return queryFunc(queryable, cancellationToken);
+        }) ?? await queryFunc(queryable, cancellationToken);
+    }
+
+    public void ClearCache()
+    {
+        _cache.Dispose();
+        _cache = new MemoryCache(new MemoryCacheOptions());
+    }
+}

src/Dfe.PlanTech.Application/Dfe.PlanTech.Application.csproj

@@ -9,6 +9,7 @@
     <PackageReference Include="AutoMapper" Version="13.0.1" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
   </ItemGroup>
 

src/Dfe.PlanTech.Domain/Caching/Interfaces/IQueryCacher.cs

@@ -0,0 +1,12 @@
+namespace Dfe.PlanTech.Application.Caching.Interfaces;
+
+public interface IQueryCacher
+{
+    public Task<TResult> GetOrCreateAsyncWithCache<T, TResult>(
+        string key,
+        IQueryable<T> queryable,
+        Func<IQueryable<T>, CancellationToken, Task<TResult>> queryFunc,
+        CancellationToken cancellationToken = default);
+
+    public void ClearCache();
+}

src/Dfe.PlanTech.Infrastructure.Data/CmsDbContext.cs

@@ -1,5 +1,9 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Linq.Expressions;
+using System.Security.Cryptography;
+using System.Text;
+using Dfe.PlanTech.Application.Caching.Interfaces;
+using Dfe.PlanTech.Application.Caching.Models;
 using Dfe.PlanTech.Application.Persistence.Interfaces;
 using Dfe.PlanTech.Domain.Content.Models;
 using Dfe.PlanTech.Domain.Content.Models.Buttons;
@@ -114,15 +118,18 @@ public class CmsDbContext : DbContext, ICmsDbContext
     #endregion
 
     private readonly ContentfulOptions _contentfulOptions;
+    private readonly IQueryCacher _queryCacher;
 
     public CmsDbContext()
     {
         _contentfulOptions = new ContentfulOptions(false);
+        _queryCacher = new QueryCacher();
     }
 
     public CmsDbContext(DbContextOptions<CmsDbContext> options) : base(options)
     {
         _contentfulOptions = this.GetService<ContentfulOptions>() ?? throw new MissingServiceException($"Could not find service {nameof(ContentfulOptions)}");
+        _queryCacher = this.GetService<IQueryCacher>() ?? throw new MissingServiceException($"Could not find service {nameof(IQueryCacher)}");
     }
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -226,16 +233,33 @@ public virtual Task<int> SetComponentPublishedAndDeletedStatuses(ContentComponen
     private Expression<Func<ContentComponentDbEntity, bool>> ShouldShowEntity()
         => entity => (_contentfulOptions.UsePreview || entity.Published) && !entity.Archived && !entity.Deleted;
 
-    public Task<PageDbEntity?> GetPageBySlug(string slug, CancellationToken cancellationToken = default)
-        => Pages.Include(page => page.BeforeTitleContent)
-            .Include(page => page.Content)
-            .Include(page => page.Title)
-            .AsSplitQuery()
-            .FirstOrDefaultAsync(page => page.Slug == slug, cancellationToken);
+    private static string GetCacheKey(IQueryable query)
+    {
+        var queryString = query.ToQueryString();
+        var hash = SHA256.HashData(Encoding.UTF8.GetBytes(queryString));
+        return Convert.ToBase64String(hash);
+    }
 
-    public Task<List<T>> ToListAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default) =>
-        queryable.ToListAsync(cancellationToken: cancellationToken);
+    public Task<PageDbEntity?> GetPageBySlug(string slug, CancellationToken cancellationToken = default)
+        => FirstOrDefaultAsync(
+            Pages.Where(page => page.Slug == slug)
+                .Include(page => page.BeforeTitleContent)
+                .Include(page => page.Content)
+                .Include(page => page.Title)
+                .AsSplitQuery(),
+            cancellationToken);
+
+    public async Task<List<T>> ToListAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default)
+    {
+        var key = GetCacheKey(queryable);
+        return await _queryCacher.GetOrCreateAsyncWithCache(key, queryable,
+            (q, ctoken) => q.ToListAsync(ctoken), cancellationToken);
+    }
 
-    public Task<T?> FirstOrDefaultAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default)
-        => queryable.FirstOrDefaultAsync(cancellationToken);
+    public async Task<T?> FirstOrDefaultAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default)
+    {
+        var key = GetCacheKey(queryable);
+        return await _queryCacher.GetOrCreateAsyncWithCache(key, queryable,
+            (q, ctoken) => q.FirstOrDefaultAsync(ctoken), cancellationToken);
+    }
 }

src/Dfe.PlanTech.Infrastructure.Data/Repositories/RecommendationsRepository.cs

@@ -13,72 +13,79 @@ public class RecommendationsRepository(ICmsDbContext db, ILogger<IRecommendation
 
     public async Task<SubtopicRecommendationDbEntity?> GetCompleteRecommendationsForSubtopic(string subtopicId, CancellationToken cancellationToken)
     {
-        var recommendation = await _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId)
-                                              .Select(subtopicRecommendation => new SubtopicRecommendationDbEntity()
-                                              {
-                                                  Subtopic = new SectionDbEntity()
-                                                  {
-                                                      Name = subtopicRecommendation.Subtopic.Name
-                                                  },
-                                                  Section = new RecommendationSectionDbEntity()
-                                                  {
-                                                      Id = subtopicRecommendation.Section.Id,
-                                                      Answers = subtopicRecommendation!.Section.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(),
-                                                  },
-                                                  SectionId = subtopicRecommendation!.SectionId,
-                                                  Id = subtopicRecommendation.Id
-                                              })
-                                              .FirstOrDefaultAsync(cancellationToken);
+        var recommendation = await _db.FirstOrDefaultAsync(
+            _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId)
+                .Select(subtopicRecommendation => new SubtopicRecommendationDbEntity()
+                {
+                    Subtopic = new SectionDbEntity()
+                    {
+                        Name = subtopicRecommendation.Subtopic.Name
+                    },
+                    Section = new RecommendationSectionDbEntity()
+                    {
+                        Id = subtopicRecommendation.Section.Id,
+                        Answers = subtopicRecommendation!.Section.Answers
+                            .Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(),
+                    },
+                    SectionId = subtopicRecommendation!.SectionId,
+                    Id = subtopicRecommendation.Id
+                }), cancellationToken);
 
         if (recommendation == null)
         {
             return null;
         }
 
-        var intros = await _db.RecommendationIntros.Where(intro => intro.SubtopicRecommendations.Any(subtopicRec => subtopicRec.Id == recommendation.Id))
-                                                  .Include(intro => intro.Header)
-                                                  .ToListAsync(cancellationToken);
+        var intros = await _db.ToListAsync(
+            _db.RecommendationIntros
+                .Where(intro => intro.SubtopicRecommendations.Any(subtopicRec => subtopicRec.Id == recommendation.Id))
+                .Include(intro => intro.Header), cancellationToken);
 
-        var chunks = await _db.RecommendationChunks.Where(chunk => chunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId))
-                                                    .Select(chunk => new RecommendationChunkDbEntity()
-                                                    {
-                                                        Header = chunk.Header,
-                                                        Answers = chunk.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(),
-                                                        Id = chunk.Id,
-                                                        Order = chunk.Order,
-                                                        CSLink = chunk.CSLink
-                                                    })
-                                                    .OrderBy(chunk => chunk.Order)
-                                                    .ToListAsync(cancellationToken);
+        var chunks = await _db.ToListAsync(
+            _db.RecommendationChunks
+                .Where(chunk => chunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId))
+                .Select(chunk => new RecommendationChunkDbEntity()
+                {
+                    Header = chunk.Header,
+                    Answers = chunk.Answers.Select(answer => new AnswerDbEntity() { Id = answer.Id }).ToList(),
+                    Id = chunk.Id,
+                    Order = chunk.Order,
+                    CSLink = chunk.CSLink
+                })
+                .OrderBy(chunk => chunk.Order), cancellationToken);
 
-        var introContent = await _db.RecommendationIntroContents.Where(introContent => introContent.RecommendationIntro != null &&
-                                                                                       introContent.RecommendationIntro.SubtopicRecommendations.Any(rec => rec.Id == recommendation.Id))
-                                                                .Select(introContent => new RecommendationIntroContentDbEntity()
-                                                                {
-                                                                    RecommendationIntroId = introContent.RecommendationIntroId,
-                                                                    ContentComponent = introContent.ContentComponent,
-                                                                    ContentComponentId = introContent.ContentComponentId,
-                                                                    Id = introContent.Id
-                                                                })
-                                                                .ToListAsync(cancellationToken);
+        var introContent = await _db.ToListAsync(
+            _db.RecommendationIntroContents.Where(introContent => introContent.RecommendationIntro != null &&
+                                                                  introContent.RecommendationIntro
+                                                                      .SubtopicRecommendations
+                                                                      .Any(rec => rec.Id == recommendation.Id))
+                .Select(introContent => new RecommendationIntroContentDbEntity()
+                {
+                    RecommendationIntroId = introContent.RecommendationIntroId,
+                    ContentComponent = introContent.ContentComponent,
+                    ContentComponentId = introContent.ContentComponentId,
+                    Id = introContent.Id
+                }), cancellationToken);
 
-        var chunkContent = await _db.RecommendationChunkContents.Where(chunkContent => chunkContent.RecommendationChunk != null &&
-                                                                                       chunkContent.RecommendationChunk.RecommendationSections.Any(section => section.Id == recommendation.SectionId))
-                                                                .Select(chunkContent => new RecommendationChunkContentDbEntity()
-                                                                {
-                                                                    RecommendationChunkId = chunkContent.RecommendationChunkId,
-                                                                    ContentComponent = chunkContent.ContentComponent,
-                                                                    ContentComponentId = chunkContent.ContentComponentId,
-                                                                    Id = chunkContent.Id
-                                                                })
-                                                                .ToListAsync(cancellationToken);
+        var chunkContent = await _db.ToListAsync(
+            _db.RecommendationChunkContents.Where(chunkContent =>
+                    chunkContent.RecommendationChunk != null &&
+                    chunkContent.RecommendationChunk.RecommendationSections.Any(section =>
+                        section.Id == recommendation.SectionId))
+                .Select(chunkContent => new RecommendationChunkContentDbEntity()
+                {
+                    RecommendationChunkId = chunkContent.RecommendationChunkId,
+                    ContentComponent = chunkContent.ContentComponent,
+                    ContentComponentId = chunkContent.ContentComponentId,
+                    Id = chunkContent.Id
+                }), cancellationToken);
 
         LogInvalidJoinRows(introContent);
         LogInvalidJoinRows(chunkContent);
 
-        await _db.RichTextContentWithSubtopicRecommendationIds
-          .Where(rt => rt.SubtopicRecommendationId == recommendation.Id)
-          .ToListAsync(cancellationToken);
+        await _db.ToListAsync(
+            _db.RichTextContentWithSubtopicRecommendationIds
+                .Where(rt => rt.SubtopicRecommendationId == recommendation.Id), cancellationToken);
 
         return new SubtopicRecommendationDbEntity()
         {
@@ -113,11 +120,13 @@ await _db.RichTextContentWithSubtopicRecommendationIds
         };
     }
 
-    public Task<RecommendationsViewDto?> GetRecommenationsViewDtoForSubtopicAndMaturity(string subtopicId, string maturity, CancellationToken cancellationToken)
-    => _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId)
-                                  .Select(subtopicRecommendation => subtopicRecommendation.Intros.FirstOrDefault(intro => intro.Maturity == maturity))
-                                  .Select(intro => intro != null ? new RecommendationsViewDto(intro.Slug, intro.Header.Text) : null)
-                                  .FirstOrDefaultAsync(cancellationToken: cancellationToken);
+    public Task<RecommendationsViewDto?> GetRecommenationsViewDtoForSubtopicAndMaturity(string subtopicId,
+        string maturity, CancellationToken cancellationToken)
+        => _db.FirstOrDefaultAsync(
+            _db.SubtopicRecommendations.Where(subtopicRecommendation => subtopicRecommendation.SubtopicId == subtopicId)
+                .Select(subtopicRecommendation => subtopicRecommendation.Intros.FirstOrDefault(intro => intro.Maturity == maturity))
+                .Select(intro => intro != null ? new RecommendationsViewDto(intro.Slug, intro.Header.Text) : null),
+            cancellationToken);
 
     /// <summary>
     /// Check for invalid join rows, and log any errored rows.

src/Dfe.PlanTech.Web/Caching/CacheClearer.cs

@@ -1,9 +1,10 @@
+using Dfe.PlanTech.Application.Caching.Interfaces;
 using Dfe.PlanTech.Domain.Caching.Interfaces;
-using EFCoreSecondLevelCacheInterceptor;
+using Microsoft.AspNetCore.Mvc;
 
 namespace Dfe.PlanTech.Web.Caching;
 
-public class CacheClearer(IEFCacheServiceProvider cacheServiceProvider, ILogger<CacheClearer> logger) : ICacheClearer
+public class CacheClearer([FromServices] IQueryCacher queryCacher, ILogger<CacheClearer> logger) : ICacheClearer
 {
     /// <summary>
     /// Makes a call to the plan tech web app that invalidates the database cache.
@@ -12,7 +13,7 @@ public bool ClearCache()
     {
         try
         {
-            cacheServiceProvider.ClearAllCachedEntries();
+            queryCacher.ClearCache();
             logger.LogInformation("Database cache has been cleared");
             return true;
         }