V2.9.0

### Bug Fixes

    * add schema to trigger operations and support caching (9760749)
    * allow for triggers on tables (9e21913)
    * amend C&S links to open in same window (915dd40)
    * Application insights ordering (316d6b7)
    * Handle errors thrown by retrieving a page from DB (5ec1216)
    * more reverting of debug changes (d07c52e)
    * Move query filter back to dbcontext (45e7ca7)
    * query (8a0e513)
    * query (62761a4)
    * removal of content (0b740a5)
    * remove output from proc that doesnt need it (0e039a9)
    * revert debug test (8718d81)
    * Revert PR 814 (dbb3dfa)
    * Revert PR 816 (4326f97)
    * turn off triggers when updating response table (1c21dca)
    * update unit tests (6cb77d9)

### Features

    * Add non-cached versions of ToListAsync and FirstOrDefaultAsync (8d6c616)
    * utilise the dateLastUpdated columns (c24156e)

docs/adr/0040-sql-schema-improvements.md

@@ -1,6 +1,6 @@
 # 0040 - SQL Schema Improvements
 
-* **Status**: proposed
+* **Status**: accepted
 
 ## Context and Problem Statement
 
@@ -47,8 +47,6 @@ blue is tables we could change, red tables to delete (displayed with two differe
 
 ### Contentful schema
 
-- Change `GetCurrentSubmissionId` to a UDF
-    - It calculates one value and outputs it, which is what UDFs are perfect for and will simplify existing usage of it
 - Add a dateCreated and dateLastUpdated column to the `ContentComponents` table.
     - This is useful for developers to see when content has changed, and makes it easier to debug content issues.
 - Add missing foreign keys
@@ -69,6 +67,8 @@ blue is tables we could change, red tables to delete (displayed with two differe
 
 ### dbo schema
 
+- Change `GetCurrentSubmissionId` to a UDF
+    - It calculates one value and outputs it, which is what UDFs are perfect for and will simplify existing usage of it
 - Alter the `SubmitAnswer` procedure to reduce data duplication
     - Currently every time an answer is submitted it puts a new record into the `questions` and `answers` tables with
       the question and answer text
@@ -80,3 +80,5 @@ blue is tables we could change, red tables to delete (displayed with two differe
 - Add an index to the `response` table on `submissionId` as this is frequently used to connect the two tables
 
 ## Decision Outcome
+
+Make the recommended changes outlined above to the dbo and Contentful schemas.

docs/adr/README.md

@@ -50,4 +50,4 @@ General information about architectural decision records is available at <https:
 | [0037 - Content Validation Process](./0037-content-validation-process.md)                                   | Proposed |
 | [0038 - SQL Caching Mechanism](0038-sql-caching-mechanism.md)                                               | Proposed |
 | [0039 - Consolidate Azure Function App into Web App](0039-consolidate-function-app.md)                      | Accepted |
-| [0040 - SQL Schema Improvements](0040-sql-schema-improvements.md)                                           | Proposed |
+| [0040 - SQL Schema Improvements](0040-sql-schema-improvements.md)                                           | Accepted |

src/Dfe.PlanTech.Application/Content/Queries/GetButtonWithEntryReferencesQuery.cs

@@ -2,21 +2,14 @@
 using Dfe.PlanTech.Domain.Content.Interfaces;
 using Dfe.PlanTech.Domain.Content.Models;
 using Dfe.PlanTech.Domain.Content.Models.Buttons;
+using Dfe.PlanTech.Domain.Questionnaire.Models;
 using Microsoft.Extensions.Logging;
 
 namespace Dfe.PlanTech.Application.Content.Queries;
 
-public class GetButtonWithEntryReferencesQuery : IGetPageChildrenQuery
+public class GetButtonWithEntryReferencesQuery(ICmsDbContext db, ILogger<GetButtonWithEntryReferencesQuery> logger)
+    : IGetPageChildrenQuery
 {
-    private readonly ICmsDbContext _db;
-    private readonly ILogger<GetButtonWithEntryReferencesQuery> _logger;
-
-    public GetButtonWithEntryReferencesQuery(ICmsDbContext db, ILogger<GetButtonWithEntryReferencesQuery> logger)
-    {
-        _db = db;
-        _logger = logger;
-    }
-
     /// <summary>
     /// If the Page.Content has any <see cref="ButtonWithEntryReferenceDbEntity"/>s, load the link to entry reference from the database for it
     /// </summary>
@@ -27,18 +20,31 @@ public async Task TryLoadChildren(PageDbEntity page, CancellationToken cancellat
     {
         try
         {
-            var buttons = page.Content.Exists(content => content is ButtonWithEntryReferenceDbEntity);
+            var pageButtonWithEntryReferences = page.GetAllContentOfType<ButtonWithEntryReferenceDbEntity>().ToArray();
 
-            if (!buttons)
+            if (pageButtonWithEntryReferences.Length == 0)
                 return;
 
             var buttonQuery = ButtonWithEntryReferencesQueryable(page);
 
-            await _db.ToListAsync(buttonQuery, cancellationToken);
+            var buttons = await db.ToListCachedAsync(buttonQuery, cancellationToken);
+
+            foreach (var button in buttons)
+            {
+                var matching = pageButtonWithEntryReferences.FirstOrDefault(pb => pb.Id == button.Id);
+
+                if (matching == null)
+                {
+                    logger.LogError("Couldn't find matching button for Id {Id}", button.Id);
+                    continue;
+                }
+
+                matching.LinkToEntry = button.LinkToEntry;
+            }
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error loading button references for {page}", page.Id);
+            logger.LogError(ex, "Error loading button references for {page}", page.Id);
             throw new InvalidOperationException("An unexpected error occurred while loading button references.", ex);
         }
     }
@@ -49,13 +55,32 @@ public async Task TryLoadChildren(PageDbEntity page, CancellationToken cancellat
     /// <param name="page"></param>
     /// <returns></returns>
     private IQueryable<ButtonWithEntryReferenceDbEntity> ButtonWithEntryReferencesQueryable(PageDbEntity page)
-    => _db.ButtonWithEntryReferences.Where(button => button.ContentPages.Any(contentPage => contentPage.Id == page.Id))
-                                    .Select(button => new ButtonWithEntryReferenceDbEntity()
+    {
+        var buttonWithEntryReferences = db.ButtonWithEntryReferences.Where(button => button.ContentPages.Any(contentPage => contentPage.Id == page.Id));
+
+        var linkedPages = db.Pages.Where(p => buttonWithEntryReferences.Any(button => button.LinkToEntryId == p.Id))
+                                    .Select(p => new
                                     {
-                                        Id = button.Id,
-                                        LinkToEntry = new PageDbEntity()
-                                        {
-                                            Slug = ((IHasSlug)button.LinkToEntry).Slug
-                                        }
+                                        p.Id,
+                                        p.Slug
                                     });
+
+        var linkedQuestions = db.Questions.Where(question => buttonWithEntryReferences.Any(button => button.LinkToEntryId == page.Id))
+                                            .Select(q => new
+                                            {
+                                                q.Id,
+                                                q.Slug
+                                            });
+
+        return buttonWithEntryReferences.Select(button => new
+        {
+            button.Id,
+            page = linkedPages.FirstOrDefault(pageForButton => pageForButton.Id == button.LinkToEntryId),
+            question = linkedQuestions.FirstOrDefault(questionForButton => questionForButton.Id == button.LinkToEntryId),
+        }).Select(button => new ButtonWithEntryReferenceDbEntity
+        {
+            Id = button.Id,
+            LinkToEntry = button.page != null ? new PageDbEntity { Slug = button.page.Slug } : new QuestionDbEntity { Slug = button.question!.Slug }
+        });
+    }
 }

src/Dfe.PlanTech.Application/Content/Queries/GetCategorySectionsQuery.cs

@@ -23,7 +23,7 @@ public async Task TryLoadChildren(PageDbEntity page, CancellationToken cancellat
             if (!pageHasCategories)
                 return;
 
-            var sections = await db.ToListAsync(SectionsForPageQueryable(page), cancellationToken);
+            var sections = await db.ToListCachedAsync(SectionsForPageQueryable(page), cancellationToken);
 
             CopySectionsToPage(page, sections);
         }
@@ -45,16 +45,15 @@ private void CopySectionsToPage(PageDbEntity page, List<SectionDbEntity> section
 
         foreach (var cat in sectionsGroupedByCategory)
         {
-            var matching = page.Content.OfType<CategoryDbEntity>()
-                                        .FirstOrDefault(category => category != null && category.Id == cat.Key);
+            var matching = page.Content.OfType<CategoryDbEntity>().FirstOrDefault(category => category.Id == cat.Key);
 
             if (matching == null)
             {
                 logger.LogError("Could not find matching category {CategoryId} in {PageSlug}", cat.Key, page.Slug);
                 continue;
             }
 
-            bool sectionsValid = AllSectionsValid(sections);
+            var sectionsValid = AllSectionsValid(sections);
 
             if (!sectionsValid)
             {

src/Dfe.PlanTech.Application/Content/Queries/GetNavigationQuery.cs

@@ -39,7 +39,7 @@ private async Task<List<NavigationLinkDbEntity>> GetFromDatabase()
     {
         try
         {
-            var navigationLinks = await _db.ToListAsync(_db.NavigationLink);
+            var navigationLinks = await _db.ToListCachedAsync(_db.NavigationLink);
 
             return navigationLinks;
         }

src/Dfe.PlanTech.Application/Content/Queries/GetPageFromDbQuery.cs

@@ -7,21 +7,13 @@
 
 namespace Dfe.PlanTech.Application.Content.Queries;
 
-public class GetPageFromDbQuery : IGetPageQuery
+public class GetPageFromDbQuery(
+    ICmsDbContext db,
+    ILogger<GetPageFromDbQuery> logger,
+    IMapper mapperConfiguration,
+    IEnumerable<IGetPageChildrenQuery> getPageChildrenQueries)
+    : IGetPageQuery
 {
-    private readonly ICmsDbContext _db;
-    private readonly ILogger<GetPageFromDbQuery> _logger;
-    private readonly IMapper _mapperConfiguration;
-    private readonly IEnumerable<IGetPageChildrenQuery> _getPageChildrenQueries;
-
-    public GetPageFromDbQuery(ICmsDbContext db, ILogger<GetPageFromDbQuery> logger, IMapper mapperConfiguration, IEnumerable<IGetPageChildrenQuery> getPageChildrenQueries)
-    {
-        _db = db;
-        _logger = logger;
-        _mapperConfiguration = mapperConfiguration;
-        _getPageChildrenQueries = getPageChildrenQueries;
-    }
-
     /// <summary>
     /// Fetches page from <see cref="ICmsDbContext"/> by slug
     /// </summary>
@@ -36,11 +28,11 @@ public GetPageFromDbQuery(ICmsDbContext db, ILogger<GetPageFromDbQuery> logger,
             if (page == null)
                 return null;
 
-            return _mapperConfiguration.Map<PageDbEntity, Page>(page);
+            return mapperConfiguration.Map<PageDbEntity, Page>(page);
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error fetching {page} from database", slug);
+            logger.LogError(ex, "Error fetching {page} from database", slug);
             throw new InvalidOperationException("Error while fetching page", ex);
         }
     }
@@ -56,7 +48,7 @@ public GetPageFromDbQuery(ICmsDbContext db, ILogger<GetPageFromDbQuery> logger,
 
         await LoadPageChildrenFromDatabase(page, cancellationToken);
 
-        _logger.LogTrace("Successfully retrieved {page} from DB", slug);
+        logger.LogTrace("Successfully retrieved {page} from DB", slug);
 
         return page;
     }
@@ -65,7 +57,7 @@ public GetPageFromDbQuery(ICmsDbContext db, ILogger<GetPageFromDbQuery> logger,
     {
         try
         {
-            var page = await _db.GetPageBySlug(slug, cancellationToken);
+            var page = await db.GetPageBySlug(slug, cancellationToken);
 
             if (page == null)
                 return null;
@@ -76,23 +68,23 @@ public GetPageFromDbQuery(ICmsDbContext db, ILogger<GetPageFromDbQuery> logger,
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error fetching {page} from database", slug);
-            throw new InvalidOperationException("Error while fetching page from database", ex);
+            logger.LogError(ex, "Error fetching {page} from database", slug);
+            return null;
         }
     }
 
     private async Task LoadPageChildrenFromDatabase(PageDbEntity? page, CancellationToken cancellationToken)
     {
         try
         {
-            foreach (var query in _getPageChildrenQueries)
+            foreach (var query in getPageChildrenQueries)
             {
                 await query.TryLoadChildren(page!, cancellationToken);
             }
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error loading children from database for {page}", page!.Id);
+            logger.LogError(ex, "Error loading children from database for {page}", page!.Id);
             throw new InvalidOperationException("Error while loading page children", ex);
         }
     }
@@ -107,16 +99,15 @@ private bool IsValidPage(PageDbEntity? page, string slug)
     {
         if (page == null)
         {
-            _logger.LogInformation("Could not find page {slug} in DB - checking Contentful", slug);
+            logger.LogInformation("Could not find page {slug} in DB - checking Contentful", slug);
             return false;
         }
 
-        if (page.Content == null || page.Content.Count == 0)
-        {
-            _logger.LogWarning("Page {slug} has no 'Content' in DB - checking Contentful", slug);
-            return false;
-        }
+        if (page.Content.Count != 0)
+            return true;
+
+        logger.LogWarning("Page {slug} has no 'Content' in DB - checking Contentful", slug);
+        return false;
 
-        return true;
     }
 }

src/Dfe.PlanTech.Application/Content/Queries/GetPageQuery.cs

@@ -5,18 +5,15 @@ namespace Dfe.PlanTech.Application.Content.Queries;
 
 public class GetPageQuery(GetPageFromContentfulQuery getPageFromContentfulQuery, GetPageFromDbQuery getPageFromDbQuery) : IGetPageQuery
 {
-    private readonly GetPageFromDbQuery _getPageFromDbQuery = getPageFromDbQuery;
-    private readonly GetPageFromContentfulQuery _getPageFromContentfulQuery = getPageFromContentfulQuery;
-
     /// <summary>
     /// Fetches page from <see chref="IContentRepository"/> by slug
     /// </summary>
     /// <param name="slug">Slug for the Page</param>
+    /// <param name="cancellationToken"></param>
     /// <returns>Page matching slug</returns>
     public async Task<Page?> GetPageBySlug(string slug, CancellationToken cancellationToken = default)
     {
-        var page = await _getPageFromDbQuery.GetPageBySlug(slug, cancellationToken) ??
-                    await _getPageFromContentfulQuery.GetPageBySlug(slug, cancellationToken);
+        var page = await getPageFromDbQuery.GetPageBySlug(slug, cancellationToken) ?? await getPageFromContentfulQuery.GetPageBySlug(slug, cancellationToken);
 
         return page;
     }

src/Dfe.PlanTech.Application/Content/Queries/GetRichTextsQuery.cs

@@ -22,16 +22,27 @@ public async Task TryLoadChildren(PageDbEntity page, CancellationToken cancellat
     {
         try
         {
-            var hasTextBodyContents = page.Content.Concat(page.BeforeTitleContent)
-                                                .OfType<IHasRichText>()
-                                                .Any();
+            var richTextParents = page.GetAllContentOfType<IHasRichText>().ToArray();
 
-            if (!hasTextBodyContents)
+            if (richTextParents.Length == 0)
                 return;
 
             var richTextContentQuery = _db.RichTextContentWithSlugs.Where(PageMatchesSlugAndPublishedOrIsPreview(page));
 
-            await _db.ToListAsync(richTextContentQuery, cancellationToken);
+            var richTexts = await _db.ToListCachedAsync(richTextContentQuery, cancellationToken);
+
+            foreach (var parent in richTextParents)
+            {
+                var matching = richTexts.FirstOrDefault(rt => rt.Id == parent.RichTextId);
+
+                if (matching == null)
+                {
+                    logger.LogError("Unable to find matching rich text for rich text Id {Id}", parent.RichTextId);
+                    continue;
+                }
+
+                parent.RichText = matching;
+            }
         }
         catch (Exception ex)
         {

src/Dfe.PlanTech.Application/Persistence/Interfaces/IDbContext.cs

@@ -2,6 +2,8 @@ namespace Dfe.PlanTech.Application.Persistence.Interfaces;
 
 public interface IDbContext
 {
+    public Task<T?> FirstOrDefaultCachedAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default);
+    public Task<List<T>> ToListCachedAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default);
     public Task<T?> FirstOrDefaultAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default);
     public Task<List<T>> ToListAsync<T>(IQueryable<T> queryable, CancellationToken cancellationToken = default);
     public Task<int> SaveChangesAsync(CancellationToken cancellationToken);

src/Dfe.PlanTech.Application/Persistence/Mappings/CategoryMapper.cs

@@ -34,6 +34,6 @@ protected override async Task PostUpdateEntityCallback(MappedEntity mappedEntity
         await EntityUpdater.UpdateReferences(incoming, existing, (category) => category.Sections, _incomingSections, true, cancellationToken);
     }
 
-    private static async Task<List<SectionDbEntity>> GetExistingSections(ICmsDbContext db, CategoryDbEntity incoming, CancellationToken cancellationToken)
-        => await db.ToListAsync(db.Sections.Where(section => section.CategoryId == incoming.Id).Select(section => section), cancellationToken);
+    private static Task<List<SectionDbEntity>> GetExistingSections(ICmsDbContext db, CategoryDbEntity incoming, CancellationToken cancellationToken)
+        => db.ToListAsync(db.Sections.Where(section => section.CategoryId == incoming.Id).Select(section => section), cancellationToken);
 }

src/Dfe.PlanTech.Application/Questionnaire/Queries/GetSectionQuery.cs

@@ -34,7 +34,7 @@ public GetSectionQuery(ICmsDbContext db, IContentRepository repository, IMapper
         var query = _db.Sections.Where(SlugMatchesInterstitialPage(sectionSlug))
                                 .Select(ProjectSection);
 
-        var section = await _db.FirstOrDefaultAsync(query, cancellationToken);
+        var section = await _db.FirstOrDefaultCachedAsync(query, cancellationToken);
 
         if (section == null)
             return null;

src/Dfe.PlanTech.Application/Submissions/Commands/DeleteCurrentSubmissionCommand.cs

@@ -22,8 +22,7 @@ public async Task DeleteCurrentSubmission(ISectionComponent section, Cancellatio
 
         await _db.ExecuteSqlAsync($@"EXEC DeleteCurrentSubmission
                                         @establishmentId={establishmentId},
-                                        @sectionId={section.Sys.Id},
-                                        @sectionName={section.Name}",
+                                        @sectionId={section.Sys.Id}",
                                         cancellationToken);
     }
 }

src/Dfe.PlanTech.DatabaseUpgrader/DatabaseExecutor.cs

@@ -14,6 +14,7 @@ public class DatabaseExecutor
 
     private const string SCRIPTS_NAMESPACE = "Dfe.PlanTech.DatabaseUpgrader.Scripts";
     private const string ENVIRONMENT_SPECIFIC_SCRIPTS_NAMESPACE = "Dfe.PlanTech.DatabaseUpgrader.EnvironmentSpecificScripts";
+    private const int EXECUTION_TIMEOUT_MINUTES = 10;
 
     public DatabaseExecutor(Options options, Logger logger)
     {
@@ -43,6 +44,7 @@ private UpgradeEngine CreateUpgradeEngine()
         return engine.LogToConsole()
         .LogScriptOutput()
         .WithTransaction()
+        .WithExecutionTimeout(TimeSpan.FromMinutes(EXECUTION_TIMEOUT_MINUTES))
         .Build();
     }