From 0b353bb2d4ab49958f54c42e56f96546f6955613 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 11:34:24 +0700 Subject: [PATCH 1/7] Revisit filenames we use for CrdtMerge Use project code and ID in root folder name, use "crdt" and "fw" inside root folder no matter what the project's actual code or name is. --- backend/CrdtMerge/CrdtMerge.csproj | 1 + backend/CrdtMerge/CrdtMergeKernel.cs | 1 + backend/CrdtMerge/Program.cs | 41 ++++++++++++------- backend/CrdtMerge/ProjectLookupService.cs | 16 ++++++++ backend/CrdtMerge/SendReceiveHelpers.cs | 12 +++--- backend/CrdtMerge/SendReceiveService.cs | 6 ++- .../CrdtMerge/appsettings.Development.json | 3 ++ backend/LexData/DataKernel.cs | 3 +- 8 files changed, 59 insertions(+), 24 deletions(-) create mode 100644 backend/CrdtMerge/ProjectLookupService.cs diff --git a/backend/CrdtMerge/CrdtMerge.csproj b/backend/CrdtMerge/CrdtMerge.csproj index d828a99e4..0380eb189 100644 --- a/backend/CrdtMerge/CrdtMerge.csproj +++ b/backend/CrdtMerge/CrdtMerge.csproj @@ -20,6 +20,7 @@ + diff --git a/backend/CrdtMerge/CrdtMergeKernel.cs b/backend/CrdtMerge/CrdtMergeKernel.cs index caa7e6113..0c312c074 100644 --- a/backend/CrdtMerge/CrdtMergeKernel.cs +++ b/backend/CrdtMerge/CrdtMergeKernel.cs @@ -15,6 +15,7 @@ public static void AddCrdtMerge(this IServiceCollection services) .ValidateDataAnnotations() .ValidateOnStart(); services.AddScoped(); + services.AddScoped(); services .AddLcmCrdtClient() .AddFwDataBridge() diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index 030692c48..62217dd1d 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -2,6 +2,8 @@ using FwDataMiniLcmBridge; using FwLiteProjectSync; using LcmCrdt; +using LexData; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; using MiniLcm; using Scalar.AspNetCore; @@ -12,6 +14,11 @@ // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); +builder.Services.AddLexData( + autoApplyMigrations: false, + useOpenIddict: false +); + builder.Services.AddCrdtMerge(); var app = builder.Build(); @@ -30,56 +37,60 @@ app.Run(); -static async Task ExecuteMergeRequest( +static async Task, NotFound>> ExecuteMergeRequest( ILogger logger, IServiceProvider services, SendReceiveService srService, IOptions config, FwDataFactory fwDataFactory, ProjectsService projectsService, + ProjectLookupService projectLookupService, CrdtFwdataProjectSyncService syncService, string projectCode, - // string projectName, // TODO: Add this to the API eventually bool dryRun = false) { logger.LogInformation("About to execute sync request for {projectCode}", projectCode); if (dryRun) { logger.LogInformation("Dry run, not actually syncing"); - return new(0, 0); + return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0)); + } + + var projectId = await projectLookupService.GetProjectId(projectCode); + if (projectId is null) + { + logger.LogError("Project code {projectCode} not found", projectCode); + return TypedResults.NotFound(); } - // TODO: Instead of projectCode here, we'll evetually look up project ID and use $"{projectName}-{projectId}" as the project folder - var projectFolder = Path.Join(config.Value.ProjectStorageRoot, projectCode); + var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}"); if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder); - // TODO: add projectName parameter and use it instead of projectCode here - var crdtFile = Path.Join(projectFolder, $"{projectCode}.sqlite"); + var crdtFile = Path.Join(projectFolder, "crdt.sqlite"); - var fwDataProject = new FwDataProject(projectCode, projectFolder); // TODO: use projectName (once we have it) instead of projectCode here + var fwDataProject = new FwDataProject("fw", projectFolder); logger.LogDebug("crdtFile: {crdtFile}", crdtFile); logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath); if (File.Exists(fwDataProject.FilePath)) { - var srResult = srService.SendReceive(fwDataProject); + var srResult = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } else { - var srResult = srService.Clone(fwDataProject); + var srResult = srService.Clone(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true); - // var crdtProject = projectsService.GetProject(crdtProjectName); var crdtProject = File.Exists(crdtFile) ? - new CrdtProject(projectCode, crdtFile) : // TODO: use projectName (once we have it) instead of projectCode here - await projectsService.CreateProject(new(projectCode, SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); + new CrdtProject("crdt", crdtFile) : + await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); var miniLcmApi = await services.OpenCrdtProject(crdtProject); var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); - var srResult2 = srService.SendReceive(fwDataProject); + var srResult2 = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); - return result; + return TypedResults.Ok(result); } diff --git a/backend/CrdtMerge/ProjectLookupService.cs b/backend/CrdtMerge/ProjectLookupService.cs new file mode 100644 index 000000000..2301fc888 --- /dev/null +++ b/backend/CrdtMerge/ProjectLookupService.cs @@ -0,0 +1,16 @@ +using LexData; +using Microsoft.EntityFrameworkCore; + +namespace CrdtMerge; + +public class ProjectLookupService(LexBoxDbContext dbContext) +{ + public async ValueTask GetProjectId(string projectCode) + { + var projectId = await dbContext.Projects + .Where(p => p.Code == projectCode) + .Select(p => p.Id) + .FirstOrDefaultAsync(); + return projectId; + } +} diff --git a/backend/CrdtMerge/SendReceiveHelpers.cs b/backend/CrdtMerge/SendReceiveHelpers.cs index b3fb283af..8eb1404c0 100644 --- a/backend/CrdtMerge/SendReceiveHelpers.cs +++ b/backend/CrdtMerge/SendReceiveHelpers.cs @@ -45,14 +45,14 @@ private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendR return builder.Uri; } - public static LfMergeBridgeResult SendReceive(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) + public static LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) { - // If projectCode not given, calculate it from the fwdataPath + projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); if (fwdataInfo.Directory is null) throw new InvalidOperationException( $"Not allowed to Send/Receive root-level directories like C:\\, was '{project.FilePath}'"); - var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); + var repoUrl = BuildSendReceiveUrl(baseUrl, projectCode, auth); var flexBridgeOptions = new Dictionary { @@ -68,13 +68,13 @@ public static LfMergeBridgeResult SendReceive(FwDataProject project, string base return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions); } - public static LfMergeBridgeResult CloneProject(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") + public static LfMergeBridgeResult CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") { - // If projectCode not given, calculate it from the fwdataPath + projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); if (fwdataInfo.Directory is null) throw new InvalidOperationException($"Not allowed to Send/Receive root-level directories like C:\\ '{project.FilePath}'"); - var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); + var repoUrl = BuildSendReceiveUrl(baseUrl, projectCode, auth); var flexBridgeOptions = new Dictionary { diff --git a/backend/CrdtMerge/SendReceiveService.cs b/backend/CrdtMerge/SendReceiveService.cs index 093427bec..14db577ae 100644 --- a/backend/CrdtMerge/SendReceiveService.cs +++ b/backend/CrdtMerge/SendReceiveService.cs @@ -5,10 +5,11 @@ namespace CrdtMerge; public class SendReceiveService(IOptions config) { - public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? commitMessage = null) + public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) { return SendReceiveHelpers.SendReceive( project: project, + projectCode: projectCode, baseUrl: config.Value.HgWebUrl, auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), fdoDataModelVersion: config.Value.FdoDataModelVersion, @@ -16,10 +17,11 @@ public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, ); } - public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project) + public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project, string? projectCode) { return SendReceiveHelpers.CloneProject( project: project, + projectCode: projectCode, baseUrl: config.Value.HgWebUrl, auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), fdoDataModelVersion: config.Value.FdoDataModelVersion diff --git a/backend/CrdtMerge/appsettings.Development.json b/backend/CrdtMerge/appsettings.Development.json index c1e390265..0995c2046 100644 --- a/backend/CrdtMerge/appsettings.Development.json +++ b/backend/CrdtMerge/appsettings.Development.json @@ -6,6 +6,9 @@ "LexboxPassword": "pass", "FdoDataModelVersion": "7000072" }, + "DbConfig": { + "LexBoxConnectionString": "Host=localhost;Port=5433;Username=postgres;Password=972b722e63f549938d07bd8c4ee5086c;Database=lexbox;Include Error Detail=true" + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/backend/LexData/DataKernel.cs b/backend/LexData/DataKernel.cs index 861252845..e12fc003c 100644 --- a/backend/LexData/DataKernel.cs +++ b/backend/LexData/DataKernel.cs @@ -9,6 +9,7 @@ public static class DataKernel { public static void AddLexData(this IServiceCollection services, bool autoApplyMigrations, + bool useOpenIddict = true, ServiceLifetime dbContextLifeTime = ServiceLifetime.Scoped) { services.AddScoped(); @@ -17,7 +18,7 @@ public static void AddLexData(this IServiceCollection services, options.EnableDetailedErrors(); options.UseNpgsql(serviceProvider.GetRequiredService>().Value.LexBoxConnectionString); options.UseProjectables(); - options.UseOpenIddict(); + if (useOpenIddict) options.UseOpenIddict(); #if DEBUG options.EnableSensitiveDataLogging(); #endif From da2a2395a7f478e77938fa6c1bdfe7cf1f54cba1 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 12:00:14 +0700 Subject: [PATCH 2/7] Update CRDT caching so same filename isn't a problem Using the name "crdt" everywhere as a project name caused a couple of caching issues in existing CRDT code, but the change is pretty easy. --- backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 2 +- backend/FwLite/LcmCrdt/CurrentProjectService.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 6322aff39..3490145a8 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -38,7 +38,7 @@ public FwDataMiniLcmApi GetFwDataMiniLcmApi(string projectName, bool saveOnDispo return GetFwDataMiniLcmApi(project, saveOnDispose); } - private string CacheKey(FwDataProject project) => $"{nameof(FwDataFactory)}|{project.FileName}"; + private string CacheKey(FwDataProject project) => $"{nameof(FwDataFactory)}|{project.FilePath}"; public FwDataMiniLcmApi GetFwDataMiniLcmApi(FwDataProject project, bool saveOnDispose) { diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index 336ad465c..f2c69dd05 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -27,7 +27,7 @@ public async ValueTask GetProjectData() private static string CacheKey(CrdtProject project) { - return project.Name + "|ProjectData"; + return project.DbPath + "|ProjectData"; } private static string CacheKey(Guid projectId) From a9aed68fc8ebd8a7d505fde3aa6ae6833d0db12a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 12:10:35 +0700 Subject: [PATCH 3/7] Now use project ID, not code, in CrdtMerge API --- backend/CrdtMerge/Program.cs | 10 +++++----- backend/CrdtMerge/ProjectLookupService.cs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index 62217dd1d..cb1a6310b 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -46,20 +46,20 @@ ProjectsService projectsService, ProjectLookupService projectLookupService, CrdtFwdataProjectSyncService syncService, - string projectCode, + Guid projectId, bool dryRun = false) { - logger.LogInformation("About to execute sync request for {projectCode}", projectCode); + logger.LogInformation("About to execute sync request for {projectId}", projectId); if (dryRun) { logger.LogInformation("Dry run, not actually syncing"); return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0)); } - var projectId = await projectLookupService.GetProjectId(projectCode); - if (projectId is null) + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) { - logger.LogError("Project code {projectCode} not found", projectCode); + logger.LogError("Project ID {projectId} not found", projectId); return TypedResults.NotFound(); } diff --git a/backend/CrdtMerge/ProjectLookupService.cs b/backend/CrdtMerge/ProjectLookupService.cs index 2301fc888..ac8244e99 100644 --- a/backend/CrdtMerge/ProjectLookupService.cs +++ b/backend/CrdtMerge/ProjectLookupService.cs @@ -5,12 +5,12 @@ namespace CrdtMerge; public class ProjectLookupService(LexBoxDbContext dbContext) { - public async ValueTask GetProjectId(string projectCode) + public async ValueTask GetProjectCode(Guid projectId) { - var projectId = await dbContext.Projects - .Where(p => p.Code == projectCode) - .Select(p => p.Id) + var projectCode = await dbContext.Projects + .Where(p => p.Id == projectId) + .Select(p => p.Code) .FirstOrDefaultAsync(); - return projectId; + return projectCode; } } From 2b4d5646e98133fa6248b5a7177989f743b22c91 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 13:42:17 +0700 Subject: [PATCH 4/7] Fix weird bug where current project wasn't seen Details are a bit of a long story but has to do with how AsyncLocal works, and which async variables are in scope at which point. --- backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index eabf4aae5..90dee868c 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -152,10 +152,15 @@ public static void ConfigureCrdt(CrdtConfig config) .Add(); } - public static async Task OpenCrdtProject(this IServiceProvider services, CrdtProject project) + public static Task OpenCrdtProject(this IServiceProvider services, CrdtProject project) { var projectsService = services.GetRequiredService(); projectsService.SetProjectScope(project); + return LoadMiniLcmApi(services); + } + + private static async Task LoadMiniLcmApi(IServiceProvider services) + { await services.GetRequiredService().PopulateProjectDataCache(); return services.GetRequiredService(); } From db99b632b176b1d69055666535b9107de8474c2d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 13:44:50 +0700 Subject: [PATCH 5/7] Log project code after looking it up --- backend/CrdtMerge/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index cb1a6310b..e2604a9ec 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -62,6 +62,7 @@ logger.LogError("Project ID {projectId} not found", projectId); return TypedResults.NotFound(); } + logger.LogInformation("Project code is {projectCode}", projectCode); var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}"); if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder); From 97a25b957640d9d66732ded45bf0cdccbd39821b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 28 Oct 2024 14:29:50 +0700 Subject: [PATCH 6/7] test `OpenCrdtProject` and add a comment explaining why it works this way --- .../FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj | 2 +- .../FwLite/LcmCrdt.Tests/OpenProjectTests.cs | 26 +++++++++++++++++++ backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs diff --git a/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj b/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj index 1eacc4fa7..2b9260c62 100644 --- a/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj +++ b/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj @@ -14,7 +14,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs new file mode 100644 index 000000000..e53ea52e8 --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs @@ -0,0 +1,26 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace LcmCrdt.Tests; + +public class OpenProjectTests +{ + [Fact] + public async Task OpeningAProjectWorks() + { + var sqliteConnectionString = "OpeningAProjectWorks.sqlite"; + var builder = Host.CreateEmptyApplicationBuilder(null); + builder.Services.AddLcmCrdtClient(); + using var host = builder.Build(); + var services = host.Services; + var asyncScope = services.CreateAsyncScope(); + await asyncScope.ServiceProvider.GetRequiredService() + .CreateProject(new(Name: "OpeningAProjectWorks", Path: "")); + + var miniLcmApi = (CrdtMiniLcmApi)await asyncScope.ServiceProvider.OpenCrdtProject(new CrdtProject("OpeningAProjectWorks", sqliteConnectionString)); + miniLcmApi.ProjectData.Name.Should().Be("OpeningAProjectWorks"); + + await asyncScope.ServiceProvider.GetRequiredService().Database.EnsureDeletedAsync(); + } +} diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 90dee868c..93ea9587d 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -154,6 +154,9 @@ public static void ConfigureCrdt(CrdtConfig config) public static Task OpenCrdtProject(this IServiceProvider services, CrdtProject project) { + //this method must not be async, otherwise Setting the project scope will not work as expected. + //the project is stored in the async scope, if a new scope is created in this method then it will be gone once the method returns + //making the lcm api unusable var projectsService = services.GetRequiredService(); projectsService.SetProjectScope(project); return LoadMiniLcmApi(services); From cde1ab364242b58f7f6292764105894a557940a1 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 28 Oct 2024 14:37:16 +0700 Subject: [PATCH 7/7] ensure fw project is saved during the sync --- backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 8b7278591..5b18b044b 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -22,6 +22,7 @@ public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataA } var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); + fwdataApi.Save(); if (!dryRun) {