Sync CRDTs with lexbox during crdt merge (#1206)

* move crdt sync service into LcmCrdt

* setup fw-headless to sync crdt changes back to lexbox

* close fwdata project once sync is done

* defer closing the fw project with a using statement. Make SendReceive async

* change sync endpoint to 'api/crdt-sync'

backend/FwHeadless/CrdtSyncService.cs

@@ -0,0 +1,27 @@
+using LcmCrdt;
+using LcmCrdt.RemoteSync;
+using SIL.Harmony;
+
+namespace FwHeadless;
+
+public class CrdtSyncService(
+    CrdtHttpSyncService httpSyncService,
+    IHttpClientFactory httpClientFactory,
+    CurrentProjectService currentProjectService,
+    DataModel dataModel,
+    ILogger<CrdtSyncService> logger)
+{
+    public async Task Sync()
+    {
+        var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable(
+            currentProjectService.ProjectData,
+            httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)
+        );
+        var syncResults = await dataModel.SyncWith(lexboxRemoteServer);
+        if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed");
+        logger.LogInformation(
+            "Synced with Lexbox, Downloaded changes: {MissingFromLocal}, Uploaded changes: {MissingFromRemote}",
+            syncResults.MissingFromLocal.Length,
+            syncResults.MissingFromRemote.Length);
+    }
+}

backend/FwHeadless/FwHeadlessKernel.cs

@@ -2,11 +2,13 @@
 using FwHeadless.Services;
 using FwLiteProjectSync;
 using LcmCrdt;
+using Microsoft.Extensions.Options;
 
 namespace FwHeadless;
 
 public static class FwHeadlessKernel
 {
+    public const string LexboxHttpClientName = "LexboxHttpClient";
     public static void AddFwHeadless(this IServiceCollection services)
     {
         services
@@ -23,5 +25,12 @@ public static void AddFwHeadless(this IServiceCollection services)
             .AddLcmCrdtClient()
             .AddFwDataBridge()
             .AddFwLiteProjectSync();
+        services.AddScoped<CrdtSyncService>();
+        services.AddTransient<HttpClientAuthHandler>();
+        services.AddHttpClient(LexboxHttpClientName,
+            (provider, client) =>
+            {
+                client.BaseAddress = new Uri(provider.GetRequiredService<IOptions<FwHeadlessConfig>>().Value.LexboxUrl);
+            }).AddHttpMessageHandler<HttpClientAuthHandler>();
     }
-};
+}

backend/FwHeadless/HttpClientAuthHandler.cs

@@ -0,0 +1,73 @@
+using System.Net;
+using LexCore;
+using LexCore.Auth;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless;
+
+public class HttpClientAuthHandler(IOptions<FwHeadlessConfig> config, IMemoryCache cache, ILogger<HttpClientAuthHandler> logger) : DelegatingHandler
+{
+    protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        throw new NotSupportedException("use async apis");
+    }
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        var lexboxUrl = new Uri(config.Value.LexboxUrl);
+        if (request.RequestUri?.Authority != lexboxUrl.Authority)
+        {
+            return await base.SendAsync(request, cancellationToken);
+        }
+        try
+        {
+            await SetAuthHeader(request, cancellationToken, lexboxUrl);
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to set auth header", e);
+        }
+        return await base.SendAsync(request, cancellationToken);
+    }
+
+    private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl)
+    {
+        var cookieContainer = new CookieContainer();
+        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority));
+        request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl));
+    }
+
+    private async ValueTask<string> GetToken(CancellationToken cancellationToken)
+    {
+        try
+        {
+            return await cache.GetOrCreateAsync("LexboxAuthToken",
+                async entry =>
+                {
+                    if (InnerHandler is null) throw new InvalidOperationException("InnerHandler is null");
+                    logger.LogInformation("Getting auth token");
+                    var client = new HttpClient(InnerHandler);
+                    client.BaseAddress = new Uri(config.Value.LexboxUrl);
+                    var response = await client.PostAsJsonAsync("/api/login",
+                        new LoginRequest(config.Value.LexboxPassword, config.Value.LexboxUsername),
+                        cancellationToken);
+                    response.EnsureSuccessStatusCode();
+                    var cookies = response.Headers.GetValues("Set-Cookie");
+                    var cookieContainer = new CookieContainer();
+                    cookieContainer.SetCookies(response.RequestMessage!.RequestUri!, cookies.Single());
+                    var authCookie = cookieContainer.GetAllCookies()
+                        .FirstOrDefault(c => c.Name == LexAuthConstants.AuthCookieName);
+                    if (authCookie is null) throw new InvalidOperationException("Auth cookie not found");
+                    entry.SetValue(authCookie.Value);
+                    entry.AbsoluteExpiration = authCookie.Expires;
+                    logger.LogInformation("Got auth token: {AuthToken}", authCookie.Value);
+                    return authCookie.Value;
+                }) ?? throw new NullReferenceException("unable to get the login token");
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to get auth token", e);
+        }
+    }
+}

backend/FwHeadless/Program.cs

@@ -1,7 +1,9 @@
 using FwHeadless;
 using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.Api;
 using FwLiteProjectSync;
 using LcmCrdt;
+using LcmCrdt.RemoteSync;
 using LexData;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.Extensions.Options;
@@ -45,11 +47,11 @@
 
 app.MapHealthChecks("/api/healthz");
 
-app.MapPost("/sync", ExecuteMergeRequest);
+app.MapPost("/api/crdt-sync", ExecuteMergeRequest);
 
 app.Run();
 
-static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound>> ExecuteMergeRequest(
+static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
@@ -58,6 +60,8 @@
     ProjectsService projectsService,
     ProjectLookupService projectLookupService,
     CrdtFwdataProjectSyncService syncService,
+    CrdtHttpSyncService crdtHttpSyncService,
+    IHttpClientFactory httpClientFactory,
     Guid projectId,
     bool dryRun = false)
 {
@@ -75,6 +79,11 @@
         return TypedResults.NotFound();
     }
     logger.LogInformation("Project code is {projectCode}", projectCode);
+    //if we can't sync with lexbox fail fast
+    if (!await crdtHttpSyncService.TestAuth(httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)))
+    {
+        return TypedResults.Problem("Unable to authenticate with Lexbox");
+    }
 
     var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}");
     if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder);
@@ -85,25 +94,71 @@
     logger.LogDebug("crdtFile: {crdtFile}", crdtFile);
     logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath);
 
+    var fwdataApi = await SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory);
+    using var deferCloseFwData = fwDataFactory.DeferClose(fwDataProject);
+    var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl);
+
+    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
+    var crdtSyncService = services.GetRequiredService<CrdtSyncService>();
+    await crdtSyncService.Sync();
+
+
+    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
+    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
+
+    await crdtSyncService.Sync();
+    var srResult2 = await srService.SendReceive(fwDataProject, projectCode);
+    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
+    return TypedResults.Ok(result);
+}
+
+static async Task<FwDataMiniLcmApi> SetupFwData(FwDataProject fwDataProject,
+    SendReceiveService srService,
+    string projectCode,
+    ILogger<Program> logger,
+    FwDataFactory fwDataFactory)
+{
     if (File.Exists(fwDataProject.FilePath))
     {
-        var srResult = srService.SendReceive(fwDataProject, projectCode);
+        var srResult = await srService.SendReceive(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
     else
     {
-        var srResult = srService.Clone(fwDataProject, projectCode);
+        var srResult = await srService.Clone(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
+
     var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true);
-    var crdtProject = File.Exists(crdtFile) ?
-        new CrdtProject("crdt", crdtFile) :
-        await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId));
-    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
-    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
-    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
-    var srResult2 = srService.SendReceive(fwDataProject, projectCode);
-    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
-    return TypedResults.Ok(result);
+    return fwdataApi;
+}
+
+static async Task<CrdtProject> SetupCrdtProject(string crdtFile,
+    ProjectLookupService projectLookupService,
+    Guid projectId,
+    ProjectsService projectsService,
+    string projectFolder,
+    Guid fwProjectId,
+    string lexboxUrl)
+{
+    if (File.Exists(crdtFile))
+    {
+        return new CrdtProject("crdt", crdtFile);
+    }
+    else
+    {
+        if (await projectLookupService.IsCrdtProject(projectId))
+        {
+            //todo determine what to do in this case, maybe we just download the project?
+            throw new InvalidOperationException("Project already exists, not sure why it's not on the server");
+        }
+        return await projectsService.CreateProject(new("crdt",
+            SeedNewProjectData: false,
+            Id: projectId,
+            Path: projectFolder,
+            FwProjectId: fwProjectId,
+            Domain: new Uri(lexboxUrl)));
+    }
+
 }
 

backend/FwHeadless/ProjectLookupService.cs

@@ -1,5 +1,6 @@
 using LexData;
 using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Core;
 
 namespace FwHeadless;
 
@@ -13,4 +14,9 @@ public class ProjectLookupService(LexBoxDbContext dbContext)
             .FirstOrDefaultAsync();
         return projectCode;
     }
+
+    public async Task<bool> IsCrdtProject(Guid projectId)
+    {
+        return await dbContext.Set<ServerCommit>().AnyAsync(c => c.ProjectId == projectId);
+    }
 }

backend/FwHeadless/SendReceiveHelpers.cs

@@ -17,10 +17,14 @@ public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, co
 
     public record LfMergeBridgeResult(string Output, string ProgressMessages);
 
-    private static LfMergeBridgeResult CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions, IProgress? progress = null)
+    private static async Task<LfMergeBridgeResult> CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions, IProgress? progress = null)
     {
         var sbProgress = new StringBuilderProgress();
-        LfMergeBridge.LfMergeBridge.Execute(method, progress ?? sbProgress, flexBridgeOptions.ToDictionary(), out var lfMergeBridgeOutputForClient);
+        var lfMergeBridgeOutputForClient = await Task.Run(() =>
+        {
+            LfMergeBridge.LfMergeBridge.Execute(method, progress ?? sbProgress, flexBridgeOptions.ToDictionary(), out var output);
+            return output;
+        });
         return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress == null ? sbProgress.ToString() : "");
     }
 
@@ -45,7 +49,7 @@ private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendR
         return builder.Uri;
     }
 
-    public static LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null, IProgress? progress = null)
+    public static async Task<LfMergeBridgeResult> SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null, IProgress? progress = null)
     {
         projectCode ??= project.Name;
         var fwdataInfo = new FileInfo(project.FilePath);
@@ -65,10 +69,10 @@ public static LfMergeBridgeResult SendReceive(FwDataProject project, string? pro
             { "user", "LexBox" },
         };
         if (commitMessage is not null) flexBridgeOptions["commitMessage"] = commitMessage;
-        return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions, progress);
+        return await CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions, progress);
     }
 
-    public static LfMergeBridgeResult CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", IProgress? progress = null)
+    public static async Task<LfMergeBridgeResult> CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", IProgress? progress = null)
     {
         projectCode ??= project.Name;
         var fwdataInfo = new FileInfo(project.FilePath);
@@ -84,6 +88,6 @@ public static LfMergeBridgeResult CloneProject(FwDataProject project, string? pr
             { "languageDepotRepoUri", repoUrl.ToString() },
             { "deleteRepoIfNoSuchBranch", "false" },
         };
-        return CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions, progress);
+        return await CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions, progress);
     }
 }

backend/FwHeadless/SendReceiveService.cs

@@ -6,9 +6,9 @@ namespace FwHeadless;
 
 public class SendReceiveService(IOptions<FwHeadlessConfig> config, SafeLoggingProgress progress)
 {
-    public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null)
+    public async Task<SendReceiveHelpers.LfMergeBridgeResult> SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null)
     {
-        return SendReceiveHelpers.SendReceive(
+        return await SendReceiveHelpers.SendReceive(
             project: project,
             projectCode: projectCode,
             baseUrl: config.Value.HgWebUrl,
@@ -19,9 +19,9 @@ public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project,
         );
     }
 
-    public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project, string? projectCode)
+    public async Task<SendReceiveHelpers.LfMergeBridgeResult> Clone(FwDataProject project, string? projectCode)
     {
-        return SendReceiveHelpers.CloneProject(
+        return await SendReceiveHelpers.CloneProject(
             project: project,
             projectCode: projectCode,
             baseUrl: config.Value.HgWebUrl,

backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs

@@ -1,5 +1,6 @@
 using FwDataMiniLcmBridge.Api;
 using FwDataMiniLcmBridge.LcmUtils;
+using LexCore.Utils;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -120,7 +121,7 @@ public void CloseCurrentProject()
         CloseProject(fwDataProject);
     }
 
-    private void CloseProject(FwDataProject project)
+    public void CloseProject(FwDataProject project)
     {
         // if we are shutting down, don't do anything because we want project dispose to be called as part of the shutdown process.
         if (_shuttingDown) return;
@@ -130,4 +131,9 @@ private void CloseProject(FwDataProject project)
         if (lcmCache is null) return;
         cache.Remove(cacheKey);
     }
+
+    public IDisposable DeferClose(FwDataProject project)
+    {
+        return Defer.Action(() => CloseProject(project));
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj

@@ -28,6 +28,7 @@
         <InternalsVisibleTo Include="LcmDebugger" />
     </ItemGroup>
     <ItemGroup>
+        <ProjectReference Include="..\..\LexCore\LexCore.csproj" />
         <ProjectReference Include="..\MiniLcm\MiniLcm.csproj" />
     </ItemGroup>
 

backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs

@@ -58,7 +58,7 @@ public async Task InitializeAsync()
         if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true);
         Directory.CreateDirectory(crdtProjectsFolder);
         var crdtProject = await _services.ServiceProvider.GetRequiredService<ProjectsService>()
-            .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId));
+            .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true));
         CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject);
     }
 

backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs

@@ -16,7 +16,7 @@ public async Task OpeningAProjectWorks()
         var services = host.Services;
         var asyncScope = services.CreateAsyncScope();
         await asyncScope.ServiceProvider.GetRequiredService<ProjectsService>()
-            .CreateProject(new(Name: "OpeningAProjectWorks", Path: ""));
+            .CreateProject(new(Name: "OpeningAProjectWorks", Path: "", SeedNewProjectData: true));
 
         var miniLcmApi = (CrdtMiniLcmApi)await asyncScope.ServiceProvider.OpenCrdtProject(new CrdtProject("OpeningAProjectWorks", sqliteConnectionString));
         miniLcmApi.ProjectData.Name.Should().Be("OpeningAProjectWorks");