-
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create a new web service called CrdtMerge, which offers a single API endpoint, `/sync?projectCode=(code)`. It will eventually take a `projectName` parameter as well. This does a Send/Receive first to ensure that any changes from others have been pulled into the .fwdata file. It then synchronizes the CRDT view of the project with the .fwdata file, and then does another Send/Receive to push the CRDT changes (now in .fwdata) up to the project on Lexbox so that other FieldWorks users can receive them. This PR does not contain a Kubernetes deployment for CrdtMerge; that will come in a follow-up PR. --------- Co-authored-by: Kevin Hahn <kevin_hahn@sil.org>
- Loading branch information
Showing
24 changed files
with
476 additions
and
129 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net9.0</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Mercurial4ChorusDestDir>$(MSBuildProjectDirectory)</Mercurial4ChorusDestDir> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.0-rc.1.24452.1" /> | ||
| <PackageReference Include="Scalar.AspNetCore" Version="1.2.22" /> | ||
| <PackageReference Include="SIL.ChorusPlugin.LfMergeBridge" Version="4.2.0-beta0027" /> | ||
| <PackageReference Include="SIL.Chorus.Mercurial" Version="6.5.1.*" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="../harmony/src/SIL.Harmony/SIL.Harmony.csproj" /> | ||
| <ProjectReference Include="../FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj" /> | ||
| <ProjectReference Include="../FwLite/LcmCrdt/LcmCrdt.csproj" /> | ||
| <ProjectReference Include="../FwLite/FwLiteProjectSync/FwLiteProjectSync.csproj" /> | ||
| <ProjectReference Include="../FixFwData/FixFwData.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Content Include="Mercurial\**" CopyToOutputDirectory="Always" /> | ||
| <Content Include="MercurialExtensions\**" CopyToOutputDirectory="Always" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| using System.ComponentModel.DataAnnotations; | ||
|
|
||
| namespace CrdtMerge; | ||
|
|
||
| public class CrdtMergeConfig | ||
| { | ||
| [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] | ||
| public required string LexboxUrl { get; init; } | ||
| public string HgWebUrl => $"{LexboxUrl}hg/"; | ||
| [Required] | ||
| public required string LexboxUsername { get; init; } | ||
| [Required] | ||
| public required string LexboxPassword { get; init; } | ||
| [Required] | ||
| public required string ProjectStorageRoot { get; init; } | ||
| public string FdoDataModelVersion { get; init; } = "7000072"; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| using FwDataMiniLcmBridge; | ||
| using FwLiteProjectSync; | ||
| using LcmCrdt; | ||
|
|
||
| namespace CrdtMerge; | ||
|
|
||
| public static class CrdtMergeKernel | ||
| { | ||
| public static void AddCrdtMerge(this IServiceCollection services) | ||
| { | ||
| services | ||
| .AddLogging(builder => builder.AddConsole().AddDebug().AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning)); | ||
| services.AddOptions<CrdtMergeConfig>() | ||
| .BindConfiguration("SendReceiveConfig") | ||
| .ValidateDataAnnotations() | ||
| .ValidateOnStart(); | ||
| services.AddScoped<SendReceiveService>(); | ||
| services | ||
| .AddLcmCrdtClient() | ||
| .AddFwDataBridge() | ||
| .AddFwLiteProjectSync(); | ||
| } | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| using CrdtMerge; | ||
| using FwDataMiniLcmBridge; | ||
| using FwLiteProjectSync; | ||
| using LcmCrdt; | ||
| using Microsoft.Extensions.Options; | ||
| using MiniLcm; | ||
| using Scalar.AspNetCore; | ||
|
|
||
| var builder = WebApplication.CreateBuilder(args); | ||
|
|
||
| // Add services to the container. | ||
| // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi | ||
| builder.Services.AddOpenApi(); | ||
|
|
||
| builder.Services.AddCrdtMerge(); | ||
|
|
||
| var app = builder.Build(); | ||
|
|
||
| // Configure the HTTP request pipeline. | ||
| if (app.Environment.IsDevelopment()) | ||
| { | ||
| app.MapOpenApi(); | ||
| //access at /scalar/v1 | ||
| app.MapScalarApiReference(); | ||
| } | ||
|
|
||
| app.UseHttpsRedirection(); | ||
|
|
||
| app.MapPost("/sync", ExecuteMergeRequest); | ||
|
|
||
| app.Run(); | ||
|
|
||
| static async Task<CrdtFwdataProjectSyncService.SyncResult> ExecuteMergeRequest( | ||
| ILogger<Program> logger, | ||
| IServiceProvider services, | ||
| SendReceiveService srService, | ||
| IOptions<CrdtMergeConfig> config, | ||
| FwDataFactory fwDataFactory, | ||
| ProjectsService projectsService, | ||
| CrdtFwdataProjectSyncService syncService, | ||
| string projectCode, | ||
| // string projectName, // TODO: Add this to the API eventually | ||
| bool dryRun = false) | ||
| { | ||
| logger.LogInformation("About to execute sync request for {projectCode}", projectCode); | ||
| if (dryRun) | ||
| { | ||
| logger.LogInformation("Dry run, not actually syncing"); | ||
| return new(0, 0); | ||
| } | ||
|
|
||
| // TODO: Instead of projectCode here, we'll evetually look up project ID and use $"{projectName}-{projectId}" as the project folder | ||
| var projectFolder = Path.Join(config.Value.ProjectStorageRoot, projectCode); | ||
| if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder); | ||
|
|
||
| // TODO: add projectName parameter and use it instead of projectCode here | ||
| var crdtFile = Path.Join(projectFolder, $"{projectCode}.sqlite"); | ||
|
|
||
| var fwDataProject = new FwDataProject(projectCode, projectFolder); // TODO: use projectName (once we have it) instead of projectCode here | ||
| logger.LogDebug("crdtFile: {crdtFile}", crdtFile); | ||
| logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath); | ||
|
|
||
| if (File.Exists(fwDataProject.FilePath)) | ||
| { | ||
| var srResult = srService.SendReceive(fwDataProject); | ||
| logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); | ||
| } | ||
| else | ||
| { | ||
| var srResult = srService.Clone(fwDataProject); | ||
| logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); | ||
| } | ||
| var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true); | ||
| // var crdtProject = projectsService.GetProject(crdtProjectName); | ||
| var crdtProject = File.Exists(crdtFile) ? | ||
| new CrdtProject(projectCode, crdtFile) : // TODO: use projectName (once we have it) instead of projectCode here | ||
| await projectsService.CreateProject(new(projectCode, fwdataApi.ProjectId, SeedNewProjectData: false, Path: projectFolder)); | ||
| var miniLcmApi = await services.OpenCrdtProject(crdtProject); | ||
| var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); | ||
| logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); | ||
| var srResult2 = srService.SendReceive(fwDataProject); | ||
| logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); | ||
| return result; | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "$schema": "https://json.schemastore.org/launchsettings.json", | ||
| "profiles": { | ||
| "http": { | ||
| "commandName": "Project", | ||
| "dotnetRunMessages": true, | ||
| "applicationUrl": "http://localhost:5275", | ||
| "environmentVariables": { | ||
| "ASPNETCORE_ENVIRONMENT": "Development" | ||
| } | ||
| }, | ||
| "https": { | ||
| "commandName": "Project", | ||
| "dotnetRunMessages": true, | ||
| "applicationUrl": "https://localhost:7003;http://localhost:5275", | ||
| "environmentVariables": { | ||
| "ASPNETCORE_ENVIRONMENT": "Development" | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| using FwDataMiniLcmBridge; | ||
| using SIL.Progress; | ||
|
|
||
| namespace CrdtMerge; | ||
|
|
||
| public static class SendReceiveHelpers | ||
| { | ||
| public record ProjectPath(string Code, string Dir) | ||
| { | ||
| public string FwDataFile { get; } = Path.Join(Dir, $"{Code}.fwdata"); | ||
| } | ||
|
|
||
| public record SendReceiveAuth(string Username, string Password) | ||
| { | ||
| public SendReceiveAuth(CrdtMergeConfig config) : this(config.LexboxUsername, config.LexboxPassword) { } | ||
| }; | ||
|
|
||
| public record LfMergeBridgeResult(string Output, string ProgressMessages); | ||
|
|
||
| private static LfMergeBridgeResult CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions) | ||
| { | ||
| var progress = new StringBuilderProgress(); | ||
| LfMergeBridge.LfMergeBridge.Execute(method, progress, flexBridgeOptions.ToDictionary(), out var lfMergeBridgeOutputForClient); | ||
| return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress.ToString()); | ||
| } | ||
|
|
||
| private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendReceiveAuth? auth) | ||
| { | ||
| var baseUri = new Uri(baseUrl); | ||
| var projectUri = new Uri(baseUri, projectCode); | ||
| if (auth == null) return projectUri; | ||
| // Stop Chorus from saving passwords, since we're not a GUI app (and it calls Windows-only APIs anyway) | ||
| var chorusSettings = new Chorus.Model.ServerSettingsModel | ||
| { | ||
| RememberPassword = false, | ||
| Username = auth.Username, | ||
| Password = auth.Password | ||
| }; | ||
| // Chorus relies too much on its global ServerSettingsModel.PasswordForSession variable | ||
| chorusSettings.SaveUserSettings(); | ||
| // TODO: Consider a global S/R lock because of Chorus's PasswordForSession behavior | ||
| var builder = new UriBuilder(projectUri); | ||
| builder.UserName = auth.Username; | ||
| builder.Password = auth.Password; | ||
| return builder.Uri; | ||
| } | ||
|
|
||
| public static LfMergeBridgeResult SendReceive(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) | ||
| { | ||
| // If projectCode not given, calculate it from the fwdataPath | ||
| var fwdataInfo = new FileInfo(project.FilePath); | ||
| if (fwdataInfo.Directory is null) throw new InvalidOperationException( | ||
| $"Not allowed to Send/Receive root-level directories like C:\\, was '{project.FilePath}'"); | ||
|
|
||
| var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); | ||
|
|
||
| var flexBridgeOptions = new Dictionary<string, string> | ||
| { | ||
| { "fullPathToProject", fwdataInfo.Directory.FullName }, | ||
| { "fwdataFilename", fwdataInfo.Name }, | ||
| { "fdoDataModelVersion", fdoDataModelVersion }, | ||
| { "languageDepotRepoName", "LexBox" }, | ||
| { "languageDepotRepoUri", repoUrl.AbsoluteUri }, | ||
| { "deleteRepoIfNoSuchBranch", "false" }, | ||
| { "user", "LexBox" }, | ||
| }; | ||
| if (commitMessage is not null) flexBridgeOptions["commitMessage"] = commitMessage; | ||
| return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions); | ||
| } | ||
|
|
||
| public static LfMergeBridgeResult CloneProject(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") | ||
| { | ||
| // If projectCode not given, calculate it from the fwdataPath | ||
| var fwdataInfo = new FileInfo(project.FilePath); | ||
| if (fwdataInfo.Directory is null) throw new InvalidOperationException($"Not allowed to Send/Receive root-level directories like C:\\ '{project.FilePath}'"); | ||
|
|
||
| var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); | ||
|
|
||
| var flexBridgeOptions = new Dictionary<string, string> | ||
| { | ||
| { "fullPathToProject", fwdataInfo.Directory.FullName }, | ||
| { "fdoDataModelVersion", fdoDataModelVersion }, | ||
| { "languageDepotRepoName", "LexBox" }, | ||
| { "languageDepotRepoUri", repoUrl.ToString() }, | ||
| { "deleteRepoIfNoSuchBranch", "false" }, | ||
| }; | ||
| return CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| using FwDataMiniLcmBridge; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace CrdtMerge; | ||
|
|
||
| public class SendReceiveService(IOptions<CrdtMergeConfig> config) | ||
| { | ||
| public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? commitMessage = null) | ||
| { | ||
| return SendReceiveHelpers.SendReceive( | ||
| project: project, | ||
| baseUrl: config.Value.HgWebUrl, | ||
| auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), | ||
| fdoDataModelVersion: config.Value.FdoDataModelVersion, | ||
| commitMessage: commitMessage | ||
| ); | ||
| } | ||
|
|
||
| public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project) | ||
| { | ||
| return SendReceiveHelpers.CloneProject( | ||
| project: project, | ||
| baseUrl: config.Value.HgWebUrl, | ||
| auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), | ||
| fdoDataModelVersion: config.Value.FdoDataModelVersion | ||
| ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "SendReceiveConfig": { | ||
| "ProjectStorageRoot": "../../hgweb/repos", | ||
| "LexboxUrl": "http://localhost/", | ||
| "LexboxUsername": "admin", | ||
| "LexboxPassword": "pass", | ||
| "FdoDataModelVersion": "7000072" | ||
| }, | ||
| "Logging": { | ||
| "LogLevel": { | ||
| "Default": "Information", | ||
| "Microsoft.AspNetCore": "Warning" | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "CrdtMergeConfig": { | ||
| "LexboxUsername": null | ||
| }, | ||
| "Logging": { | ||
| "LogLevel": { | ||
| "Default": "Information", | ||
| "Microsoft.AspNetCore": "Warning" | ||
| } | ||
| }, | ||
| "AllowedHosts": "*" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.