add complex form type to miniLcm API, ensure complex forms get imported

backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs

@@ -35,14 +35,19 @@ public async Task InitializeAsync()
         await _fixture.FwDataApi.CreateEntry(_testEntry);
         await _fixture.FwDataApi.CreateEntry(new Entry()
         {
-            Id = _complexEntryId, LexemeForm = { { "en", "Pineapple" } },
-            ComplexForms = [new ComplexFormComponent() {
-                Id = Guid.NewGuid(),
-                ComponentEntryId = _complexEntryId,
-                ComponentHeadword = "Pineapple",
-                ComplexFormEntryId = _testEntry.Id,
-                ComplexFormHeadword = "Apple"
-            }]
+            Id = _complexEntryId,
+            LexemeForm = { { "en", "Pineapple" } },
+            Components =
+            [
+                new ComplexFormComponent()
+                {
+                    Id = Guid.NewGuid(),
+                    ComplexFormEntryId = _complexEntryId,
+                    ComplexFormHeadword = "Pineapple",
+                    ComponentEntryId = _testEntry.Id,
+                    ComponentHeadword = "Apple"
+                }
+            ]
         });
     }
 
@@ -66,7 +71,9 @@ public async Task FirstSyncJustDoesAnImport()
 
         var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
         var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
-        crdtEntries.Should().BeEquivalentTo(fwdataEntries);
+        crdtEntries.Should().BeEquivalentTo(fwdataEntries,
+            options => options.For(e => e.Components).Exclude(c => c.Id)
+                              .For(e => e.ComplexForms).Exclude(c => c.Id));
     }
 
     [Fact]
@@ -96,7 +103,9 @@ await crdtApi.CreateEntry(new Entry()
 
         var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
         var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
-        crdtEntries.Should().BeEquivalentTo(fwdataEntries);
+        crdtEntries.Should().BeEquivalentTo(fwdataEntries,
+            options => options.For(e => e.Components).Exclude(c => c.Id)
+                .For(e => e.ComplexForms).Exclude(c => c.Id));
     }
 
     [Fact]
@@ -117,7 +126,9 @@ public async Task UpdatingAnEntryInEachProjectSyncsAcrossBoth()
 
         var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
         var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
-        crdtEntries.Should().BeEquivalentTo(fwdataEntries);
+        crdtEntries.Should().BeEquivalentTo(fwdataEntries,
+            options => options.For(e => e.Components).Exclude(c => c.Id)
+                .For(e => e.ComplexForms).Exclude(c => c.Id));
     }
 
     [Fact]
@@ -142,6 +153,8 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth()
 
         var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
         var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
-        crdtEntries.Should().BeEquivalentTo(fwdataEntries);
+        crdtEntries.Should().BeEquivalentTo(fwdataEntries,
+            options => options.For(e => e.Components).Exclude(c => c.Id)
+                .For(e => e.ComplexForms).Exclude(c => c.Id));
     }
 }

backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs

@@ -42,6 +42,8 @@ private async Task<SyncResult> Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi,
             return new SyncResult(entryCount, 0);
         }
 
+        //todo sync complex form types, parts of speech, semantic domains, writing systems
+
         var currentFwDataEntries = await fwdataApi.GetEntries().ToArrayAsync();
         var crdtChanges = await EntrySync(currentFwDataEntries, projectSnapshot.Entries, crdtApi);
         LogDryRun(crdtApi, "crdt");

backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs

@@ -57,6 +57,18 @@ public Task CreateSemanticDomain(SemanticDomain semanticDomain)
         return Task.CompletedTask;
     }
 
+    public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
+    {
+        return api.GetComplexFormTypes();
+    }
+
+    public Task<ComplexFormType> CreateComplexFormType(ComplexFormType complexFormType)
+    {
+        DryRunRecords.Add(new DryRunRecord(nameof(CreateComplexFormType),
+            $"Create complex form type {complexFormType.Name}"));
+        return Task.FromResult(complexFormType);
+    }
+
     public IAsyncEnumerable<Entry> GetEntries(QueryOptions? options = null)
     {
         return api.GetEntries(options);

backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs

@@ -28,6 +28,12 @@ public async Task ImportProject(IMiniLcmApi importTo, IMiniLcmApi importFrom, in
             logger.LogInformation("Imported part of speech {Id}", partOfSpeech.Id);
         }
 
+        await foreach (var complexFormType in importFrom.GetComplexFormTypes())
+        {
+            await importTo.CreateComplexFormType(complexFormType);
+            logger.LogInformation("Imported complex form type {Id}", complexFormType.Id);
+        }
+
 
         var semanticDomains = importFrom.GetSemanticDomains();
         var entries = importFrom.GetEntries(new QueryOptions(Count: 100_000, Offset: 0));

backend/FwLite/LcmCrdt.Tests/Changes/ComplexFormTests.cs

@@ -51,8 +51,8 @@ public async Task AddEntryComponent()
         var coatEntry = await fixture.Api.CreateEntry(new() { LexemeForm = { { "en", "Coat" } }, });
         var rackEntry = await fixture.Api.CreateEntry(new() { LexemeForm = { { "en", "Rack" } }, });
 
-        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(complexEntry, coatEntry));
-        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(complexEntry, rackEntry));
+        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(ComplexFormComponent.FromEntries(complexEntry, coatEntry)));
+        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(ComplexFormComponent.FromEntries(complexEntry, rackEntry)));
         complexEntry = await fixture.Api.GetEntry(complexEntry.Id);
         complexEntry.Should().NotBeNull();
         complexEntry!.Components.Should().ContainSingle(e => e.ComponentEntryId == coatEntry.Id);
@@ -70,8 +70,8 @@ public async Task DeleteEntryComponent()
         var coatEntry = await fixture.Api.CreateEntry(new() { LexemeForm = { { "en", "Coat" } }, });
         var rackEntry = await fixture.Api.CreateEntry(new() { LexemeForm = { { "en", "Rack" } }, });
 
-        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(complexEntry, coatEntry));
-        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(complexEntry, rackEntry));
+        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(ComplexFormComponent.FromEntries(complexEntry, coatEntry)));
+        await fixture.DataModel.AddChange(Guid.NewGuid(), new AddEntryComponentChange(ComplexFormComponent.FromEntries(complexEntry, rackEntry)));
         complexEntry = await fixture.Api.GetEntry(complexEntry.Id);
         complexEntry.Should().NotBeNull();
         var component = complexEntry!.Components.First();

backend/FwLite/LcmCrdt/Changes/Entries/AddEntryComponentChange.cs

@@ -1,5 +1,6 @@
 using System.Text.Json.Serialization;
 using LcmCrdt.Objects;
+using MiniLcm.Models;
 using SIL.Harmony;
 using SIL.Harmony.Changes;
 using SIL.Harmony.Entities;
@@ -9,17 +10,17 @@ namespace LcmCrdt.Changes.Entries;
 public class AddEntryComponentChange : CreateChange<CrdtComplexFormComponent>, ISelfNamedType<AddEntryComponentChange>
 {
     public Guid ComplexFormEntryId { get; }
-    public string ComplexFormHeadword { get; }
+    public string? ComplexFormHeadword { get; }
     public Guid ComponentEntryId { get; }
     public Guid? ComponentSenseId { get; }
-    public string ComponentHeadword { get; }
+    public string? ComponentHeadword { get; }
 
     [JsonConstructor]
-    protected AddEntryComponentChange(Guid entityId,
+    public AddEntryComponentChange(Guid entityId,
         Guid complexFormEntryId,
-        string complexFormHeadword,
+        string? complexFormHeadword,
         Guid componentEntryId,
-        string componentHeadword,
+        string? componentHeadword,
         Guid? componentSenseId = null) : base(entityId)
     {
         ComplexFormEntryId = complexFormEntryId;
@@ -29,15 +30,12 @@ protected AddEntryComponentChange(Guid entityId,
         ComponentSenseId = componentSenseId;
     }
 
-    public AddEntryComponentChange(
-        MiniLcm.Models.Entry complexEntry,
-        MiniLcm.Models.Entry componentEntry,
-        Guid? componentSenseId = null) : this(Guid.NewGuid(),
-        complexEntry.Id,
-        complexEntry.Headword(),
-        componentEntry.Id,
-        componentEntry.Headword(),
-        componentSenseId)
+    public AddEntryComponentChange(ComplexFormComponent component) : this(component.Id == default ? Guid.NewGuid() : component.Id,
+        component.ComplexFormEntryId,
+        component.ComplexFormHeadword,
+        component.ComponentEntryId,
+        component.ComponentHeadword,
+        component.ComponentSenseId)
     {
     }
 

backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs

@@ -4,6 +4,7 @@
 using SIL.Harmony;
 using SIL.Harmony.Changes;
 using LcmCrdt.Changes;
+using LcmCrdt.Changes.Entries;
 using LcmCrdt.Objects;
 using MiniLcm;
 using LinqToDB;
@@ -21,6 +22,7 @@ public class CrdtMiniLcmApi(DataModel dataModel, JsonSerializerOptions jsonOptio
 
     private IQueryable<Entry> Entries => dataModel.GetLatestObjects<Entry>();
     private IQueryable<CrdtComplexFormComponent> ComplexFormComponents => dataModel.GetLatestObjects<CrdtComplexFormComponent>();
+    private IQueryable<CrdtComplexFormType> ComplexFormTypes => dataModel.GetLatestObjects<CrdtComplexFormType>();
     private IQueryable<Sense> Senses => dataModel.GetLatestObjects<Sense>();
     private IQueryable<ExampleSentence> ExampleSentences => dataModel.GetLatestObjects<ExampleSentence>();
     private IQueryable<WritingSystem> WritingSystems => dataModel.GetLatestObjects<WritingSystem>();
@@ -97,9 +99,15 @@ public async Task BulkImportSemanticDomains(IEnumerable<MiniLcm.Models.SemanticD
         await dataModel.AddChanges(ClientId, semanticDomains.Select(sd => new CreateSemanticDomainChange(sd.Id, sd.Name, sd.Code)));
     }
 
-    public async Task CreateComplexFormType(MiniLcm.Models.ComplexFormType complexFormType)
+    public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
+    {
+        return ComplexFormTypes.AsAsyncEnumerable();
+    }
+
+    public async Task<ComplexFormType> CreateComplexFormType(MiniLcm.Models.ComplexFormType complexFormType)
     {
         await dataModel.AddChange(ClientId, new CreateComplexFormType(complexFormType.Id, complexFormType.Name));
+        return await ComplexFormTypes.SingleAsync(c => c.Id == complexFormType.Id);
     }
 
     public IAsyncEnumerable<MiniLcm.Models.Entry> GetEntries(QueryOptions? options = null)
@@ -153,6 +161,7 @@ public async Task CreateComplexFormType(MiniLcm.Models.ComplexFormType complexFo
             .Take(options.Count);
         var entries = await queryable.ToArrayAsyncLinqToDB();
         await LoadSenses(entries);
+        await LoadComplexFormData(entries);
 
         return entries;
     }
@@ -182,6 +191,20 @@ private async Task LoadSenses(Entry[] entries)
         }
     }
 
+    private async Task LoadComplexFormData(Entry[] entries)
+    {
+        var allComponents = await ComplexFormComponents
+            .Where(c => entries.Select(e => e.Id).Contains(c.ComplexFormEntryId) || entries.Select(e => e.Id).Contains(c.ComponentEntryId))
+            .ToArrayAsyncEF();
+        var componentLookup = allComponents.ToLookup(c => c.ComplexFormEntryId).ToDictionary(c => c.Key, c => c.ToArray());
+        var complexFormLookup = allComponents.ToLookup(c => c.ComponentEntryId).ToDictionary(c => c.Key, c => c.ToArray());
+        foreach (var entry in entries)
+        {
+            entry.Components = componentLookup.TryGetValue(entry.Id, out var components) ? components.ToArray() : [];
+            entry.ComplexForms = complexFormLookup.TryGetValue(entry.Id, out var complexForms) ? complexForms.ToArray() : [];
+        }
+    }
+
     public async Task<MiniLcm.Models.Entry?> GetEntry(Guid id)
     {
         var entry = await Entries.SingleOrDefaultAsync(e => e.Id == id);
@@ -230,6 +253,16 @@ public async Task BulkCreateEntries(IAsyncEnumerable<MiniLcm.Models.Entry> entri
     private IEnumerable<IChange> CreateEntryChanges(MiniLcm.Models.Entry entry, Dictionary<Guid, SemanticDomain> semanticDomains, Dictionary<Guid, Objects.PartOfSpeech> partsOfSpeech)
     {
         yield return new CreateEntryChange(entry);
+
+        //only add components, if we add both components and complex forms we'll get duplicates
+        foreach (var addEntryComponentChange in entry.Components.Select(c => new AddEntryComponentChange(c)))
+        {
+            yield return addEntryComponentChange;
+        }
+        foreach (var addComplexFormTypeChange in entry.ComplexFormTypes.Select(c => new AddComplexFormTypeChange(entry.Id, c)))
+        {
+            yield return addComplexFormTypeChange;
+        }
         foreach (var sense in entry.Senses)
         {
             sense.SemanticDomains = sense.SemanticDomains
@@ -256,7 +289,10 @@ await dataModel.AddChanges(ClientId,
             new CreateEntryChange(entry),
             ..await entry.Senses.ToAsyncEnumerable()
                 .SelectMany(s => CreateSenseChanges(entry.Id, s))
-                .ToArrayAsync()
+                .ToArrayAsync(),
+            ..entry.Components.Select(c => new AddEntryComponentChange(c)),
+            ..entry.ComplexForms.Select(c => new AddEntryComponentChange(c)),
+            ..entry.ComplexFormTypes.Select(c => new AddComplexFormTypeChange(entry.Id, c))
         ]);
         return await GetEntry(entry.Id) ?? throw new NullReferenceException();
     }

backend/FwLite/LcmCrdt/Objects/CrdtComplexFormComponent.cs

@@ -36,6 +36,7 @@ public IObjectBase Copy()
         {
             Id = Id,
             ComplexFormEntryId = ComplexFormEntryId,
+            ComplexFormHeadword = ComplexFormHeadword,
             ComponentEntryId = ComponentEntryId,
             ComponentHeadword = ComponentHeadword,
             ComponentSenseId = ComponentSenseId,

backend/FwLite/MiniLcm/IMiniLcmReadApi.cs

@@ -8,6 +8,7 @@ public interface IMiniLcmReadApi
     Task<WritingSystems> GetWritingSystems();
     IAsyncEnumerable<PartOfSpeech> GetPartsOfSpeech();
     IAsyncEnumerable<SemanticDomain> GetSemanticDomains();
+    IAsyncEnumerable<ComplexFormType> GetComplexFormTypes();
     IAsyncEnumerable<Entry> GetEntries(QueryOptions? options = null);
     IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options = null);
     Task<Entry?> GetEntry(Guid id);

backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs

@@ -14,6 +14,7 @@ Task<WritingSystem> UpdateWritingSystem(WritingSystemId id,
 
     Task CreatePartOfSpeech(PartOfSpeech partOfSpeech);
     Task CreateSemanticDomain(SemanticDomain semanticDomain);
+    Task<ComplexFormType> CreateComplexFormType(ComplexFormType complexFormType);
     Task<Entry> CreateEntry(Entry entry);
     Task<Entry> UpdateEntry(Guid id, UpdateObjectInput<Entry> update);
     Task DeleteEntry(Guid id);

backend/FwLite/MiniLcm/InMemoryApi.cs

@@ -150,6 +150,16 @@ public IAsyncEnumerable<SemanticDomain> GetSemanticDomains()
         throw new NotImplementedException();
     }
 
+    public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<ComplexFormType> CreateComplexFormType(ComplexFormType complexFormType)
+    {
+        throw new NotImplementedException();
+    }
+
     private readonly string[] _exemplars = Enumerable.Range('a', 'z').Select(c => ((char)c).ToString()).ToArray();
 
     public Task<Entry> CreateEntry(Entry entry)

backend/FwLite/MiniLcm/Models/Entry.cs

@@ -39,6 +39,18 @@ public string Headword()
 
 public class ComplexFormComponent
 {
+    public static ComplexFormComponent FromEntries(Entry complexFormEntry, Entry componentEntry, Guid? componentSenseId = null)
+    {
+        return new ComplexFormComponent
+        {
+            Id = Guid.NewGuid(),
+            ComplexFormEntryId = complexFormEntry.Id,
+            ComplexFormHeadword = complexFormEntry.Headword(),
+            ComponentEntryId = componentEntry.Id,
+            ComponentHeadword = componentEntry.Headword(),
+            ComponentSenseId = componentSenseId,
+        };
+    }
     public Guid Id { get; set; }
     public required Guid ComplexFormEntryId { get; set; }
     public string? ComplexFormHeadword { get; set; }

backend/LfClassicData/LfClassicMiniLcmApi.cs

@@ -11,6 +11,10 @@ namespace LfClassicData;
 public class LfClassicMiniLcmApi(string projectCode, ProjectDbContext dbContext, SystemDbContext systemDbContext) : IMiniLcmReadApi
 {
     private IMongoCollection<Entities.Entry> Entries => dbContext.Entries(projectCode);
+    public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
+    {
+        return AsyncEnumerable.Empty<ComplexFormType>();
+    }
 
     public async Task<WritingSystems> GetWritingSystems()
     {