Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement update and delete apis for ComplexFormTypes and sync them in CrdtFwdataProjectSyncService #1295

Merged
merged 16 commits into from
Dec 4, 2024
Merged

implement update and delete apis for ComplexFormTypes and sync them in CrdtFwdataProjectSyncService #1295

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
367ab33
implement update and delete apis for ComplexFormTypes and sync them i…
hahn-kev Nov 29, 2024
43241cf
also sync writing system changes
hahn-kev Nov 29, 2024
f94a5ed
enable test which imports all data as a sync instead of the dedicated…
hahn-kev Nov 29, 2024
b09089e
enable better error logging
hahn-kev Dec 2, 2024
6459687
pull paging application into QueryOptions to make querying all consis…
hahn-kev Dec 2, 2024
840a4c1
assert that all entries were imported on sync
hahn-kev Dec 2, 2024
5a2d23b
create `GetAllEntries` helper method, and use from all sync code
hahn-kev Dec 2, 2024
8fb0e83
enable DryRunSync_MakesTheSameChangesAsSync test
hahn-kev Dec 2, 2024
44a49d7
Merge branch 'develop' into chore/sync-complex-form-types
hahn-kev Dec 3, 2024
924600b
use DisposeAsync instead of Dispose on root service provider
hahn-kev Dec 3, 2024
aaca6ed
test creating writing systems, make create WS throw if it already exists
hahn-kev Dec 3, 2024
0068f9f
fix issue with conflicts between the multiple sync tests running at o…
hahn-kev Dec 3, 2024
6464001
update verified db model
hahn-kev Dec 3, 2024
9192462
remove writing system creation from test code where redundant
hahn-kev Dec 3, 2024
55ad9da
increase timeout on fw lite build and test job
hahn-kev Dec 4, 2024
513774f
remove commited .vs files
hahn-kev Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 44 additions & 3 deletions backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,14 @@ public IAsyncEnumerable<ComplexFormType> GetComplexFormTypes()
{
return ComplexFormTypesFlattened.Select(ToComplexFormType).ToAsyncEnumerable();
}

public Task<ComplexFormType?> GetComplexFormType(Guid id)
{
var lexEntryType = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (lexEntryType is null) return Task.FromResult<ComplexFormType?>(null);
return Task.FromResult<ComplexFormType?>(ToComplexFormType(lexEntryType));
}

private ComplexFormType ToComplexFormType(ILexEntryType t)
{
return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) };
Expand All @@ -400,6 +408,40 @@ public async Task<ComplexFormType> CreateComplexFormType(ComplexFormType complex
return ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == complexFormType.Id));
}

public Task<ComplexFormType> UpdateComplexFormType(Guid id, UpdateObjectInput<ComplexFormType> update)
{
var type = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (type is null) throw new NullReferenceException($"unable to find complex form type with id {id}");
UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Update Complex Form Type",
"Revert Complex Form Type",
Cache.ServiceLocator.ActionHandler,
() =>
{
var updateProxy = new UpdateComplexFormTypeProxy(type, null, this);
update.Apply(updateProxy);
});
return Task.FromResult(ToComplexFormType(type));
}

public async Task<ComplexFormType> UpdateComplexFormType(ComplexFormType before, ComplexFormType after)
{
await ComplexFormTypeSync.Sync(before, after, this);
return ToComplexFormType(ComplexFormTypesFlattened.Single(c => c.Guid == after.Id));
}

public async Task DeleteComplexFormType(Guid id)
{
var type = ComplexFormTypesFlattened.SingleOrDefault(c => c.Guid == id);
if (type is null) return;
await Cache.DoUsingNewOrCurrentUOW("Delete Complex Form Type",
"Revert delete",
() =>
{
type.Delete();
return ValueTask.CompletedTask;
});
}

public IAsyncEnumerable<VariantType> GetVariantTypes()
{
return VariantTypes.PossibilitiesOS
Expand Down Expand Up @@ -591,9 +633,8 @@ public IAsyncEnumerable<Entry> GetEntries(
string? text = e.CitationForm.get_String(sortWs).Text;
text ??= e.LexemeFormOA.Form.get_String(sortWs).Text;
return text?.Trim(LcmHelpers.WhitespaceChars);
})
.Skip(options.Offset)
.Take(options.Count);
});
entries = options.ApplyPaging(entries);

return entries.ToAsyncEnumerable().Select(FromLexEntry);
}
Expand Down
16 changes: 13 additions & 3 deletions backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateComplexFormTypeProxy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,35 @@ namespace FwDataMiniLcmBridge.Api.UpdateProxy;
public record UpdateComplexFormTypeProxy : ComplexFormType
{
private readonly ILexEntryType _lexEntryType;
private readonly ILexEntry _lcmEntry;
private readonly ILexEntry? _lcmEntry;
private readonly FwDataMiniLcmApi _lexboxLcmApi;

[SetsRequiredMembers]
public UpdateComplexFormTypeProxy(ILexEntryType lexEntryType, ILexEntry lcmEntry, FwDataMiniLcmApi lexboxLcmApi)
public UpdateComplexFormTypeProxy(ILexEntryType lexEntryType, ILexEntry? lcmEntry, FwDataMiniLcmApi lexboxLcmApi)
{
_lexEntryType = lexEntryType;
_lcmEntry = lcmEntry;
_lexboxLcmApi = lexboxLcmApi;
Name = new();
Name = base.Name = new();
}

public override Guid Id
{
get => _lexEntryType.Guid;
set
{
if (_lcmEntry is null)
throw new InvalidOperationException("Cannot update complex form type Id on a null entry");
_lexboxLcmApi.RemoveComplexFormType(_lcmEntry, _lexEntryType.Guid);
_lexboxLcmApi.AddComplexFormType(_lcmEntry, value);
}
}

public override required MultiString Name
{
get => new UpdateMultiStringProxy(_lexEntryType.Name, _lexboxLcmApi);
set
{
}
}
}
36 changes: 25 additions & 11 deletions backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class Sena3SyncTests : IClassFixture<Sena3Fixture>, IAsyncLifetime
private CrdtMiniLcmApi _crdtApi = null!;
private FwDataMiniLcmApi _fwDataApi = null!;
private IDisposable? _cleanup;
private MiniLcmImport _miniLcmImport = null!;


public Sena3SyncTests(Sena3Fixture fixture)
Expand All @@ -29,6 +30,7 @@ public async Task InitializeAsync()
{
(_crdtApi, _fwDataApi, var services, _cleanup) = await _fixture.SetupProjects();
_syncService = services.GetRequiredService<CrdtFwdataProjectSyncService>();
_miniLcmImport = services.GetRequiredService<MiniLcmImport>();
_fwDataApi.EntryCount.Should().BeGreaterThan(100, "project should be loaded and have entries");
}

Expand Down Expand Up @@ -56,9 +58,11 @@ private void ShouldAllBeEquivalentTo(Dictionary<Guid, Entry> crdtEntries, Dictio
}

//by default the first sync is an import, this will skip that so that the sync will actually sync data
private async Task BypassImport()
private async Task BypassImport(bool wsImported = false)
{
await _syncService.SaveProjectSnapshot(_fwDataApi.Project, new ([], [], []));
var snapshot = CrdtFwdataProjectSyncService.ProjectSnapshot.Empty;
if (wsImported) snapshot = snapshot with { WritingSystems = await _fwDataApi.GetWritingSystems() };
await _syncService.SaveProjectSnapshot(_fwDataApi.Project, snapshot);
}

//this lets us query entries when there is no writing system
Expand Down Expand Up @@ -98,28 +102,37 @@ public async Task DryRunSync_MakesNoChanges()
_crdtApi.GetEntries().ToBlockingEnumerable().Should().BeEmpty();
}

[Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")]
public async Task DryRunSync_MakesTheSameChangesAsImport()
[Fact]
public async Task DryRunSync_MakesTheSameChangesAsSync()
{
await BypassImport();
//syncing requires querying entries, which fails if there are no writing systems, so we import those first
await _miniLcmImport.ImportWritingSystems(_crdtApi, _fwDataApi);
await BypassImport(true);

var dryRunSyncResult = await _syncService.SyncDryRun(_crdtApi, _fwDataApi);
var syncResult = await _syncService.Sync(_crdtApi, _fwDataApi);
dryRunSyncResult.Should().BeEquivalentTo(syncResult);
dryRunSyncResult.CrdtChanges.Should().Be(syncResult.CrdtChanges);
//can't test fwdata changes as they will not work correctly since the sync code expects Crdts to contain data from FWData
//this throws off the algorithm and it will try to delete everything in fwdata since there's no data in the crdt since it was a dry run
}

[Fact]
public async Task FirstSena3SyncJustDoesAnSync()
{
_fwDataApi.EntryCount.Should().BeGreaterThan(1000,
"projects with less than 1000 entries don't trip over the default query limit");

var results = await _syncService.Sync(_crdtApi, _fwDataApi);
results.FwdataChanges.Should().Be(0);
results.CrdtChanges.Should().BeGreaterThanOrEqualTo(_fwDataApi.EntryCount);

var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id);
var crdtEntries = await _crdtApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
fwdataEntries.Count.Should().Be(_fwDataApi.EntryCount);
ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries);
}

[Fact(Skip = "this test is waiting for syncing ComplexFormTypes and WritingSystems")]
[Fact]
public async Task SyncWithoutImport_CrdtShouldMatchFwdata()
{
await BypassImport();
Expand All @@ -128,8 +141,9 @@ public async Task SyncWithoutImport_CrdtShouldMatchFwdata()
results.FwdataChanges.Should().Be(0);
results.CrdtChanges.Should().BeGreaterThan(_fwDataApi.EntryCount);

var crdtEntries = await _crdtApi.GetEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetEntries().ToDictionaryAsync(e => e.Id);
var crdtEntries = await _crdtApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
var fwdataEntries = await _fwDataApi.GetAllEntries().ToDictionaryAsync(e => e.Id);
fwdataEntries.Count.Should().Be(_fwDataApi.EntryCount);
ShouldAllBeEquivalentTo(crdtEntries, fwdataEntries);
}

Expand Down
50 changes: 32 additions & 18 deletions backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ await _fixture.FwDataApi.CreateEntry(new Entry()

public async Task DisposeAsync()
{
await foreach (var entry in _fixture.FwDataApi.GetEntries())
await foreach (var entry in _fixture.FwDataApi.GetAllEntries())
{
await _fixture.FwDataApi.DeleteEntry(entry.Id);
}
foreach (var entry in await _fixture.CrdtApi.GetEntries().ToArrayAsync())
foreach (var entry in await _fixture.CrdtApi.GetAllEntries().ToArrayAsync())
{
await _fixture.CrdtApi.DeleteEntry(entry.Id);
}
Expand All @@ -82,8 +82,8 @@ public async Task FirstSyncJustDoesAnImport()
var fwdataApi = _fixture.FwDataApi;
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -144,8 +144,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -180,8 +180,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.SyncDryRun(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Select(e => e.Id).Should().NotContain(fwDataEntryId);
fwdataEntries.Select(e => e.Id).Should().NotContain(crdtEntryId);
}
Expand Down Expand Up @@ -222,8 +222,8 @@ public async Task CreatingAComplexEntryInFwDataSyncsWithoutIssue()
hatstand.Components = [component1, component2];
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -304,8 +304,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand Down Expand Up @@ -383,8 +383,8 @@ await crdtApi.CreateEntry(new Entry()
});
await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand All @@ -406,8 +406,8 @@ public async Task UpdatingAnEntryInEachProjectSyncsAcrossBoth()
results.CrdtChanges.Should().Be(1);
results.FwdataChanges.Should().Be(1);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options
.For(e => e.Components).Exclude(c => c.Id)
Expand Down Expand Up @@ -474,8 +474,8 @@ public async Task AddingASenseToAnEntryInEachProjectSyncsAcrossBoth()

await _syncService.Sync(crdtApi, fwdataApi);

var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var crdtEntries = await crdtApi.GetAllEntries().ToArrayAsync();
var fwdataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtEntries.Should().BeEquivalentTo(fwdataEntries,
options => options.For(e => e.Components).Exclude(c => c.Id)
.For(e => e.ComplexForms).Exclude(c => c.Id));
Expand All @@ -500,4 +500,18 @@ public async Task CanCreateAComplexFormAndItsComponentInOneSync()
//one of the entries will be created first, it will try to create the reference to the other but it won't exist yet
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);
}

[Fact]
public async Task CanCreateAComplexFormTypeAndSyncsIt()
{
//ensure they are synced so a real sync will happen when we want it to
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);

var complexFormEntry = await _fixture.CrdtApi.CreateComplexFormType(new() { Name = new() { { "en", "complexFormType" } } });

//one of the entries will be created first, it will try to create the reference to the other but it won't exist yet
await _fixture.SyncService.Sync(_fixture.CrdtApi, _fixture.FwDataApi);

_fixture.FwDataApi.GetComplexFormTypes().ToBlockingEnumerable().Should().ContainEquivalentOf(complexFormEntry);
}
}
32 changes: 24 additions & 8 deletions backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@ public async Task<SyncResult> Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataA
{
await SaveProjectSnapshot(fwdataApi.Project,
new ProjectSnapshot(
await fwdataApi.GetEntries().ToArrayAsync(),
await fwdataApi.GetAllEntries().ToArrayAsync(),
await fwdataApi.GetPartsOfSpeech().ToArrayAsync(),
await fwdataApi.GetSemanticDomains().ToArrayAsync()));
await fwdataApi.GetSemanticDomains().ToArrayAsync(),
await fwdataApi.GetComplexFormTypes().ToArrayAsync(),
await fwdataApi.GetWritingSystems()));
}
return result;
}
Expand All @@ -62,21 +64,27 @@ private async Task<SyncResult> Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi,
return new SyncResult(entryCount, 0);
}

//todo sync complex form types, writing systems
var currentFwDataWritingSystems = await fwdataApi.GetWritingSystems();
var crdtChanges = await WritingSystemSync.Sync(currentFwDataWritingSystems, projectSnapshot.WritingSystems, crdtApi);
var fwdataChanges = await WritingSystemSync.Sync(await crdtApi.GetWritingSystems(), currentFwDataWritingSystems, fwdataApi);

var currentFwDataPartsOfSpeech = await fwdataApi.GetPartsOfSpeech().ToArrayAsync();
var crdtChanges = await PartOfSpeechSync.Sync(currentFwDataPartsOfSpeech, projectSnapshot.PartsOfSpeech, crdtApi);
var fwdataChanges = await PartOfSpeechSync.Sync(await crdtApi.GetPartsOfSpeech().ToArrayAsync(), currentFwDataPartsOfSpeech, fwdataApi);
crdtChanges += await PartOfSpeechSync.Sync(currentFwDataPartsOfSpeech, projectSnapshot.PartsOfSpeech, crdtApi);
fwdataChanges += await PartOfSpeechSync.Sync(await crdtApi.GetPartsOfSpeech().ToArrayAsync(), currentFwDataPartsOfSpeech, fwdataApi);

var currentFwDataSemanticDomains = await fwdataApi.GetSemanticDomains().ToArrayAsync();
crdtChanges += await SemanticDomainSync.Sync(currentFwDataSemanticDomains, projectSnapshot.SemanticDomains, crdtApi);
fwdataChanges += await SemanticDomainSync.Sync(await crdtApi.GetSemanticDomains().ToArrayAsync(), currentFwDataSemanticDomains, fwdataApi);

var currentFwDataEntries = await fwdataApi.GetEntries().ToArrayAsync();
var currentFwDataComplexFormTypes = await fwdataApi.GetComplexFormTypes().ToArrayAsync();
crdtChanges += await ComplexFormTypeSync.Sync(currentFwDataComplexFormTypes, projectSnapshot.ComplexFormTypes, crdtApi);
fwdataChanges += await ComplexFormTypeSync.Sync(await crdtApi.GetComplexFormTypes().ToArrayAsync(), currentFwDataComplexFormTypes, fwdataApi);

var currentFwDataEntries = await fwdataApi.GetAllEntries().ToArrayAsync();
crdtChanges += await EntrySync.Sync(currentFwDataEntries, projectSnapshot.Entries, crdtApi);
LogDryRun(crdtApi, "crdt");

fwdataChanges += await EntrySync.Sync(await crdtApi.GetEntries().ToArrayAsync(), currentFwDataEntries, fwdataApi);
fwdataChanges += await EntrySync.Sync(await crdtApi.GetAllEntries().ToArrayAsync(), currentFwDataEntries, fwdataApi);
LogDryRun(fwdataApi, "fwdata");

//todo push crdt changes to lexbox
Expand All @@ -100,7 +108,15 @@ private void LogDryRun(IMiniLcmApi api, string type)
return ((DryRunMiniLcmApi)api).DryRunRecords;
}

public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains);
public record ProjectSnapshot(
Entry[] Entries,
PartOfSpeech[] PartsOfSpeech,
SemanticDomain[] SemanticDomains,
ComplexFormType[] ComplexFormTypes,
WritingSystems WritingSystems)
{
internal static ProjectSnapshot Empty { get; } = new([], [], [], [], new WritingSystems());
}

private async Task<ProjectSnapshot?> GetProjectSnapshot(FwDataProject project)
{
Expand Down
Loading