Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extract language tags from FLEx projects and store in DB metadata #952

Merged
merged 26 commits into from
Jul 23, 2024
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
ac43b18
Initial work extracting language tags
rmunn Jul 11, 2024
bda54e1
Load all four ws lists from .langproj file
rmunn Jul 12, 2024
2c9bbe8
Remove language-guessing logic for now
rmunn Jul 12, 2024
ea91cfd
Add new DB columns for FLEx writing systems
rmunn Jul 12, 2024
6534353
Attempt to configure EF for WritingSystems
rmunn Jul 12, 2024
2fab0b2
Better return type for ws-parsing method
rmunn Jul 12, 2024
49b954f
add debug logging to testing services
hahn-kev Jul 12, 2024
eb90a00
refactor xml parsing into HgService, add UpdateProjectLangTags to Pro…
hahn-kev Jul 12, 2024
8306ea0
create migration to add writing systems to flex project metadata
hahn-kev Jul 12, 2024
dd2ffa7
write test for writing language tags into flex project metadata
hahn-kev Jul 12, 2024
c8e7f4d
Add IsDefault field to ws tags in project data
rmunn Jul 15, 2024
ecbc444
Add command to extract LangProject GUID as well
rmunn Jul 15, 2024
90d62a0
Add LangProjectId column to FlexProjectMetadata
rmunn Jul 15, 2024
54a140a
Explain grep regex in a comment
rmunn Jul 15, 2024
cb663b8
Fix typo in language tag parsing
rmunn Jul 15, 2024
6744abb
Make writing system ordering match what FLEx does
rmunn Jul 15, 2024
ef228fd
Add VERY basic UI for writing systems on proj page
rmunn Jul 15, 2024
f8008cc
Translate label for writing system list
rmunn Jul 15, 2024
7886e8f
Improve UI for writing system list
rmunn Jul 15, 2024
0f3b465
Add button for admins to refresh language list
rmunn Jul 15, 2024
d272e5e
Improve types for WritingSystemList component
rmunn Jul 15, 2024
b356f76
add project controller actions to update missing langprojectid and wr…
hahn-kev Jul 16, 2024
27699a0
Use helper method to get lang tags from XML
rmunn Jul 18, 2024
3cbc6fd
Remove migration that does nothing
rmunn Jul 18, 2024
47117b4
Show loading indicator at correct time
rmunn Jul 18, 2024
388e474
Fix failing unit tests
rmunn Jul 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion backend/LexBoxApi/Services/HgService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,42 @@ await Task.Run(() =>
});
}

/// <summary>
/// Returns either an empty string, or XML (in string form) with a root LangTags element containing five child elements: AnalysisWss, CurAnalysisWss, VernWss, CurVernWss, and CurPronunWss.
/// Each child element will contain a single `<Uni>` element whose text content is a list of tags separated by spaces.
/// </summary>
private async Task<string> GetLangTagsAsXml(ProjectCode code, CancellationToken token = default)
{
var result = await ExecuteHgCommandServerCommand(code, "flexwritingsystems", token);
var xmlBody = await result.ReadAsStringAsync(token);
if (string.IsNullOrEmpty(xmlBody)) return string.Empty;
return $"<LangTags>{xmlBody}</LangTags>";
}

public async Task<ProjectWritingSystems?> GetProjectWritingSystems(ProjectCode code, CancellationToken token = default)
{
var langTagsXml = await GetLangTagsAsXml(code, token);
if (string.IsNullOrEmpty(langTagsXml)) return null;
var doc = new System.Xml.XmlDocument();
doc.LoadXml(langTagsXml);
var root = doc.DocumentElement;
if (root is null) return null;
var vernWssStr = root["VernVss"]?["Uni"]?.InnerText ?? "";
var analysisWssStr = root["AnalysisWss"]?["Uni"]?.InnerText ?? "";
var curVernWssStr = root["CurVernVss"]?["Uni"]?.InnerText ?? "";
var curAnalysisWssStr = root["CurAnalysisWss"]?["Uni"]?.InnerText ?? "";
var vernWss = vernWssStr.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
var analysisWss = analysisWssStr.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
var curVernWss = curVernWssStr.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).ToHashSet();
var curAnalysisWss = curAnalysisWssStr.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).ToHashSet();
var vernWsIds = vernWss.Select(tag => new FLExWsId { Tag = tag, IsActive = curVernWss.Contains(tag) }).ToList();
var analysisWsIds = analysisWss.Select(tag => new FLExWsId { Tag = tag, IsActive = curAnalysisWss.Contains(tag) }).ToList();
return new ProjectWritingSystems
{
VernacularWss = vernWsIds,
AnalysisWss = analysisWsIds
};
}

public Task RevertRepo(ProjectCode code, string revHash)
{
Expand Down Expand Up @@ -254,11 +290,11 @@ public async Task<Changeset[]> GetChangesets(ProjectCode projectCode)
return logResponse?.Changesets ?? Array.Empty<Changeset>();
}


public Task<HttpContent> VerifyRepo(ProjectCode code, CancellationToken token)
{
return ExecuteHgCommandServerCommand(code, "verify", token);
}

public async Task<HttpContent> ExecuteHgRecover(ProjectCode code, CancellationToken token)
{
var response = await ExecuteHgCommandServerCommand(code, "recover", token);
Expand Down
12 changes: 12 additions & 0 deletions backend/LexBoxApi/Services/ProjectService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,18 @@ public async Task<Guid> CreateProject(CreateProjectInput input)
return projectId;
}

public async Task UpdateProjectLangTags(Guid projectId)
{
var project = await dbContext.Projects.FindAsync(projectId);
if (project is null || project.Type != ProjectType.FLEx) return;
await dbContext.Entry(project).Reference(p => p.FlexProjectMetadata).LoadAsync();
var langTags = await hgService.GetProjectWritingSystems(project.Code);
if (langTags is null) return;
project.FlexProjectMetadata ??= new FlexProjectMetadata();
project.FlexProjectMetadata.WritingSystems = langTags;
await dbContext.SaveChangesAsync();
}

public async Task<Guid> CreateDraftProject(CreateProjectInput input)
{
// No need for a transaction if we're just saving a single item
Expand Down
13 changes: 13 additions & 0 deletions backend/LexCore/Entities/FlexProjectMetadata.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,17 @@ public class FlexProjectMetadata
{
public Guid ProjectId { get; set; }
public int? LexEntryCount { get; set; }
public ProjectWritingSystems? WritingSystems { get; set; }
}

public class ProjectWritingSystems
{
public required List<FLExWsId> VernacularWss { get; set; } = [];
public required List<FLExWsId> AnalysisWss { get; set; } = [];
}

public class FLExWsId
{
public required string Tag { get; set; }
public bool IsActive { get; set; }
}
1 change: 1 addition & 0 deletions backend/LexCore/ServiceInterfaces/IHgService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ public interface IHgService
Task<ProjectType> DetermineProjectType(ProjectCode projectCode);
Task DeleteRepo(ProjectCode code);
Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix);
Task<ProjectWritingSystems?> GetProjectWritingSystems(ProjectCode code, CancellationToken token = default);
BackupExecutor? BackupRepo(ProjectCode code);
Task ResetRepo(ProjectCode code);
Task FinishReset(ProjectCode code, Stream zipFile);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,11 @@ public class FlexProjectMetadataEntityConfiguration: IEntityTypeConfiguration<Fl
public virtual void Configure(EntityTypeBuilder<FlexProjectMetadata> builder)
{
builder.HasKey(e => e.ProjectId);
builder.OwnsOne(e => e.WritingSystems, wsb =>
{
wsb.ToJson();
wsb.OwnsMany(e => e.AnalysisWss);
wsb.OwnsMany(e => e.VernacularWss);
});
}
}
Loading
Loading