From 3c622101e8f867b268cc1d4a1a7d7de3ba07274f Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 29 May 2024 21:44:08 +0200 Subject: [PATCH 1/8] Bump Chorus version (#833) --- backend/Testing/Testing.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Testing/Testing.csproj b/backend/Testing/Testing.csproj index d9a754155..4ba343277 100644 --- a/backend/Testing/Testing.csproj +++ b/backend/Testing/Testing.csproj @@ -15,7 +15,7 @@ - + From 20ffa08b5b5e983428a5ae201d2383f9ee512d58 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 30 May 2024 11:33:52 +0700 Subject: [PATCH 2/8] Try to fix reset-project Playwright tests (#789) We believe the failures in the Playwright tests are due to NFS caching the directory entry, and thus not noticing that the directory has new contents, resulting in Mercurial thinking the directory is still empty for some time after the zip file upload has completed. It is possible to force NFS to refresh its cache of a directory by doing opendir() and closedir() on that directory. We will add an `invalidatedircache` command to our hgweb command runner which will run `ls` (which, among other system calls, calls opendir() and closedir()), and call that command after the lexbox pod changes the repo. The other piece of the puzzle appears to be caching on the side of the LexBox API pod's NFS client. According to the `nfs(5)` manpage, NFS writes are cached until one of the following events occurs: * Memory pressure forces reclamation of system memory resources. * An application flushes file data explicitly with sync, msync, or fsync. * An application closes a file with close. * The file is locked/unlocked via fcntl. So doing an `opendir/closedir` on the hgweb pod is not enough; we also need to force the lexbox pod's NFS client to send its data to the NFS server. I've implemented that by doing the equivalent of running `mkdir random-name; rmdir random-name` inside the Mercurial repo after the LexBox API pod touches it (e.g. a project reset or FinishReset operation), followed by a call to `ls` on the hgweb side. --- backend/LexBoxApi/Services/HgService.cs | 69 +++++++++++++++++++ backend/LexBoxApi/Services/ProjectService.cs | 8 +++ .../LexCore/ServiceInterfaces/IHgService.cs | 2 + .../LexCore/Services/HgServiceTests.cs | 11 ++- backend/Testing/Services/Utils.cs | 5 -- .../SendReceiveServiceTests.cs | 12 ---- backend/Testing/Testing.csproj | 1 + frontend/tests/resetProject.test.ts | 48 +++++-------- hgweb/command-runner.sh | 6 +- 9 files changed, 111 insertions(+), 51 deletions(-) diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 7d9ffd5d9..4e314fa48 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -23,6 +23,8 @@ public partial class HgService : IHgService, IHostedService private const string DELETED_REPO_FOLDER = "_____deleted_____"; private const string TEMP_REPO_FOLDER = "_____temp_____"; + private const string AllZeroHash = "0000000000000000000000000000000000000000"; + private readonly IOptions _options; private readonly Lazy _hgClient; private readonly ILogger _logger; @@ -67,6 +69,8 @@ await Task.Run(() => { InitRepoAt(new DirectoryInfo(PrefixRepoFilePath(code))); }); + await InvalidateDirCache(code); + await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } private void InitRepoAt(DirectoryInfo repoDirectory) @@ -104,6 +108,8 @@ public async Task ResetRepo(string code) await SoftDeleteRepo(code, $"{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}__reset"); //we must init the repo as uploading a zip is optional tmpRepo.MoveTo(PrefixRepoFilePath(code)); + await InvalidateDirCache(code); + await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } public async Task FinishReset(string code, Stream zipFile) @@ -137,6 +143,11 @@ await Task.Run(() => // Now we're ready to move the new repo into place, replacing the old one await DeleteRepo(code); tempRepo.MoveTo(PrefixRepoFilePath(code)); + await InvalidateDirCache(code); + // If someone uploaded an *empty* repo, we don't want to wait forever for a non-empty state + var changelogPath = Path.Join(PrefixRepoFilePath(code), ".hg", "store", "00changelog.i"); + var expectedState = File.Exists(changelogPath) ? RepoEmptyState.NonEmpty : RepoEmptyState.Empty; + await WaitForRepoEmptyState(code, expectedState); } /// @@ -262,6 +273,58 @@ public async Task ExecuteHgRecover(string code, CancellationToken t return response; } + public Task InvalidateDirCache(string code, CancellationToken token = default) + { + var repoPath = Path.Join(PrefixRepoFilePath(code)); + if (Directory.Exists(repoPath)) + { + // Invalidate NFS directory cache by forcing a write and re-read of the repo directory + var randomPath = Path.Join(repoPath, Path.GetRandomFileName()); + while (File.Exists(randomPath) || Directory.Exists(randomPath)) { randomPath = Path.Join(repoPath, Path.GetRandomFileName()); } + try + { + // Create and delete a directory since that's slightly safer than a file + var d = Directory.CreateDirectory(randomPath); + d.Delete(); + } + catch (Exception) { } + } + var result = ExecuteHgCommandServerCommand(code, "invalidatedircache", token); + return result; + } + + public async Task GetTipHash(string code, CancellationToken token = default) + { + var content = await ExecuteHgCommandServerCommand(code, "tip", token); + return await content.ReadAsStringAsync(); + } + + private async Task WaitForRepoEmptyState(string code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default) + { + // Set timeout so unforeseen errors can't cause an infinite loop + using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(token); + timeoutSource.CancelAfter(timeoutMs); + var done = false; + try + { + while (!done && !timeoutSource.IsCancellationRequested) + { + var hash = await GetTipHash(code, timeoutSource.Token); + var isEmpty = hash == AllZeroHash; + done = expectedState switch + { + RepoEmptyState.Empty => isEmpty, + RepoEmptyState.NonEmpty => !isEmpty + }; + if (!done) await Task.Delay(2500, timeoutSource.Token); + } + } + // We don't want to actually throw if we hit the timeout, because the operation *will* succeed eventually + // once the NFS caches synchronize, so we don't want to propagate an error message to the end user. So + // even if the timeout is hit, return as if we succeeded. + catch (OperationCanceledException) { } + } + public async Task GetLexEntryCount(string code, ProjectType projectType) { var command = projectType switch @@ -408,3 +471,9 @@ public class BrowseResponse { public BrowseFilesResponse[]? Files { get; set; } } + +public enum RepoEmptyState +{ + Empty, + NonEmpty +} diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index db8a3c0f8..5e351fc38 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -117,6 +117,10 @@ public async Task FinishReset(string code, Stream? zipFile = null) await hgService.FinishReset(code, zipFile); await UpdateProjectMetadata(project); } + else + { + await hgService.InvalidateDirCache(code); + } project.ResetStatus = ResetStatus.None; project.UpdateUpdatedDate(); await dbContext.SaveChangesAsync(); @@ -146,6 +150,10 @@ public async Task UpdateProjectMetadata(Project project) project.FlexProjectMetadata.LexEntryCount = count; } } + else + { + await hgService.InvalidateDirCache(project.Code); + } project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code); // Caller is responsible for caling dbContext.SaveChangesAsync() diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index a5ddadd26..cc4703b7a 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -15,9 +15,11 @@ public interface IHgService Task ResetRepo(string code); Task FinishReset(string code, Stream zipFile); Task VerifyRepo(string code, CancellationToken token); + Task GetTipHash(string code, CancellationToken token = default); Task GetLexEntryCount(string code, ProjectType projectType); Task GetRepositoryIdentifier(Project project); Task ExecuteHgRecover(string code, CancellationToken token); + Task InvalidateDirCache(string code, CancellationToken token = default); bool HasAbandonedTransactions(string projectCode); Task HgCommandHealth(); } diff --git a/backend/Testing/LexCore/Services/HgServiceTests.cs b/backend/Testing/LexCore/Services/HgServiceTests.cs index c1bb0c6fa..502b0a3b1 100644 --- a/backend/Testing/LexCore/Services/HgServiceTests.cs +++ b/backend/Testing/LexCore/Services/HgServiceTests.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; +using Moq.Contrib.HttpClient; using Shouldly; using Testing.Fixtures; @@ -30,8 +31,16 @@ public HgServiceTests() HgResumableUrl = LexboxResumable, SendReceiveDomain = LexboxHgWeb }; + var handler = new Mock(MockBehavior.Strict); + + // This may need to become more sophisticated if our FinishReset tests are changed to include + // a Mercurial repo with actual commits in it, but this is good enough at the moment. + var AllZeroHash = "0000000000000000000000000000000000000000"; + handler.SetupAnyRequest().ReturnsResponse(AllZeroHash); + + var mockFactory = handler.CreateClientFactory(); _hgService = new HgService(new OptionsWrapper(_hgConfig), - Mock.Of(), + mockFactory, NullLogger.Instance); CleanUpTempDir(); } diff --git a/backend/Testing/Services/Utils.cs b/backend/Testing/Services/Utils.cs index 76cf059cd..dd8a6559d 100644 --- a/backend/Testing/Services/Utils.cs +++ b/backend/Testing/Services/Utils.cs @@ -84,11 +84,6 @@ public static async Task WaitForHgRefreshIntervalAsync() await Task.Delay(TestingEnvironmentVariables.HgRefreshInterval); } - public static async Task WaitForLexboxMetadataUpdateAsync() - { - await Task.Delay(3000); - } - private static string GetNewProjectDir(string projectCode, [CallerMemberName] string projectName = "") { diff --git a/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs b/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs index b2b927a85..baa0bd71e 100644 --- a/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs +++ b/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs @@ -81,14 +81,10 @@ public async Task ModifyProjectData(HgProtocol protocol) var projectConfig = _srFixture.InitLocalFlexProjectWithRepo(); await using var project = await RegisterProjectInLexBox(projectConfig, _adminApiTester); - await WaitForHgRefreshIntervalAsync(); - // Push the project to the server var sendReceiveParams = new SendReceiveParams(protocol, projectConfig); _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); - await WaitForLexboxMetadataUpdateAsync(); - // Verify pushed and store last commit var lastCommitDate = await _adminApiTester.GetProjectLastCommit(projectConfig.Code); lastCommitDate.ShouldNotBeNullOrEmpty(); @@ -101,8 +97,6 @@ public async Task ModifyProjectData(HgProtocol protocol) // Push changes _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth, "Modify project data automated test"); - await WaitForLexboxMetadataUpdateAsync(); - // Verify the push updated the last commit date var lastCommitDateAfter = await _adminApiTester.GetProjectLastCommit(projectConfig.Code); lastCommitDateAfter.ShouldBeGreaterThan(lastCommitDate); @@ -117,8 +111,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) var projectConfig = _srFixture.InitLocalFlexProjectWithRepo(protocol, "SR_AfterReset"); await using var project = await RegisterProjectInLexBox(projectConfig, _adminApiTester); - await WaitForHgRefreshIntervalAsync(); - var sendReceiveParams = new SendReceiveParams(protocol, projectConfig); var srResult = _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); @@ -144,8 +136,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) await _adminApiTester.HttpClient.PostAsync($"{_adminApiTester.BaseUrl}/api/project/resetProject/{projectConfig.Code}", null); await _adminApiTester.HttpClient.PostAsync($"{_adminApiTester.BaseUrl}/api/project/finishResetProject/{projectConfig.Code}", null); - await WaitForHgRefreshIntervalAsync(); // TODO 765: Remove this - // Step 2: verify project is now empty, i.e. tip is "0000000..." response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri); jsonResult = await response.Content.ReadFromJsonAsync(); @@ -169,8 +159,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) var srResultStep3 = _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); _output.WriteLine(srResultStep3); - await WaitForHgRefreshIntervalAsync(); // TODO 765: Remove this - // Step 4: verify project tip is same hash as original project tip response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri); jsonResult = await response.Content.ReadFromJsonAsync(); diff --git a/backend/Testing/Testing.csproj b/backend/Testing/Testing.csproj index 4ba343277..9f3bbc134 100644 --- a/backend/Testing/Testing.csproj +++ b/backend/Testing/Testing.csproj @@ -15,6 +15,7 @@ + diff --git a/frontend/tests/resetProject.test.ts b/frontend/tests/resetProject.test.ts index f809179cd..359909e7a 100644 --- a/frontend/tests/resetProject.test.ts +++ b/frontend/tests/resetProject.test.ts @@ -37,20 +37,14 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 2: Get tip hash and file list from hgweb, check some known values - // It can take a while for the server to pick up the new repo - let beforeResetJson: HgWebJson; - await expect(async () => { - const beforeResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - beforeResetJson = await beforeResetResponse.json() as HgWebJson; - expect(beforeResetJson).toHaveProperty('node'); - expect(beforeResetJson.node).not.toEqual(allZeroHash); - expect(beforeResetJson).toHaveProperty('files'); - expect(beforeResetJson.files).toHaveLength(1); - expect(beforeResetJson.files[0]).toHaveProperty('basename'); - expect(beforeResetJson.files[0].basename).toBe('hello.txt'); - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const beforeResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const beforeResetJson = await beforeResetResponse.json() as HgWebJson; + expect(beforeResetJson).toHaveProperty('node'); + expect(beforeResetJson.node).not.toEqual(allZeroHash); + expect(beforeResetJson).toHaveProperty('files'); + expect(beforeResetJson.files).toHaveLength(1); + expect(beforeResetJson.files[0]).toHaveProperty('basename'); + expect(beforeResetJson.files[0].basename).toBe('hello.txt'); // Step 3: reset project, do not upload zip file await projectPage.goto(); @@ -65,16 +59,11 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 4: confirm it's empty now - // It can take a while for the server to pick up the new repo - await expect(async () => { - const afterResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - const afterResetJson = await afterResetResponse.json() as HgWebJson; - expect(afterResetJson.node).toEqual(allZeroHash); - expect(afterResetJson).toHaveProperty('files'); - expect(afterResetJson.files).toHaveLength(0); - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const afterResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const afterResetJson = await afterResetResponse.json() as HgWebJson; + expect(afterResetJson.node).toEqual(allZeroHash); + expect(afterResetJson).toHaveProperty('files'); + expect(afterResetJson.files).toHaveLength(0); // Step 5: reset project again, uploading zip file downloaded from step 1 await projectPage.goto(); @@ -88,12 +77,7 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 6: confirm tip hash and contents are same as before reset - // It can take a while for the server to pick up the new repo - await expect(async () => { - const afterUploadResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - const afterResetJSon = await afterUploadResponse.json() as HgWebJson; - expect(afterResetJSon).toEqual(beforeResetJson); // NOT .toBe(), which would check that they're the same object. - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const afterUploadResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const afterResetJSon = await afterUploadResponse.json() as HgWebJson; + expect(afterResetJSon).toEqual(beforeResetJson); // NOT .toBe(), which would check that they're the same object. }); diff --git a/hgweb/command-runner.sh b/hgweb/command-runner.sh index bc511e19e..d03074d54 100644 --- a/hgweb/command-runner.sh +++ b/hgweb/command-runner.sh @@ -1,7 +1,7 @@ #!/bin/bash # Define the list of allowed commands -allowed_commands=("verify" "tip" "wesaylexentrycount" "lexentrycount" "recover" "healthz") +allowed_commands=("verify" "tip" "wesaylexentrycount" "lexentrycount" "recover" "healthz" "invalidatedircache") # Get the project code and command name from the URL IFS='/' read -ra PATH_SEGMENTS <<< "$PATH_INFO" @@ -44,6 +44,10 @@ echo "" # Run the hg command, simply output to stdout first_char=$(echo $project_code | cut -c1) +# Ensure NFS cache is refreshed in case project repo changed in another pod (e.g., project reset) +ls /var/hg/repos/$first_char/$project_code/.hg >/dev/null 2>/dev/null # Don't need output; this is enough to refresh NFS dir cache +# Sometimes invalidatedircache is called after deleting a project, so the cd would fail. So exit fast in that case. +[ "x$command_name" = "xinvalidatedircache" ] && exit 0 cd /var/hg/repos/$first_char/$project_code case $command_name in From a99490ee420f359221114c4b31894351c81f1764 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 30 May 2024 09:22:17 -0600 Subject: [PATCH 3/8] change task ui:build to be recursive, causing the viewer to be built first --- .idea/.idea.LexBox/.idea/indexLayout.xml | 4 +++- frontend/Taskfile.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.idea/.idea.LexBox/.idea/indexLayout.xml b/.idea/.idea.LexBox/.idea/indexLayout.xml index 24b63bcb7..8e8320cf9 100644 --- a/.idea/.idea.LexBox/.idea/indexLayout.xml +++ b/.idea/.idea.LexBox/.idea/indexLayout.xml @@ -18,6 +18,8 @@ frontend/.svelte-kit/output + frontend/build + frontend/viewer/dist-web-component - + \ No newline at end of file diff --git a/frontend/Taskfile.yml b/frontend/Taskfile.yml index 53a59f524..ece4c16b6 100644 --- a/frontend/Taskfile.yml +++ b/frontend/Taskfile.yml @@ -24,7 +24,7 @@ tasks: aliases: [ b ] deps: [ install ] cmds: - - pnpm run build + - pnpm run -r build check: desc: "Runs the frontend code checks done in CI. Note: the app must be built." aliases: [ sc, svelte-check ] From cf3b2060086862ae8dc2fcbd8cd864fc88f4772d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 29 May 2024 14:29:23 -0600 Subject: [PATCH 4/8] replace string with ProjectCode in HgService to force validation and to avoid requiring validation anywhere in HgService. Also add some tests for ProjectCode validation. closes #823 --- backend/LexBoxApi/Services/HgService.cs | 73 +++++++------------ backend/LexCore/Entities/ProjectCode.cs | 43 +++++++++++ .../LexCore/ServiceInterfaces/IHgService.cs | 30 ++++---- backend/Testing/LexCore/ProjectCodeTests.cs | 35 +++++++++ 4 files changed, 121 insertions(+), 60 deletions(-) create mode 100644 backend/LexCore/Entities/ProjectCode.cs create mode 100644 backend/Testing/LexCore/ProjectCodeTests.cs diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 4e314fa48..0b4288650 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -18,10 +18,10 @@ namespace LexBoxApi.Services; -public partial class HgService : IHgService, IHostedService +public class HgService : IHgService, IHostedService { - private const string DELETED_REPO_FOLDER = "_____deleted_____"; - private const string TEMP_REPO_FOLDER = "_____temp_____"; + private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER; + private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER; private const string AllZeroHash = "0000000000000000000000000000000000000000"; @@ -36,17 +36,12 @@ public HgService(IOptions options, IHttpClientFactory clientFactory, I _hgClient = new(() => clientFactory.CreateClient("HgWeb")); } - [GeneratedRegex(Project.ProjectCodeRegex)] - private static partial Regex ProjectCodeRegex(); + public static string PrefixRepoRequestPath(ProjectCode code) => $"{code.Value[0]}/{code}"; + private string PrefixRepoFilePath(ProjectCode code) => Path.Combine(_options.Value.RepoPath, code.Value[0].ToString(), code.Value); + private string GetTempRepoPath(ProjectCode code, string reason) => Path.Combine(_options.Value.RepoPath, TEMP_REPO_FOLDER, $"{code}__{reason}__{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}"); - public static string PrefixRepoRequestPath(string code) => $"{code[0]}/{code}"; - private string PrefixRepoFilePath(string code) => Path.Combine(_options.Value.RepoPath, code[0].ToString(), code); - private string GetTempRepoPath(string code, string reason) => Path.Combine(_options.Value.RepoPath, TEMP_REPO_FOLDER, $"{code}__{reason}__{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}"); - - private async Task GetResponseMessage(string code, string requestPath) + private async Task GetResponseMessage(ProjectCode code, string requestPath) { - if (!ProjectCodeRegex().IsMatch(code)) - throw new ArgumentException($"Invalid project code: {code}."); var client = _hgClient.Value; var urlPrefix = DetermineProjectUrlPrefix(HgType.hgWeb, _options.Value); @@ -60,9 +55,8 @@ private async Task GetResponseMessage(string code, string r /// Note: The repo is unstable and potentially unavailable for a short while after creation, so don't read from it right away. /// See: https://github.com/sillsdev/languageforge-lexbox/issues/173#issuecomment-1665478630 /// - public async Task InitRepo(string code) + public async Task InitRepo(ProjectCode code) { - AssertIsSafeRepoName(code); if (Directory.Exists(PrefixRepoFilePath(code))) throw new AlreadyExistsException($"Repo already exists: {code}."); await Task.Run(() => @@ -83,12 +77,12 @@ private void InitRepoAt(DirectoryInfo repoDirectory) ); } - public async Task DeleteRepo(string code) + public async Task DeleteRepo(ProjectCode code) { await Task.Run(() => Directory.Delete(PrefixRepoFilePath(code), true)); } - public BackupExecutor? BackupRepo(string code) + public BackupExecutor? BackupRepo(ProjectCode code) { string repoPath = PrefixRepoFilePath(code); if (!Directory.Exists(repoPath)) @@ -101,7 +95,7 @@ public async Task DeleteRepo(string code) }, token)); } - public async Task ResetRepo(string code) + public async Task ResetRepo(ProjectCode code) { var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset")); InitRepoAt(tmpRepo); @@ -112,7 +106,7 @@ public async Task ResetRepo(string code) await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } - public async Task FinishReset(string code, Stream zipFile) + public async Task FinishReset(ProjectCode code, Stream zipFile) { var tempRepoPath = GetTempRepoPath(code, "upload"); var tempRepo = Directory.CreateDirectory(tempRepoPath); @@ -167,7 +161,7 @@ await Task.Run(() => } - public Task RevertRepo(string code, string revHash) + public Task RevertRepo(ProjectCode code, string revHash) { throw new NotImplementedException(); // Steps: @@ -179,7 +173,7 @@ public Task RevertRepo(string code, string revHash) // Will need an SSH key as a k8s secret, put it into authorized_keys on the hgweb side so that lexbox can do "ssh hgweb hg clone ..." } - public async Task SoftDeleteRepo(string code, string deletedRepoSuffix) + public async Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix) { var deletedRepoName = $"{code}__{deletedRepoSuffix}"; await Task.Run(() => @@ -216,12 +210,12 @@ private static void SetPermissionsRecursively(DirectoryInfo rootDir) } } - public bool HasAbandonedTransactions(string projectCode) + public bool HasAbandonedTransactions(ProjectCode projectCode) { return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "journal")); } - public bool RepoIsLocked(string projectCode) + public bool RepoIsLocked(ProjectCode projectCode) { return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "lock")); } @@ -232,7 +226,7 @@ public bool RepoIsLocked(string projectCode) return json?["entries"]?.AsArray().FirstOrDefault()?["node"].Deserialize(); } - public async Task GetLastCommitTimeFromHg(string projectCode) + public async Task GetLastCommitTimeFromHg(ProjectCode projectCode) { var json = await GetCommit(projectCode, "tip"); //format is this: [1678687688, offset] offset is @@ -247,13 +241,13 @@ public bool RepoIsLocked(string projectCode) return date.ToUniversalTime(); } - private async Task GetCommit(string projectCode, string rev) + private async Task GetCommit(ProjectCode projectCode, string rev) { var response = await GetResponseMessage(projectCode, $"log?style=json-lex&rev={rev}"); return await response.Content.ReadFromJsonAsync(); } - public async Task GetChangesets(string projectCode) + public async Task GetChangesets(ProjectCode projectCode) { var response = await GetResponseMessage(projectCode, "log?style=json-lex"); var logResponse = await response.Content.ReadFromJsonAsync(); @@ -261,11 +255,11 @@ public async Task GetChangesets(string projectCode) } - public Task VerifyRepo(string code, CancellationToken token) + public Task VerifyRepo(ProjectCode code, CancellationToken token) { return ExecuteHgCommandServerCommand(code, "verify", token); } - public async Task ExecuteHgRecover(string code, CancellationToken token) + public async Task ExecuteHgRecover(ProjectCode code, CancellationToken token) { var response = await ExecuteHgCommandServerCommand(code, "recover", token); // Can't do this with a streamed response, unfortunately. Will have to do it client-side. @@ -273,7 +267,7 @@ public async Task ExecuteHgRecover(string code, CancellationToken t return response; } - public Task InvalidateDirCache(string code, CancellationToken token = default) + public Task InvalidateDirCache(ProjectCode code, CancellationToken token = default) { var repoPath = Path.Join(PrefixRepoFilePath(code)); if (Directory.Exists(repoPath)) @@ -293,13 +287,13 @@ public Task InvalidateDirCache(string code, CancellationToken token return result; } - public async Task GetTipHash(string code, CancellationToken token = default) + public async Task GetTipHash(ProjectCode code, CancellationToken token = default) { var content = await ExecuteHgCommandServerCommand(code, "tip", token); return await content.ReadAsStringAsync(); } - private async Task WaitForRepoEmptyState(string code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default) + private async Task WaitForRepoEmptyState(ProjectCode code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default) { // Set timeout so unforeseen errors can't cause an infinite loop using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(token); @@ -325,7 +319,7 @@ private async Task WaitForRepoEmptyState(string code, RepoEmptyState expectedSta catch (OperationCanceledException) { } } - public async Task GetLexEntryCount(string code, ProjectType projectType) + public async Task GetLexEntryCount(ProjectCode code, ProjectType projectType) { var command = projectType switch { @@ -346,7 +340,7 @@ public async Task HgCommandHealth() return version.Trim(); } - private async Task ExecuteHgCommandServerCommand(string code, string command, CancellationToken token) + private async Task ExecuteHgCommandServerCommand(ProjectCode code, string command, CancellationToken token) { var httpClient = _hgClient.Value; var baseUri = _options.Value.HgCommandServer; @@ -355,18 +349,7 @@ private async Task ExecuteHgCommandServerCommand(string code, strin return response.Content; } - private static readonly string[] SpecialDirectoryNames = [DELETED_REPO_FOLDER, TEMP_REPO_FOLDER]; - private static readonly HashSet InvalidRepoNames = [.. SpecialDirectoryNames, "api"]; - - private void AssertIsSafeRepoName(string name) - { - if (InvalidRepoNames.Contains(name, StringComparer.OrdinalIgnoreCase)) - throw new ArgumentException($"Invalid repo name: {name}."); - if (!ProjectCodeRegex().IsMatch(name)) - throw new ArgumentException($"Invalid repo name: {name}."); - } - - public async Task DetermineProjectType(string projectCode) + public async Task DetermineProjectType(ProjectCode projectCode) { var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex"); var parsed = await response.Content.ReadFromJsonAsync(); @@ -433,7 +416,7 @@ public static string DetermineProjectUrlPrefix(HgType type, HgConfig hgConfig) public Task StartAsync(CancellationToken cancellationToken) { - var repoContainerDirectories = SpecialDirectoryNames + var repoContainerDirectories = ProjectCode.SpecialDirectoryNames .Concat(Enumerable.Range('a', 'z' - 'a' + 1).Select(c => ((char)c).ToString())) .Concat(Enumerable.Range(0, 10).Select(c => c.ToString())); diff --git a/backend/LexCore/Entities/ProjectCode.cs b/backend/LexCore/Entities/ProjectCode.cs new file mode 100644 index 000000000..25153eeec --- /dev/null +++ b/backend/LexCore/Entities/ProjectCode.cs @@ -0,0 +1,43 @@ +using System.Text.RegularExpressions; + +namespace LexCore.Entities; + +public readonly partial record struct ProjectCode +{ + public ProjectCode() + { + throw new NotSupportedException("Default constructor is not supported."); + } + + public ProjectCode(string value) + { + AssertIsSafeRepoName(value); + Value = value; + } + + public string Value { get; } + public static implicit operator ProjectCode(string code) => new(code); + + public override string ToString() + { + return Value; + } + + public const string DELETED_REPO_FOLDER = "_____deleted_____"; + public const string TEMP_REPO_FOLDER = "_____temp_____"; + public static readonly string[] SpecialDirectoryNames = [DELETED_REPO_FOLDER, TEMP_REPO_FOLDER]; + + private static readonly HashSet InvalidRepoNames = + new([.. SpecialDirectoryNames, "api"], StringComparer.OrdinalIgnoreCase); + + private void AssertIsSafeRepoName(string name) + { + if (InvalidRepoNames.Contains(name)) + throw new ArgumentException($"Invalid repo name: {name}."); + if (!ProjectCodeRegex().IsMatch(name)) + throw new ArgumentException($"Invalid repo name: {name}."); + } + + [GeneratedRegex(Project.ProjectCodeRegex)] + private static partial Regex ProjectCodeRegex(); +} diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index cc4703b7a..60876ea06 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -5,21 +5,21 @@ namespace LexCore.ServiceInterfaces; public record BackupExecutor(Func ExecuteBackup); public interface IHgService { - Task InitRepo(string code); - Task GetLastCommitTimeFromHg(string projectCode); - Task GetChangesets(string projectCode); - Task DetermineProjectType(string projectCode); - Task DeleteRepo(string code); - Task SoftDeleteRepo(string code, string deletedRepoSuffix); - BackupExecutor? BackupRepo(string code); - Task ResetRepo(string code); - Task FinishReset(string code, Stream zipFile); - Task VerifyRepo(string code, CancellationToken token); - Task GetTipHash(string code, CancellationToken token = default); - Task GetLexEntryCount(string code, ProjectType projectType); + Task InitRepo(ProjectCode code); + Task GetLastCommitTimeFromHg(ProjectCode projectCode); + Task GetChangesets(ProjectCode projectCode); + Task DetermineProjectType(ProjectCode projectCode); + Task DeleteRepo(ProjectCode code); + Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix); + BackupExecutor? BackupRepo(ProjectCode code); + Task ResetRepo(ProjectCode code); + Task FinishReset(ProjectCode code, Stream zipFile); + Task VerifyRepo(ProjectCode code, CancellationToken token); + Task GetTipHash(ProjectCode code, CancellationToken token = default); + Task GetLexEntryCount(ProjectCode code, ProjectType projectType); Task GetRepositoryIdentifier(Project project); - Task ExecuteHgRecover(string code, CancellationToken token); - Task InvalidateDirCache(string code, CancellationToken token = default); - bool HasAbandonedTransactions(string projectCode); + Task ExecuteHgRecover(ProjectCode code, CancellationToken token); + Task InvalidateDirCache(ProjectCode code, CancellationToken token = default); + bool HasAbandonedTransactions(ProjectCode projectCode); Task HgCommandHealth(); } diff --git a/backend/Testing/LexCore/ProjectCodeTests.cs b/backend/Testing/LexCore/ProjectCodeTests.cs new file mode 100644 index 000000000..49b387bfa --- /dev/null +++ b/backend/Testing/LexCore/ProjectCodeTests.cs @@ -0,0 +1,35 @@ +using LexCore.Entities; +using Shouldly; + +namespace Testing.LexCore; + +public class ProjectCodeTests +{ + [Theory] + [InlineData("_____deleted_____")] + [InlineData("_____temp_____")] + [InlineData("api")] + [InlineData("../hacker")] + [InlineData("hacker/test")] + [InlineData("/hacker")] + [InlineData(@"hacker\test")] + [InlineData("❌")] + [InlineData("!")] + [InlineData("#")] + [InlineData("-not-start-with-dash")] + public void InvalidCodesThrows(string code) + { + Assert.Throws(() => new ProjectCode(code)); + } + + [Theory] + [InlineData("test-name123")] + [InlineData("123-name")] + [InlineData("test")] + public void ValidCodes(string code) + { + var projectCode = new ProjectCode(code); + projectCode.Value.ShouldBe(code); + projectCode.ToString().ShouldBe(code); + } +} From c568e9e932527715f0cd677a21ef784c6b0f284a Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 30 May 2024 10:43:41 -0600 Subject: [PATCH 5/8] sanitize turnstile token using FormUrlEncodedContent instead of string concatenation --- .../LexBoxApi/Services/TurnstileService.cs | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/backend/LexBoxApi/Services/TurnstileService.cs b/backend/LexBoxApi/Services/TurnstileService.cs index 4293b904c..b15d23e66 100644 --- a/backend/LexBoxApi/Services/TurnstileService.cs +++ b/backend/LexBoxApi/Services/TurnstileService.cs @@ -6,34 +6,26 @@ namespace LexBoxApi.Services; -public class TurnstileService +public class TurnstileService(IHttpClientFactory httpClientFactory, IOptionsSnapshot options) { - private readonly IHttpClientFactory _httpClientFactory; - private readonly IOptionsSnapshot _options; - - public TurnstileService(IHttpClientFactory httpClientFactory, IOptionsSnapshot options) - { - _httpClientFactory = httpClientFactory; - _options = options; - } - public async Task IsTokenValid(string token, string? email = null) { if (email is not null) { - var allowDomain = _options.Value.AllowDomain; + var allowDomain = options.Value.AllowDomain; if (!allowDomain.IsNullOrEmpty() && email.EndsWith($"@{allowDomain}")) { return true; } } - var httpClient = _httpClientFactory.CreateClient("cloudflare"); - var data = new StringContent( - $"secret={_options.Value.TurnstileKey}&response={token}", - Encoding.UTF8, - "application/x-www-form-urlencoded" - ); + var httpClient = httpClientFactory.CreateClient("cloudflare"); + + + var data = new FormUrlEncodedContent(new Dictionary + { + { "secret", options.Value.TurnstileKey }, { "response", token } + }); var response = await httpClient.PostAsync("https://challenges.cloudflare.com/turnstile/v0/siteverify", data); var responseJson = await response.Content.ReadFromJsonAsync(); var success = (responseJson?.RootElement.TryGetProperty("success"u8, out var prop) ?? false) && prop.GetBoolean(); From 72a3c22949eadd2774d5050f4eb7b0fcb970ecae Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 30 May 2024 14:58:22 -0600 Subject: [PATCH 6/8] apply latest harmony changes, including removing CrdtCommit from lexbox and moving it into harmony core --- backend/LcmCrdt/LcmCrdtKernel.cs | 1 + backend/LexBoxApi/Services/CrdtSyncRoutes.cs | 8 +- .../Entities/CommitEntityConfiguration.cs | 42 +---- .../LexBoxDbContextModelSnapshot.cs | 168 +++++++++--------- ...ommitTests.cs => CrdtServerCommitTests.cs} | 22 +-- backend/harmony | 2 +- 6 files changed, 105 insertions(+), 138 deletions(-) rename backend/Testing/LexCore/{CrdtCommitTests.cs => CrdtServerCommitTests.cs} (75%) diff --git a/backend/LcmCrdt/LcmCrdtKernel.cs b/backend/LcmCrdt/LcmCrdtKernel.cs index 631cd529a..c143ce732 100644 --- a/backend/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/LcmCrdt/LcmCrdtKernel.cs @@ -26,6 +26,7 @@ public static class LcmCrdtKernel public static IServiceCollection AddLcmCrdtClient(this IServiceCollection services) { LinqToDBForEFTools.Initialize(); + services.AddMemoryCache(); services.AddCrdtData( ConfigureDbOptions, diff --git a/backend/LexBoxApi/Services/CrdtSyncRoutes.cs b/backend/LexBoxApi/Services/CrdtSyncRoutes.cs index 57f51859d..c3bb92f1c 100644 --- a/backend/LexBoxApi/Services/CrdtSyncRoutes.cs +++ b/backend/LexBoxApi/Services/CrdtSyncRoutes.cs @@ -15,10 +15,10 @@ public static IEndpointConventionBuilder MapSyncApi(this IEndpointRouteBuilder e group.MapGet("/get", async (Guid id, LexBoxDbContext dbContext) => { - return await dbContext.Set().Where(c => c.ProjectId == id).GetSyncState(); + return await dbContext.Set().Where(c => c.ProjectId == id).GetSyncState(); }); group.MapPost("/add", - async (Guid id, CrdtCommit[] commits, LexBoxDbContext dbContext) => + async (Guid id, ServerCommit[] commits, LexBoxDbContext dbContext) => { foreach (var commit in commits) { @@ -31,8 +31,8 @@ public static IEndpointConventionBuilder MapSyncApi(this IEndpointRouteBuilder e group.MapPost("/changes", async (Guid id, SyncState clientHeads, LexBoxDbContext dbContext) => { - var commits = dbContext.Set().Where(c => c.ProjectId == id); - return await commits.GetChanges(clientHeads); + var commits = dbContext.Set().Where(c => c.ProjectId == id); + return await commits.GetChanges(clientHeads); }); return group; diff --git a/backend/LexData/Entities/CommitEntityConfiguration.cs b/backend/LexData/Entities/CommitEntityConfiguration.cs index e0cdb220d..d40b04341 100644 --- a/backend/LexData/Entities/CommitEntityConfiguration.cs +++ b/backend/LexData/Entities/CommitEntityConfiguration.cs @@ -9,9 +9,9 @@ namespace LexData.Entities; -public class CommitEntityConfiguration : IEntityTypeConfiguration +public class CommitEntityConfiguration : IEntityTypeConfiguration { - public void Configure(EntityTypeBuilder builder) + public void Configure(EntityTypeBuilder builder) { builder.ToTable("CrdtCommits"); builder.HasKey(c => c.Id); @@ -30,41 +30,7 @@ public void Configure(EntityTypeBuilder builder) )); } - private static JsonChange Deserialize(string s) => JsonSerializer.Deserialize(s)!; + private static ServerJsonChange Deserialize(string s) => JsonSerializer.Deserialize(s)!; - private static string Serialize(JsonChange c) => JsonSerializer.Serialize(c); -} - -public class CrdtCommit : CommitBase -{ - [JsonConstructor] - protected CrdtCommit(Guid id, string hash, string parentHash, HybridDateTime hybridDateTime) : base(id, - hash, - parentHash, - hybridDateTime) - { - } - - public CrdtCommit(Guid id) : base(id) - { - } - - public CrdtCommit() - { - } - - public Guid ProjectId { get; set; } -} - -public class JsonChange -{ - [JsonPropertyName("$type"), JsonPropertyOrder(1)] - public required string Type { get; set; } - - [JsonExtensionData, JsonPropertyOrder(2)] - public Dictionary? ExtensionData { get; set; } - - public static implicit operator JsonChange(JsonElement e) => - e.Deserialize() ?? - throw new SerializationException("Failed to deserialize JSON change"); + private static string Serialize(ServerJsonChange c) => JsonSerializer.Serialize(c); } diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs index 77c540a3e..0b5f4e5e6 100644 --- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs +++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs @@ -467,6 +467,48 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("qrtz_triggers", "quartz"); }); + modelBuilder.Entity("Crdt.Core.ServerCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.ComplexProperty>("HybridDateTime", "Crdt.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("bigint"); + + b1.Property("DateTime") + .HasColumnType("timestamp with time zone"); + }); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("CrdtCommits", (string)null); + }); + modelBuilder.Entity("LexCore.Entities.DraftProject", b => { b.Property("Id") @@ -981,48 +1023,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("OpenIddictTokens", (string)null); }); - modelBuilder.Entity("LexData.Entities.CrdtCommit", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("ClientId") - .HasColumnType("uuid"); - - b.Property("Hash") - .IsRequired() - .HasColumnType("text"); - - b.Property("Metadata") - .IsRequired() - .HasColumnType("text"); - - b.Property("ParentHash") - .IsRequired() - .HasColumnType("text"); - - b.Property("ProjectId") - .HasColumnType("uuid"); - - b.ComplexProperty>("HybridDateTime", "LexData.Entities.CrdtCommit.HybridDateTime#HybridDateTime", b1 => - { - b1.IsRequired(); - - b1.Property("Counter") - .HasColumnType("bigint"); - - b1.Property("DateTime") - .HasColumnType("timestamp with time zone"); - }); - - b.HasKey("Id"); - - b.HasIndex("ProjectId"); - - b.ToTable("CrdtCommits", (string)null); - }); - modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => { b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") @@ -1078,6 +1078,48 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("JobDetail"); }); + modelBuilder.Entity("Crdt.Core.ServerCommit", b => + { + b.HasOne("LexCore.Entities.FlexProjectMetadata", null) + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("Crdt.Core.ChangeEntity", "ChangeEntities", b1 => + { + b1.Property("ServerCommitId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Change") + .HasColumnType("text"); + + b1.Property("CommitId") + .HasColumnType("uuid"); + + b1.Property("EntityId") + .HasColumnType("uuid"); + + b1.Property("Index") + .HasColumnType("integer"); + + b1.HasKey("ServerCommitId", "Id"); + + b1.ToTable("CrdtCommits"); + + b1.ToJson("ChangeEntities"); + + b1.WithOwner() + .HasForeignKey("ServerCommitId"); + }); + + b.Navigation("ChangeEntities"); + }); + modelBuilder.Entity("LexCore.Entities.DraftProject", b => { b.HasOne("LexCore.Entities.User", "ProjectManager") @@ -1176,48 +1218,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Authorization"); }); - modelBuilder.Entity("LexData.Entities.CrdtCommit", b => - { - b.HasOne("LexCore.Entities.FlexProjectMetadata", null) - .WithMany() - .HasForeignKey("ProjectId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.OwnsMany("Crdt.Core.ChangeEntity", "ChangeEntities", b1 => - { - b1.Property("CrdtCommitId") - .HasColumnType("uuid"); - - b1.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("integer"); - - b1.Property("Change") - .HasColumnType("text"); - - b1.Property("CommitId") - .HasColumnType("uuid"); - - b1.Property("EntityId") - .HasColumnType("uuid"); - - b1.Property("Index") - .HasColumnType("integer"); - - b1.HasKey("CrdtCommitId", "Id"); - - b1.ToTable("CrdtCommits"); - - b1.ToJson("ChangeEntities"); - - b1.WithOwner() - .HasForeignKey("CrdtCommitId"); - }); - - b.Navigation("ChangeEntities"); - }); - modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => { b.Navigation("Triggers"); diff --git a/backend/Testing/LexCore/CrdtCommitTests.cs b/backend/Testing/LexCore/CrdtServerCommitTests.cs similarity index 75% rename from backend/Testing/LexCore/CrdtCommitTests.cs rename to backend/Testing/LexCore/CrdtServerCommitTests.cs index 225d59f95..d15269ccd 100644 --- a/backend/Testing/LexCore/CrdtCommitTests.cs +++ b/backend/Testing/LexCore/CrdtServerCommitTests.cs @@ -10,29 +10,29 @@ namespace Testing.LexCore; [Collection(nameof(TestingServicesFixture))] -public class CrdtCommitTests +public class CrdtServerCommitTests { private readonly LexBoxDbContext _dbContext; - public CrdtCommitTests(TestingServicesFixture testing) + public CrdtServerCommitTests(TestingServicesFixture testing) { var serviceProvider = testing.ConfigureServices(); _dbContext = serviceProvider.GetRequiredService(); } [Fact] - public async Task CanSaveCrdtCommit() + public async Task CanSaveServerCommit() { var projectId = await _dbContext.Projects.Select(p => p.Id).FirstAsync(); var commitId = Guid.NewGuid(); - _dbContext.Add(new CrdtCommit(commitId) + _dbContext.Add(new ServerCommit(commitId) { ClientId = Guid.NewGuid(), - HybridDateTime = HybridDateTime.ForTestingNow, + HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0), ProjectId = projectId, ChangeEntities = [ - new ChangeEntity() + new ChangeEntity() { Index = 0, CommitId = commitId, @@ -51,14 +51,14 @@ public async Task CanRoundTripCommitChanges() var projectId = await _dbContext.Projects.Select(p => p.Id).FirstAsync(); var commitId = Guid.NewGuid(); var changeJson = """{"$type":"test","name":"Joe"}"""; - var expectedCommit = new CrdtCommit(commitId) + var expectedCommit = new ServerCommit(commitId) { ClientId = Guid.NewGuid(), - HybridDateTime = HybridDateTime.ForTestingNow, + HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0), ProjectId = projectId, ChangeEntities = [ - new ChangeEntity() + new ChangeEntity() { Index = 0, CommitId = commitId, @@ -70,7 +70,7 @@ public async Task CanRoundTripCommitChanges() _dbContext.Add(expectedCommit); await _dbContext.SaveChangesAsync(); - var actualCommit = await _dbContext.Set().AsNoTracking().FirstAsync(c => c.Id == commitId); + var actualCommit = await _dbContext.Set().AsNoTracking().FirstAsync(c => c.Id == commitId); actualCommit.ShouldNotBeSameAs(expectedCommit); JsonSerializer.Serialize(actualCommit.ChangeEntities[0].Change).ShouldBe(changeJson); } @@ -79,7 +79,7 @@ public async Task CanRoundTripCommitChanges() public void TypePropertyShouldAlwaysBeFirst() { var changeJson = """{"name":"Joe","$type":"test"}"""; - var jsonChange = JsonSerializer.Deserialize(changeJson); + var jsonChange = JsonSerializer.Deserialize(changeJson); JsonSerializer.Serialize(jsonChange).ShouldBe("""{"$type":"test","name":"Joe"}"""); } } diff --git a/backend/harmony b/backend/harmony index ce8badb85..26d825dd8 160000 --- a/backend/harmony +++ b/backend/harmony @@ -1 +1 @@ -Subproject commit ce8badb853756e350c91994e4d967bc46fee8a80 +Subproject commit 26d825dd8509a4a0793f71cc17ff367327599363 From 0e8c326966d30c0b973fe77acce833d06cf36558 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 31 May 2024 14:18:20 +0700 Subject: [PATCH 7/8] Remove deleted draft project from GraphQL cache (#840) --- frontend/src/lib/gql/gql-client.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/gql/gql-client.ts b/frontend/src/lib/gql/gql-client.ts index 124743f60..3ba9ee085 100644 --- a/frontend/src/lib/gql/gql-client.ts +++ b/frontend/src/lib/gql/gql-client.ts @@ -26,6 +26,7 @@ import { LexGqlError, type SoftDeleteProjectMutationVariables, type BulkAddProjectMembersMutationVariables, + type DeleteDraftProjectMutationVariables, } from './types'; import type {Readable, Unsubscriber} from 'svelte/store'; import {derived} from 'svelte/store'; @@ -53,7 +54,9 @@ function createGqlClient(_gqlEndpoint?: string): Client { Mutation: { softDeleteProject: (result, args: SoftDeleteProjectMutationVariables, cache, _info) => { cache.invalidate({__typename: 'Project', id: args.input.projectId}); - cache.invalidate({__typename: 'DraftProject', id: args.input.projectId}); + }, + deleteDraftProject: (result, args: DeleteDraftProjectMutationVariables, cache, _info) => { + cache.invalidate({__typename: 'DraftProject', id: args.input.draftProjectId}); }, deleteUserByAdminOrSelf: (result, args: DeleteUserByAdminOrSelfMutationVariables, cache, _info) => { cache.invalidate({__typename: 'User', id: args.input.userId}); From 62d5a7bc700095b508b3639adc7c1de554dc18fc Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 29 May 2024 09:03:05 -0600 Subject: [PATCH 8/8] refactor move Lf classic routes into their own file, pass all query parameters to backend from viewer requests --- backend/LfClassicData/DataServiceKernel.cs | 43 ---------- backend/LfClassicData/LfClassicRoutes.cs | 81 +++++++++++++++++++ .../viewer/lfClassicLexboxApi.ts | 33 ++++++-- 3 files changed, 109 insertions(+), 48 deletions(-) create mode 100644 backend/LfClassicData/LfClassicRoutes.cs diff --git a/backend/LfClassicData/DataServiceKernel.cs b/backend/LfClassicData/DataServiceKernel.cs index 3d67e729e..196e87253 100644 --- a/backend/LfClassicData/DataServiceKernel.cs +++ b/backend/LfClassicData/DataServiceKernel.cs @@ -1,9 +1,4 @@ using LfClassicData.Configuration; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Routing; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -48,42 +43,4 @@ public static MongoClientSettings BuildMongoClientSettings(IServiceProvider prov cb.Subscribe(new DiagnosticsActivityEventSubscriber(new() { CaptureCommandText = true })); return mongoSettings; } - - public static IEndpointConventionBuilder MapLfClassicApi(this IEndpointRouteBuilder builder) - { - var group = builder.MapGroup("/api/lfclassic/{projectCode}"); - group.MapGet("/writingSystems", - (string projectCode, [FromServices] ILexboxApiProvider provider) => - { - var api = provider.GetProjectApi(projectCode); - return api.GetWritingSystems(); - }); - group.MapGet("/entries", - (string projectCode, - [FromServices] ILexboxApiProvider provider, - int count = 1000, - int offset = 0 - ) => - { - var api = provider.GetProjectApi(projectCode); - return api.GetEntries(new QueryOptions(SortOptions.Default, null, count, offset)); - }); - group.MapGet("/entries/{search}", - (string projectCode, - [FromServices] ILexboxApiProvider provider, - string search, - int count = 1000, - int offset = 0) => - { - var api = provider.GetProjectApi(projectCode); - return api.SearchEntries(search, new QueryOptions(SortOptions.Default, null, count, offset)); - }); - group.MapGet("/entry/{id:Guid}", - (string projectCode, Guid id, [FromServices] ILexboxApiProvider provider) => - { - var api = provider.GetProjectApi(projectCode); - return api.GetEntry(id); - }); - return group; - } } diff --git a/backend/LfClassicData/LfClassicRoutes.cs b/backend/LfClassicData/LfClassicRoutes.cs new file mode 100644 index 000000000..c52ee01fc --- /dev/null +++ b/backend/LfClassicData/LfClassicRoutes.cs @@ -0,0 +1,81 @@ +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Routing; +using MiniLcm; + +namespace LfClassicData; + +public static class LfClassicRoutes +{ + public static IEndpointConventionBuilder MapLfClassicApi(this IEndpointRouteBuilder builder) + { + var group = builder.MapGroup("/api/lfclassic/{projectCode}"); + group.MapGet("/writingSystems", + ([FromRoute] string projectCode, [FromServices] ILexboxApiProvider provider) => + { + var api = provider.GetProjectApi(projectCode); + return api.GetWritingSystems(); + }); + group.MapGet("/entries", + ([FromRoute] string projectCode, + [FromServices] ILexboxApiProvider provider, + [AsParameters] ClassicQueryOptions options + ) => + { + var api = provider.GetProjectApi(projectCode); + return api.GetEntries(options.ToQueryOptions()); + }); + group.MapGet("/entries/{search}", + ([FromRoute] string projectCode, + [FromServices] ILexboxApiProvider provider, + [FromRoute] string search, + [AsParameters] ClassicQueryOptions options) => + { + var api = provider.GetProjectApi(projectCode); + return api.SearchEntries(search, options.ToQueryOptions()); + }); + group.MapGet("/entry/{id:Guid}", + ([FromRoute] string projectCode, Guid id, [FromServices] ILexboxApiProvider provider) => + { + var api = provider.GetProjectApi(projectCode); + return api.GetEntry(id); + }); + return group; + } + + private class ClassicQueryOptions + { + public QueryOptions ToQueryOptions() + { + ExemplarOptions? exemplarOptions = string.IsNullOrEmpty(ExemplarValue) || ExemplarWritingSystem is null + ? null + : new (ExemplarValue, ExemplarWritingSystem.Value); + var sortField = Enum.TryParse(SortField, true, out var field) ? field : SortOptions.Default.Field; + return new QueryOptions(new SortOptions(sortField, + SortWritingSystem ?? SortOptions.Default.WritingSystem, + Ascending ?? SortOptions.Default.Ascending), + exemplarOptions, + Count ?? QueryOptions.Default.Count, + Offset ?? QueryOptions.Default.Offset); + } + + public string? SortField { get; set; } + + public WritingSystemId? SortWritingSystem { get; set; } + + [FromQuery] + public bool? Ascending { get; set; } + + [FromQuery] + public string? ExemplarValue { get; set; } + public WritingSystemId? ExemplarWritingSystem { get; set; } + + [FromQuery] + public int? Count { get; set; } + + [FromQuery] + public int? Offset { get; set; } + } +} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/viewer/lfClassicLexboxApi.ts b/frontend/src/routes/(authenticated)/project/[project_code]/viewer/lfClassicLexboxApi.ts index f1f911e23..64dc0c4ca 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/viewer/lfClassicLexboxApi.ts +++ b/frontend/src/routes/(authenticated)/project/[project_code]/viewer/lfClassicLexboxApi.ts @@ -19,19 +19,42 @@ export class LfClassicLexboxApi implements LexboxApi { } async GetEntries(_options: QueryOptions | undefined): Promise { - const result = await fetch(`/api/lfclassic/${this.projectCode}/entries?order=desc&count=100`); + //todo pass query options into query + const result = await fetch(`/api/lfclassic/${this.projectCode}/entries${this.toQueryParams(_options)}`); return (await result.json()) as IEntry[]; } - CreateWritingSystem(_type: WritingSystemType, _writingSystem: WritingSystem): Promise { - throw new Error('Method not implemented.'); + async SearchEntries(_query: string, _options: QueryOptions | undefined): Promise { + //todo pass query options into query + const result = await fetch(`/api/lfclassic/${this.projectCode}/entries/${encodeURIComponent(_query)}${this.toQueryParams(_options)}`); + return (await result.json()) as IEntry[]; } - UpdateWritingSystem(_wsId: string, _type: WritingSystemType, _update: JsonPatch): Promise { + private toQueryParams(options: QueryOptions | undefined): string { + + if (!options) return ''; + /* eslint-disable @typescript-eslint/no-unsafe-assignment */ + const asc = options.order.ascending ?? true; + const params = new URLSearchParams({ + SortField: options.order.field, + SortWritingSystem: options.order.writingSystem, + Ascending: asc ? 'true' : 'false', + Count: options.count.toString(), + Offset: options.offset.toString() + }); + if (options.exemplar) { + params.set('ExemplarValue', options.exemplar.value); + params.set('ExemplarWritingSystem', options.exemplar.writingSystem); + } + /* eslint-enable @typescript-eslint/no-unsafe-assignment */ + return '?' + params.toString(); + } + + CreateWritingSystem(_type: WritingSystemType, _writingSystem: WritingSystem): Promise { throw new Error('Method not implemented.'); } - SearchEntries(_query: string, _options: QueryOptions | undefined): Promise { + UpdateWritingSystem(_wsId: string, _type: WritingSystemType, _update: JsonPatch): Promise { throw new Error('Method not implemented.'); }