Merge remote-tracking branch 'origin/develop'

.github/workflows/develop-api.yaml

@@ -3,15 +3,27 @@ on:
   workflow_dispatch:
   push:
     paths:
-      - 'backend/**'
+      - 'backend/LexBoxApi/**'
+      - 'backend/LexCore/**'
+      - 'backend/LexData/**'
+      - 'backend/LfClassicData/**'
+      - 'backend/MiniLcm/**'
+      - 'backend/SyncReverserProxy/**'
+      - 'backend/Testing/**'
       - '.github/workflows/lexbox-api.yaml'
       - '.github/workflows/deploy.yaml'
       - 'deployment/lexbox-deployment.yaml'
     branches:
       - develop
   pull_request:
     paths:
-      - 'backend/**'
+      - 'backend/LexBoxApi/**'
+      - 'backend/LexCore/**'
+      - 'backend/LexData/**'
+      - 'backend/LfClassicData/**'
+      - 'backend/MiniLcm/**'
+      - 'backend/SyncReverserProxy/**'
+      - 'backend/Testing/**'
       - '.github/workflows/lexbox-api.yaml'
       - '.github/workflows/deploy.yaml'
       - 'deployment/lexbox-deployment.yaml'

.github/workflows/fw-lite.yaml

@@ -0,0 +1,77 @@
+name: FieldWorks Lite
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'backend/FwDataMiniLcmBridge/**'
+      - 'backend/harmony/**'
+      - 'backend/LcmCrdt/**'
+      - 'backend/LocalWebApp/**'
+      - 'backend/MiniLcm/**'
+      - 'frontend/viewer/**'
+      - '.github/workflows/fw-lite.yaml'
+    branches:
+      - develop
+  pull_request:
+    paths:
+      - 'backend/FwDataMiniLcmBridge/**'
+      - 'backend/harmony/**'
+      - 'backend/LcmCrdt/**'
+      - 'backend/LocalWebApp/**'
+      - 'backend/MiniLcm/**'
+      - 'frontend/viewer/**'
+      - '.github/workflows/fw-lite.yaml'
+    branches:
+      - develop
+
+jobs:
+  build-app:
+    name: Build FW Lite
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.x'
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: './frontend/package.json'
+      - name: Dotnet build
+        working-directory: backend/LocalWebApp
+        run: dotnet build --configuration Release
+      - name: Dotnet test
+        working-directory: backend/LcmCrdt.Tests
+        run: dotnet test --configuration Release --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
+        if: always()
+        with:
+          check_name: FW Lite Unit Tests
+          files: ./backend/LcmCrdt.Tests/test-results/*.xml
+      - name: Build viewer
+        working-directory: frontend/viewer
+        run: |
+          corepack enable
+          pnpm install
+          pnpm run build-app
+
+      - name: Publish linux
+        working-directory: backend/LocalWebApp
+        run: dotnet publish -r linux-x64 -o ./publish/linux
+      - name: Publish osx
+        working-directory: backend/LocalWebApp
+        run: dotnet publish -r osx-x64 -o ./publish/osx
+      - name: Publish win
+        working-directory: backend/LocalWebApp
+        run: dotnet publish -r win-x64 -o ./publish/win
+      - name: Upload artifacts
+        # uploading in one artifact as there's a lot of duplication between builds so compression goes far
+        uses: actions/upload-artifact@v4
+        with:
+          name: fw-lite
+          if-no-files-found: error
+          path: backend/LocalWebApp/publish/*

.github/workflows/integration-test.yaml

@@ -12,10 +12,15 @@ on:
         required: true
         type: string
       hg-version:
-        description: 'The version of hg to test (3 or 6, default 3)'
+        description: 'The version of hg to test (3 or 6, default 6)'
         required: false
-        default: '3'
+        default: '6'
         type: string
+      run-dotnet:
+        description: 'Whether to run dotnet tests'
+        required: false
+        default: true
+        type: boolean
       run-playwright:
         description: 'Whether to run Playwright tests'
         required: false
@@ -32,20 +37,31 @@ on:
         required: true
         type: string
       hg-version:
-        description: 'The version of hg to test (3 or 6, default 3)'
+        description: 'The version of hg to test (3 or 6, default 6)'
         required: false
-        default: '3'
+        default: '6'
         type: string
+      run-dotnet:
+        description: 'Whether to run dotnet tests'
+        required: false
+        default: true
+        type: boolean
       run-playwright:
         description: 'Whether to run Playwright tests'
         required: false
         default: true
         type: boolean
 
+# TODO: Set up caching for /home/runner/.cache/ms-playwright
+# TODO: Ditto for /home/runner/.pnpm-store/v3
+# TODO: Ditto for /home/runner/.nuget/packages
+# TODO: Ditto for /var/cache/apt/archives/
+
 jobs:
-  test:
+  dotnet-test:
+    if: ${{ inputs.run-dotnet }}
     timeout-minutes: 120
-    name: Test ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }} on ${{ inputs.environment }}
+    name: Dotnet tests on ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }} on ${{ inputs.environment }}
     permissions:
       checks: write
     environment:
@@ -68,23 +84,6 @@ jobs:
           sudo apt-get install -f
           rm powershell_7.4.1-1.deb_amd64.deb
           pwsh #validate that powershell installed correctly
-      # First we need to setup Node...
-      - uses: actions/setup-node@v4
-        if: ${{ inputs.run-playwright }}
-        with:
-          node-version-file: 'frontend/package.json'
-      # Then we can set up pnpm...
-      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
-        if: ${{ inputs.run-playwright }}
-        with:
-          package_json_file: 'frontend/package.json'
-      # Then we can have Noede set up package caching
-      - uses: actions/setup-node@v4
-        if: ${{ inputs.run-playwright }}
-        with:
-          node-version-file: 'frontend/package.json'
-          cache: 'pnpm'
-          cache-dependency-path: 'frontend/pnpm-lock.yaml'
       - uses: actions/setup-dotnet@v4
         env:
           DOTNET_INSTALL_DIR: ${{ inputs.runs-on == 'self-hosted' && '/opt/hostedtoolcache/dotnet' || '' }} #poor man's conditional
@@ -104,66 +103,102 @@ jobs:
       - name: Dotnet build
         # limit cpu count for self-hosted due to memory constraints
         run: dotnet build ${{ inputs.runs-on == 'self-hosted' && '-maxcpucount:2' || '' }}
-      # TODO: Set up caching for /home/runner/.cache/ms-playwright so this doesn't take as long
-      # TODO: Ditto for /home/runner/.pnpm-store/v3
-      # TODO: Ditto for /home/runner/.nuget/packages
-      # TODO: Ditto for /var/cache/apt/archives/
-      - name: Playwright setup
-        working-directory: frontend
-        if: ${{ inputs.run-playwright }}
-        run: pnpm install && pnpm pretest
-      - name: Set up Playwright dependencies
-        working-directory: frontend
-        if: ${{ inputs.run-playwright && inputs.runs-on == 'self-hosted' }}
-        run: sudo pnpm exec playwright install-deps
       - name: Build for tests
         run: "dotnet build /p:MercurialVersion=$MERCURIAL_VERSION"
         env:
           MERCURIAL_VERSION: ${{ inputs.hg-version }}
 
-      - name: Integration tests (Playwright)
-        if: ${{ inputs.run-playwright }}
+      - name: Integration tests (.NET)
         env:
           TEST_SERVER_HOSTNAME: ${{ vars.TEST_SERVER_HOSTNAME }}
           #      this is not a typo, we need to use the lf domain because it has a cert that hg will validate
           TEST_STANDARD_HG_HOSTNAME: ${{ vars.TEST_STANDARD_HG_HOSTNAME }}
           TEST_RESUMABLE_HG_HOSTNAME: ${{ vars.TEST_RESUMABLE_HG_HOSTNAME }}
           TEST_PROJECT_CODE: 'sena-3'
           TEST_DEFAULT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-        working-directory: frontend
-        run: pnpm test
+          # 1.5gb max heap size, must be in hex https://learn.microsoft.com/en-us/dotnet/core/runtime-config/garbage-collector#heap-limit
+          # the self hosted runner only has 2gb so this should prevent it from running out of memory
+          DOTNET_GCHeapHardLimit: '0x2CB417800'
 
-      - name: Integration tests (.NET)
+        run: dotnet test --output ./bin --logger trx --results-directory ./test-results --filter Category=Integration
+      - name: Publish unit test results
+        uses: EnricoMi/publish-unit-test-result-action/composite@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
+        if: ${{ always() && !env.act }}
+        with:
+          check_name: Integration Tests ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }}
+          files: ./test-results/*.trx
+      - name: Publish results to testspace
         if: always()
+        run: testspace "[.Net Integration/${{ inputs.runs-on }} HG ${{ inputs.hg-version }}]./test-results/*.trx"
+
+  playwright-test:
+    if: ${{ inputs.run-playwright }}
+    timeout-minutes: 120
+    name: Playwright tests on ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }} on ${{ inputs.environment }}
+    permissions:
+      checks: write
+    environment:
+      name: ${{ inputs.environment }}
+    runs-on: ${{ inputs.runs-on }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: testspace-com/setup-testspace@v1.0.6
+        with:
+          domain: ${{ github.repository_owner }}
+      - name: Setup self-hosted dependencies
+        if: ${{ inputs.runs-on == 'self-hosted' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y p7zip-full iputils-ping python3-venv wget zlib1g-dev libssl-dev #needed by setup python
+          wget -q https://github.com/PowerShell/PowerShell/releases/download/v7.4.1/powershell_7.4.1-1.deb_amd64.deb
+          sudo dpkg -i powershell_7.4.1-1.deb_amd64.deb
+          sudo apt-get install -f
+          rm powershell_7.4.1-1.deb_amd64.deb
+          pwsh #validate that powershell installed correctly
+      # First we need to setup Node...
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: 'frontend/package.json'
+      # Then we can set up pnpm...
+      - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
+        with:
+          package_json_file: 'frontend/package.json'
+      # Then we can have Node set up package caching
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: 'frontend/package.json'
+          cache: 'pnpm'
+          cache-dependency-path: 'frontend/pnpm-lock.yaml'
+      - name: Playwright setup
+        working-directory: frontend
+        run: pnpm install && pnpm pretest
+      - name: Set up Playwright dependencies
+        working-directory: frontend
+        if: ${{ inputs.runs-on == 'self-hosted' }}
+        run: sudo pnpm exec playwright install-deps
+
+      - name: Integration tests (Playwright)
         env:
           TEST_SERVER_HOSTNAME: ${{ vars.TEST_SERVER_HOSTNAME }}
           #      this is not a typo, we need to use the lf domain because it has a cert that hg will validate
           TEST_STANDARD_HG_HOSTNAME: ${{ vars.TEST_STANDARD_HG_HOSTNAME }}
           TEST_RESUMABLE_HG_HOSTNAME: ${{ vars.TEST_RESUMABLE_HG_HOSTNAME }}
           TEST_PROJECT_CODE: 'sena-3'
           TEST_DEFAULT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-          # 1.5gb max heap size, must be in hex https://learn.microsoft.com/en-us/dotnet/core/runtime-config/garbage-collector#heap-limit
-          # the self hosted runner only has 2gb so this should prevent it from running out of memory
-          DOTNET_GCHeapHardLimit: '0x2CB417800'
-
-        run: dotnet test --output ./bin --logger trx --results-directory ./test-results --filter Category=Integration
-
+        working-directory: frontend
+        run: pnpm test
       - name: Password protect Playwright traces
         id: password_protect_test_results
-        if: ${{ always() && inputs.run-playwright }}
+        if: ${{ always() }}
         shell: bash
         env:
           ZIP_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
         run: 7z a ./playwright-traces.7z -mx=0 -mmt=off ./frontend/test-results -p"$ZIP_PASSWORD"
-      - name: Publish unit test results
-        uses: EnricoMi/publish-unit-test-result-action/composite@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
-        if: ${{ always() && !env.act }}
-        with:
-          check_name: Integration Tests ${{ inputs.runs-on }} for Mercurial ${{ inputs.hg-version }}
-          files: ./test-results/*.trx
       - name: Publish results to testspace
         if: always()
-        run: testspace "[.Net Integration]./test-results/*.trx" "[Playwright]./frontend/test-results/*.xml"
+        run: testspace "[Playwright]./frontend/test-results/*.xml"
       - name: Upload playwright results
         if: ${{ always() && steps.password_protect_test_results.outcome == 'success' }}
         uses: actions/upload-artifact@v4

.github/workflows/lexbox-api.yaml

@@ -53,13 +53,14 @@ jobs:
       - name: Dotnet build
         run: dotnet build
       - name: Unit tests
+        working-directory: backend/Testing
         run: dotnet test --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration" --blame-hang-timeout 10m
       - name: Publish unit test results
         uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
         if: always()
         with:
           check_name: C# Unit Tests
-          files: ./test-results/*.xml
+          files: ./backend/Testing/test-results/*.xml
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4

.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 bin/
 obj/
+/data/
 /packages/
 riderModule.iml
 /_ReSharper.Caches/
@@ -17,3 +18,4 @@ dump.sql
 test-results/
 **/*.sqlite
 **/*.sqlite-*
+msal.cache

.gitmodules

@@ -1,4 +1,4 @@
  [submodule "backend/harmony"]
 	path = backend/harmony
 	url = https://github.com/hahn-kev/harmony.git
-	branch = add-crdt
+	branch = chore/performance-pass

.vscode/settings.json

@@ -29,5 +29,6 @@
     },
     "files.associations": {
       "**/frontend/**/i18n/**/*.json": "json5", // Supports json5 syntax: https://svelte-intl-precompile.com/en/docs/getting-started
-    }
+    },
+    "dotnet.defaultSolution": "LexBox.sln"
 }