[MergeDups] Separate identical vs similar duplicate finding with intermediate dialog

Backend.Tests/Controllers/MergeControllerTests.cs

@@ -148,5 +148,21 @@ public void TestGetGraylistEntriesNoPermission()
             var result = _mergeController.GetGraylistEntries("projId", 3, "userId").Result;
             Assert.That(result, Is.InstanceOf<ForbidResult>());
         }
+
+        [Test]
+        public void TestFindIdenticalPotentialDuplicatesNoPermission()
+        {
+            _mergeController.ControllerContext.HttpContext = PermissionServiceMock.UnauthorizedHttpContext();
+            var result = _mergeController.FindIdenticalPotentialDuplicates("projId", 2, 1, false).Result;
+            Assert.That(result, Is.InstanceOf<ForbidResult>());
+        }
+
+        [Test]
+        public void TestFindIdenticalPotentialDuplicates()
+        {
+            // This test verifies the endpoint returns OK and data
+            var result = _mergeController.FindIdenticalPotentialDuplicates(ProjId, 5, 10, false).Result;
+            Assert.That(result, Is.InstanceOf<OkObjectResult>());
+        }
     }
 }

Backend/Controllers/MergeController.cs

@@ -121,6 +121,35 @@ public async Task<IActionResult> GraylistAdd(string projectId, [FromBody, BindRe
             return Ok(graylistEntry.WordIds);
         }
 
+        /// <summary> Find and return lists of potential duplicates with identical vernacular. </summary>
+        /// <param name="projectId"> Id of project in which to search the frontier for potential duplicates. </param>
+        /// <param name="maxInList"> Max number of words allowed within a list of potential duplicates. </param>
+        /// <param name="maxLists"> Max number of lists of potential duplicates. </param>
+        /// <param name="ignoreProtected"> Whether to require each set to have at least one unprotected word. </param>
+        /// <returns> List of Lists of <see cref="Word"/>s, each sublist a set of potential duplicates. </returns>
+        [HttpGet("findidenticaldups/{maxInList:int}/{maxLists:int}/{ignoreProtected:bool}", Name = "FindIdenticalPotentialDuplicates")]
+        [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(List<List<Word>>))]
+        [ProducesResponseType(StatusCodes.Status403Forbidden)]
+        public async Task<IActionResult> FindIdenticalPotentialDuplicates(
+            string projectId, int maxInList, int maxLists, bool ignoreProtected)
+        {
+            using var activity = OtelService.StartActivityWithTag(otelTagName, "finding identical potential duplicates");
+
+            if (!await _permissionService.HasProjectPermission(
+                HttpContext, Permission.MergeAndReviewEntries, projectId))
+            {
+                return Forbid();
+            }
+
+            await _mergeService.UpdateMergeBlacklist(projectId);
+
+            var userId = _permissionService.GetUserId(HttpContext);
+            var dups = await _mergeService.GetPotentialDuplicates(
+                projectId, maxInList, maxLists, identicalVernacular: true, userId, ignoreProtected);
+
+            return Ok(dups);
+        }
+
         /// <summary> Start finding lists of potential duplicates for merging. </summary>
         /// <param name="projectId"> Id of project in which to search the frontier for potential duplicates. </param>
         /// <param name="maxInList"> Max number of words allowed within a list of potential duplicates. </param>

Backend/Interfaces/IMergeService.cs

@@ -17,6 +17,9 @@ public interface IMergeService
         Task<int> UpdateMergeGraylist(string projectId);
         Task<bool> GetAndStorePotentialDuplicates(
             string projectId, int maxInList, int maxLists, string userId, bool ignoreProtected = false);
+        Task<List<List<Word>>> GetPotentialDuplicates(
+            string projectId, int maxInList, int maxLists, bool identicalVernacular,
+            string? userId = null, bool ignoreProtected = false);
         List<List<Word>>? RetrieveDups(string userId);
         Task<bool> HasGraylistEntries(string projectId, string? userId = null);
         Task<List<List<Word>>> GetGraylistEntries(string projectId, int maxLists, string? userId = null);

Backend/Services/MergeService.cs

@@ -413,16 +413,16 @@ public async Task<bool> GetAndStorePotentialDuplicates(
             {
                 return false;
             }
-            var dups = await GetPotentialDuplicates(projectId, maxInList, maxLists, userId, ignoreProtected);
+            var dups = await GetPotentialDuplicates(projectId, maxInList, maxLists, false, userId, ignoreProtected);
             // Store the potential duplicates for user to retrieve later.
             return StoreDups(userId, counter, dups) == counter;
         }
 
         /// <summary>
         /// Get Lists of potential duplicate <see cref="Word"/>s in specified <see cref="Project"/>'s frontier.
         /// </summary>
-        private async Task<List<List<Word>>> GetPotentialDuplicates(
-            string projectId, int maxInList, int maxLists, string? userId = null, bool ignoreProtected = false)
+        public async Task<List<List<Word>>> GetPotentialDuplicates(string projectId, int maxInList, int maxLists,
+            bool identicalVernacular, string? userId = null, bool ignoreProtected = false)
         {
             var dupFinder = new DuplicateFinder(maxInList, maxLists, 2);
 
@@ -431,17 +431,9 @@ async Task<bool> isUnavailableSet(List<string> wordIds) =>
                 (await IsInMergeBlacklist(projectId, wordIds, userId)) ||
                 (await IsInMergeGraylist(projectId, wordIds, userId));
 
-            // First pass, only look for words with identical vernacular.
-            var wordLists = await dupFinder.GetIdenticalVernWords(collection, isUnavailableSet, ignoreProtected);
-
-            // If no such sets found, look for similar words.
-            if (wordLists.Count == 0)
-            {
-                collection = await _wordRepo.GetFrontier(projectId);
-                wordLists = await dupFinder.GetSimilarWords(collection, isUnavailableSet, ignoreProtected);
-            }
-
-            return wordLists;
+            return identicalVernacular
+                ? await dupFinder.GetIdenticalVernWords(collection, isUnavailableSet, ignoreProtected)
+                : await dupFinder.GetSimilarWords(collection, isUnavailableSet, ignoreProtected);
         }
 
         public sealed class InvalidMergeWordSetException : Exception

public/locales/en/translation.json

@@ -516,6 +516,15 @@
       "yes": "Yes",
       "no": "No"
     },
+    "identicalCompleted": {
+      "title": "All Identical Duplicates Processed",
+      "congratulations": "Congratulations! You have processed all sets of words with identical vernacular forms.",
+      "hasDeferred": "You have deferred duplicate sets that can be reviewed later.",
+      "findingSimilar": "The Combine will now search for potential duplicates with similar (non-identical) vernacular forms.",
+      "warning": "Finding similar duplicates may take several minutes.",
+      "reviewDeferred": "Review Deferred",
+      "continue": "Continue"
+    },
     "undo": {
       "undo": "Undo Merge",
       "undoDialog": "Undo this merge?",

src/api/api/merge-api.ts

@@ -107,6 +107,84 @@ export const MergeApiAxiosParamCreator = function (
         options: localVarRequestOptions,
       };
     },
+    /**
+     *
+     * @param {string} projectId
+     * @param {number} maxInList
+     * @param {number} maxLists
+     * @param {boolean} ignoreProtected
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     */
+    findIdenticalPotentialDuplicates: async (
+      projectId: string,
+      maxInList: number,
+      maxLists: number,
+      ignoreProtected: boolean,
+      options: any = {}
+    ): Promise<RequestArgs> => {
+      // verify required parameter 'projectId' is not null or undefined
+      assertParamExists(
+        "findIdenticalPotentialDuplicates",
+        "projectId",
+        projectId
+      );
+      // verify required parameter 'maxInList' is not null or undefined
+      assertParamExists(
+        "findIdenticalPotentialDuplicates",
+        "maxInList",
+        maxInList
+      );
+      // verify required parameter 'maxLists' is not null or undefined
+      assertParamExists(
+        "findIdenticalPotentialDuplicates",
+        "maxLists",
+        maxLists
+      );
+      // verify required parameter 'ignoreProtected' is not null or undefined
+      assertParamExists(
+        "findIdenticalPotentialDuplicates",
+        "ignoreProtected",
+        ignoreProtected
+      );
+      const localVarPath =
+        `/v1/projects/{projectId}/merge/findidenticaldups/{maxInList}/{maxLists}/{ignoreProtected}`
+          .replace(`{${"projectId"}}`, encodeURIComponent(String(projectId)))
+          .replace(`{${"maxInList"}}`, encodeURIComponent(String(maxInList)))
+          .replace(`{${"maxLists"}}`, encodeURIComponent(String(maxLists)))
+          .replace(
+            `{${"ignoreProtected"}}`,
+            encodeURIComponent(String(ignoreProtected))
+          );
+      // use dummy base URL string because the URL constructor only accepts absolute URLs.
+      const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
+      let baseOptions;
+      if (configuration) {
+        baseOptions = configuration.baseOptions;
+      }
+
+      const localVarRequestOptions = {
+        method: "GET",
+        ...baseOptions,
+        ...options,
+      };
+      const localVarHeaderParameter = {} as any;
+      const localVarQueryParameter = {} as any;
+
+      setSearchParams(localVarUrlObj, localVarQueryParameter, options.query);
+      let headersFromBaseOptions =
+        baseOptions && baseOptions.headers ? baseOptions.headers : {};
+      localVarRequestOptions.headers = {
+        ...localVarHeaderParameter,
+        ...headersFromBaseOptions,
+        ...options.headers,
+      };
+
+      return {
+        url: toPathString(localVarUrlObj),
+        options: localVarRequestOptions,
+      };
+    },
     /**
      *
      * @param {string} projectId
@@ -526,6 +604,42 @@ export const MergeApiFp = function (configuration?: Configuration) {
         configuration
       );
     },
+    /**
+     *
+     * @param {string} projectId
+     * @param {number} maxInList
+     * @param {number} maxLists
+     * @param {boolean} ignoreProtected
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     */
+    async findIdenticalPotentialDuplicates(
+      projectId: string,
+      maxInList: number,
+      maxLists: number,
+      ignoreProtected: boolean,
+      options?: any
+    ): Promise<
+      (
+        axios?: AxiosInstance,
+        basePath?: string
+      ) => AxiosPromise<Array<Array<Word>>>
+    > {
+      const localVarAxiosArgs =
+        await localVarAxiosParamCreator.findIdenticalPotentialDuplicates(
+          projectId,
+          maxInList,
+          maxLists,
+          ignoreProtected,
+          options
+        );
+      return createRequestFunction(
+        localVarAxiosArgs,
+        globalAxios,
+        BASE_PATH,
+        configuration
+      );
+    },
     /**
      *
      * @param {string} projectId
@@ -754,6 +868,32 @@ export const MergeApiFactory = function (
         .blacklistAdd(projectId, requestBody, options)
         .then((request) => request(axios, basePath));
     },
+    /**
+     *
+     * @param {string} projectId
+     * @param {number} maxInList
+     * @param {number} maxLists
+     * @param {boolean} ignoreProtected
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     */
+    findIdenticalPotentialDuplicates(
+      projectId: string,
+      maxInList: number,
+      maxLists: number,
+      ignoreProtected: boolean,
+      options?: any
+    ): AxiosPromise<Array<Array<Word>>> {
+      return localVarFp
+        .findIdenticalPotentialDuplicates(
+          projectId,
+          maxInList,
+          maxLists,
+          ignoreProtected,
+          options
+        )
+        .then((request) => request(axios, basePath));
+    },
     /**
      *
      * @param {string} projectId
@@ -900,6 +1040,41 @@ export interface MergeApiBlacklistAddRequest {
   readonly requestBody: Array<string>;
 }
 
+/**
+ * Request parameters for findIdenticalPotentialDuplicates operation in MergeApi.
+ * @export
+ * @interface MergeApiFindIdenticalPotentialDuplicatesRequest
+ */
+export interface MergeApiFindIdenticalPotentialDuplicatesRequest {
+  /**
+   *
+   * @type {string}
+   * @memberof MergeApiFindIdenticalPotentialDuplicates
+   */
+  readonly projectId: string;
+
+  /**
+   *
+   * @type {number}
+   * @memberof MergeApiFindIdenticalPotentialDuplicates
+   */
+  readonly maxInList: number;
+
+  /**
+   *
+   * @type {number}
+   * @memberof MergeApiFindIdenticalPotentialDuplicates
+   */
+  readonly maxLists: number;
+
+  /**
+   *
+   * @type {boolean}
+   * @memberof MergeApiFindIdenticalPotentialDuplicates
+   */
+  readonly ignoreProtected: boolean;
+}
+
 /**
  * Request parameters for findPotentialDuplicates operation in MergeApi.
  * @export
@@ -1088,6 +1263,28 @@ export class MergeApi extends BaseAPI {
       .then((request) => request(this.axios, this.basePath));
   }
 
+  /**
+   *
+   * @param {MergeApiFindIdenticalPotentialDuplicatesRequest} requestParameters Request parameters.
+   * @param {*} [options] Override http request option.
+   * @throws {RequiredError}
+   * @memberof MergeApi
+   */
+  public findIdenticalPotentialDuplicates(
+    requestParameters: MergeApiFindIdenticalPotentialDuplicatesRequest,
+    options?: any
+  ) {
+    return MergeApiFp(this.configuration)
+      .findIdenticalPotentialDuplicates(
+        requestParameters.projectId,
+        requestParameters.maxInList,
+        requestParameters.maxLists,
+        requestParameters.ignoreProtected,
+        options
+      )
+      .then((request) => request(this.axios, this.basePath));
+  }
+
   /**
    *
    * @param {MergeApiFindPotentialDuplicatesRequest} requestParameters Request parameters.

src/backend/index.ts

@@ -362,6 +362,20 @@ export async function graylistAdd(wordIds: string[]): Promise<void> {
   );
 }
 
+/** Find and return lists of potential duplicates with identical vernacular. */
+export async function findIdenticalDuplicates(
+  maxInList: number,
+  maxLists: number,
+  ignoreProtected = false
+): Promise<Word[][]> {
+  const projectId = LocalStorage.getProjectId();
+  const resp = await mergeApi.findIdenticalPotentialDuplicates(
+    { ignoreProtected, maxInList, maxLists, projectId },
+    defaultOptions()
+  );
+  return resp.data;
+}
+
 /** Start finding list of potential duplicates for merging. */
 export async function findDuplicates(
   maxInList: number,