From 25b26442a40c45c30e578ac2d9a74cc7522872dc Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 3 Nov 2025 17:15:37 +0800 Subject: [PATCH 01/22] fix #17637 --- .../api/service/impl/WorkflowDefinitionServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 7fe3d00a511e..6759b7f778c6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -2428,8 +2428,9 @@ public void deleteWorkflowDefinitionVersion(User loginUser, log.info("Delete version: {} of workflow: {}, projectCode: {}", version, code, projectCode); // delete workflow lineage + // It's safe to return 0 if no lineage exists (idempotent operation) int deleteWorkflowLineageResult = workflowLineageService.deleteWorkflowLineage(Collections.singletonList(code)); - if (deleteWorkflowLineageResult <= 0) { + if (deleteWorkflowLineageResult < 0) { log.error("Delete workflow lineage by workflow definition code error, workflowDefinitionCode: {}", code); throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); } From 5f839102bee6c3340330421b2eadfb0aef26378c Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 3 Nov 2025 19:11:38 +0800 Subject: [PATCH 02/22] Because workflow lineage only retains the current version, deleting historical versions in the workflow does not invoke lineage deletion. Therefore, logic for deleting lineages should be added when deleting workflows. --- .../impl/WorkflowDefinitionServiceImpl.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 6759b7f778c6..77991e7c7f8d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1123,6 +1123,14 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { // If delete error, we can call this interface again. workflowDefinitionDao.deleteByWorkflowDefinitionCode(workflowDefinition.getCode()); metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(code); + + // delete workflow lineage (lineage data only keeps one record per workflow code) + // It's safe to return 0 if no lineage exists (idempotent) + int deleteWorkflowLineageResult = workflowLineageService + .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); + if (deleteWorkflowLineageResult < 0) { + throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + } log.info("Success delete workflow definition workflowDefinitionCode: {}", code); } @@ -2427,13 +2435,6 @@ public void deleteWorkflowDefinitionVersion(User loginUser, } log.info("Delete version: {} of workflow: {}, projectCode: {}", version, code, projectCode); - // delete workflow lineage - // It's safe to return 0 if no lineage exists (idempotent operation) - int deleteWorkflowLineageResult = workflowLineageService.deleteWorkflowLineage(Collections.singletonList(code)); - if (deleteWorkflowLineageResult < 0) { - log.error("Delete workflow lineage by workflow definition code error, workflowDefinitionCode: {}", code); - throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); - } } private void updateWorkflowValid(User user, WorkflowDefinition oldWorkflowDefinition, From 06b7a82e3e9ba5de1ae52a6d79388fdd3317f7c3 Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 4 Nov 2025 16:03:10 +0800 Subject: [PATCH 03/22] Optimize workflow deletion logic to ensure compatibility with historical library data. --- .../impl/WorkflowDefinitionServiceImpl.java | 8 ++++++-- .../service/impl/WorkflowLineageServiceImpl.java | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 77991e7c7f8d..a01f5d1dd2d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1128,8 +1128,12 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { // It's safe to return 0 if no lineage exists (idempotent) int deleteWorkflowLineageResult = workflowLineageService .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); - if (deleteWorkflowLineageResult < 0) { - throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + if (deleteWorkflowLineageResult <= 0) { + if(deleteWorkflowLineageResult < 0) { + throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + }else{ + log.warn("No workflow lineage to delete, workflowDefinitionCode: {}", code); + } } log.info("Success delete workflow definition workflowDefinitionCode: {}", code); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 4f4f6b4c056e..892ca3b9e85f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -189,10 +189,25 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti if (workflowTaskLineage.getTaskDefinitionCode() != 0) { TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(workflowTaskLineage.getTaskDefinitionCode()); + // Handle dirty data scenario caused by historical bugs: + // There may be orphaned lineage records referencing deleted task definitions. + // Skip these records to prevent NPE and ensure the method continues processing. + // Note: These orphaned records should be cleaned up by a background cleanup task, + // not here to avoid side effects in a read-only query method. + if (taskDefinition == null) { + log.warn("Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " + + "This dirty data should be cleaned up by a background task.", + workflowTaskLineage.getTaskDefinitionCode(), workflowTaskLineage.getId()); + continue; + } taskName = taskDefinition.getName(); } taskDepStrList.add(String.format(Constants.FORMAT_S_S_COLON, workflowDefinition.getName(), taskName)); } + // If no valid task dependencies found, return empty Optional to indicate no dependencies. + if(taskDepStrList.isEmpty()) { + return Optional.empty(); + } String taskDepStr = String.join(Constants.COMMA, taskDepStrList); if (taskCode != 0) { From f4ab8a2c4c76cfd598e69ee3f38046010653c2a6 Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 4 Nov 2025 16:07:57 +0800 Subject: [PATCH 04/22] spotless --- .../api/service/impl/WorkflowDefinitionServiceImpl.java | 4 ++-- .../api/service/impl/WorkflowLineageServiceImpl.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index a01f5d1dd2d6..998fb9de9259 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1129,9 +1129,9 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { int deleteWorkflowLineageResult = workflowLineageService .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); if (deleteWorkflowLineageResult <= 0) { - if(deleteWorkflowLineageResult < 0) { + if (deleteWorkflowLineageResult < 0) { throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); - }else{ + } else { log.warn("No workflow lineage to delete, workflowDefinitionCode: {}", code); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 892ca3b9e85f..9e56e72ae934 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -195,8 +195,9 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti // Note: These orphaned records should be cleaned up by a background cleanup task, // not here to avoid side effects in a read-only query method. if (taskDefinition == null) { - log.warn("Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " - + "This dirty data should be cleaned up by a background task.", + log.warn( + "Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " + + "This dirty data should be cleaned up by a background task.", workflowTaskLineage.getTaskDefinitionCode(), workflowTaskLineage.getId()); continue; } @@ -205,7 +206,7 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti taskDepStrList.add(String.format(Constants.FORMAT_S_S_COLON, workflowDefinition.getName(), taskName)); } // If no valid task dependencies found, return empty Optional to indicate no dependencies. - if(taskDepStrList.isEmpty()) { + if (taskDepStrList.isEmpty()) { return Optional.empty(); } From 22e3d5e1ecdf8738e819a905030df77de903018f Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 3 Nov 2025 17:15:37 +0800 Subject: [PATCH 05/22] fix #17637 --- .../api/service/impl/WorkflowDefinitionServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 7fe3d00a511e..6759b7f778c6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -2428,8 +2428,9 @@ public void deleteWorkflowDefinitionVersion(User loginUser, log.info("Delete version: {} of workflow: {}, projectCode: {}", version, code, projectCode); // delete workflow lineage + // It's safe to return 0 if no lineage exists (idempotent operation) int deleteWorkflowLineageResult = workflowLineageService.deleteWorkflowLineage(Collections.singletonList(code)); - if (deleteWorkflowLineageResult <= 0) { + if (deleteWorkflowLineageResult < 0) { log.error("Delete workflow lineage by workflow definition code error, workflowDefinitionCode: {}", code); throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); } From 53fc596dbd6cba1dabdded180645298372b3283b Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 3 Nov 2025 19:11:38 +0800 Subject: [PATCH 06/22] Because workflow lineage only retains the current version, deleting historical versions in the workflow does not invoke lineage deletion. Therefore, logic for deleting lineages should be added when deleting workflows. --- .../impl/WorkflowDefinitionServiceImpl.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 6759b7f778c6..77991e7c7f8d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1123,6 +1123,14 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { // If delete error, we can call this interface again. workflowDefinitionDao.deleteByWorkflowDefinitionCode(workflowDefinition.getCode()); metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(code); + + // delete workflow lineage (lineage data only keeps one record per workflow code) + // It's safe to return 0 if no lineage exists (idempotent) + int deleteWorkflowLineageResult = workflowLineageService + .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); + if (deleteWorkflowLineageResult < 0) { + throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + } log.info("Success delete workflow definition workflowDefinitionCode: {}", code); } @@ -2427,13 +2435,6 @@ public void deleteWorkflowDefinitionVersion(User loginUser, } log.info("Delete version: {} of workflow: {}, projectCode: {}", version, code, projectCode); - // delete workflow lineage - // It's safe to return 0 if no lineage exists (idempotent operation) - int deleteWorkflowLineageResult = workflowLineageService.deleteWorkflowLineage(Collections.singletonList(code)); - if (deleteWorkflowLineageResult < 0) { - log.error("Delete workflow lineage by workflow definition code error, workflowDefinitionCode: {}", code); - throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); - } } private void updateWorkflowValid(User user, WorkflowDefinition oldWorkflowDefinition, From dd2e35a5e101c315c46abfdd6ed44a281ad4d9ea Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 4 Nov 2025 16:03:10 +0800 Subject: [PATCH 07/22] Optimize workflow deletion logic to ensure compatibility with historical library data. --- .../impl/WorkflowDefinitionServiceImpl.java | 8 ++++++-- .../service/impl/WorkflowLineageServiceImpl.java | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 77991e7c7f8d..a01f5d1dd2d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1128,8 +1128,12 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { // It's safe to return 0 if no lineage exists (idempotent) int deleteWorkflowLineageResult = workflowLineageService .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); - if (deleteWorkflowLineageResult < 0) { - throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + if (deleteWorkflowLineageResult <= 0) { + if(deleteWorkflowLineageResult < 0) { + throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); + }else{ + log.warn("No workflow lineage to delete, workflowDefinitionCode: {}", code); + } } log.info("Success delete workflow definition workflowDefinitionCode: {}", code); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 4f4f6b4c056e..892ca3b9e85f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -189,10 +189,25 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti if (workflowTaskLineage.getTaskDefinitionCode() != 0) { TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(workflowTaskLineage.getTaskDefinitionCode()); + // Handle dirty data scenario caused by historical bugs: + // There may be orphaned lineage records referencing deleted task definitions. + // Skip these records to prevent NPE and ensure the method continues processing. + // Note: These orphaned records should be cleaned up by a background cleanup task, + // not here to avoid side effects in a read-only query method. + if (taskDefinition == null) { + log.warn("Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " + + "This dirty data should be cleaned up by a background task.", + workflowTaskLineage.getTaskDefinitionCode(), workflowTaskLineage.getId()); + continue; + } taskName = taskDefinition.getName(); } taskDepStrList.add(String.format(Constants.FORMAT_S_S_COLON, workflowDefinition.getName(), taskName)); } + // If no valid task dependencies found, return empty Optional to indicate no dependencies. + if(taskDepStrList.isEmpty()) { + return Optional.empty(); + } String taskDepStr = String.join(Constants.COMMA, taskDepStrList); if (taskCode != 0) { From 0dd946db331126d3d46a59b6c117eee06682e8e8 Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 4 Nov 2025 16:07:57 +0800 Subject: [PATCH 08/22] spotless --- .../api/service/impl/WorkflowDefinitionServiceImpl.java | 4 ++-- .../api/service/impl/WorkflowLineageServiceImpl.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index a01f5d1dd2d6..998fb9de9259 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -1129,9 +1129,9 @@ public void deleteWorkflowDefinitionByCode(User loginUser, long code) { int deleteWorkflowLineageResult = workflowLineageService .deleteWorkflowLineage(Collections.singletonList(workflowDefinition.getCode())); if (deleteWorkflowLineageResult <= 0) { - if(deleteWorkflowLineageResult < 0) { + if (deleteWorkflowLineageResult < 0) { throw new ServiceException(Status.DELETE_WORKFLOW_LINEAGE_ERROR); - }else{ + } else { log.warn("No workflow lineage to delete, workflowDefinitionCode: {}", code); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 892ca3b9e85f..9e56e72ae934 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -195,8 +195,9 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti // Note: These orphaned records should be cleaned up by a background cleanup task, // not here to avoid side effects in a read-only query method. if (taskDefinition == null) { - log.warn("Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " - + "This dirty data should be cleaned up by a background task.", + log.warn( + "Orphaned lineage record detected: taskDefinitionCode {} not found, workflowTaskLineageId: {}. " + + "This dirty data should be cleaned up by a background task.", workflowTaskLineage.getTaskDefinitionCode(), workflowTaskLineage.getId()); continue; } @@ -205,7 +206,7 @@ public Optional taskDependentMsg(long projectCode, long workflowDefiniti taskDepStrList.add(String.format(Constants.FORMAT_S_S_COLON, workflowDefinition.getName(), taskName)); } // If no valid task dependencies found, return empty Optional to indicate no dependencies. - if(taskDepStrList.isEmpty()) { + if (taskDepStrList.isEmpty()) { return Optional.empty(); } From 0fa45dbe2c53e07ace03a39aa7b394d75cb00020 Mon Sep 17 00:00:00 2001 From: luxl Date: Thu, 6 Nov 2025 20:28:11 +0800 Subject: [PATCH 09/22] add ut test --- .../WorkflowDefinitionServiceTest.java | 9 ++ .../WorkflowTaskLineageServiceTest.java | 128 ++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java index ecee150ee52a..fc788c768406 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java @@ -618,6 +618,7 @@ public void deleteWorkflowDefinitionByCodeTest() { when(scheduleMapper.deleteById(46)).thenReturn(1); when(workflowLineageService.taskDependentMsg(project.getCode(), workflowDefinition.getCode(), 0)) .thenReturn(Optional.empty()); + when(workflowLineageService.deleteWorkflowLineage(anyList())).thenReturn(1); processDefinitionService.deleteWorkflowDefinitionByCode(user, 46L); Mockito.verify(metricsCleanUpService, times(1)).cleanUpWorkflowMetricsByDefinitionCode(46L); @@ -643,8 +644,16 @@ public void deleteWorkflowDefinitionByCodeTest() { when(scheduleMapper.deleteById(schedule.getId())).thenReturn(1); when(workflowLineageService.taskDependentMsg(project.getCode(), workflowDefinition.getCode(), 0)) .thenReturn(Optional.empty()); + when(workflowLineageService.deleteWorkflowLineage(anyList())).thenReturn(1); Assertions.assertDoesNotThrow(() -> processDefinitionService.deleteWorkflowDefinitionByCode(user, 46L)); Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(46L); + + // delete success with no lineage (deleteWorkflowLineageResult == 0) + // This tests the new logic that handles idempotent deletion gracefully + when(workflowLineageService.deleteWorkflowLineage(anyList())).thenReturn(0); + Assertions.assertDoesNotThrow(() -> processDefinitionService.deleteWorkflowDefinitionByCode(user, 46L)); + Mockito.verify(metricsCleanUpService, times(3)).cleanUpWorkflowMetricsByDefinitionCode(46L); + Mockito.verify(workflowLineageService, times(3)).deleteWorkflowLineage(anyList()); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java index 26eb12692906..806225e989d7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java @@ -21,18 +21,23 @@ import org.apache.dolphinscheduler.api.service.impl.WorkflowLineageServiceImpl; import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; import org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail; +import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowTaskLineage; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.WorkflowDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.WorkflowTaskLineageDao; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -57,6 +62,12 @@ public class WorkflowTaskLineageServiceTest { @Mock private TaskDefinitionLogMapper taskDefinitionLogMapper; + @Mock + private TaskDefinitionMapper taskDefinitionMapper; + + @Mock + private WorkflowDefinitionMapper workflowDefinitionMapper; + /** * get mock Project * @@ -128,4 +139,121 @@ private List getWorkFlowLineages() { return workFlowLineages; } + @Test + public void testTaskDependentMsgWithOrphanedLineageRecord() { + // Test case: Handle dirty data scenario where taskDefinition is null + long projectCode = 1L; + long workflowDefinitionCode = 100L; + long taskCode = 200L; + + // Create orphaned lineage record (taskDefinitionCode exists but taskDefinition is null) + List dependentWorkflowList = new ArrayList<>(); + WorkflowTaskLineage orphanedLineage = new WorkflowTaskLineage(); + orphanedLineage.setId(1); + orphanedLineage.setDeptWorkflowDefinitionCode(50L); + orphanedLineage.setTaskDefinitionCode(999L); // This task definition doesn't exist + dependentWorkflowList.add(orphanedLineage); + + WorkflowDefinition workflowDefinition = new WorkflowDefinition(); + workflowDefinition.setCode(50L); + workflowDefinition.setName("TestWorkflow"); + + when(workflowTaskLineageDao.queryWorkFlowLineageByDept(projectCode, workflowDefinitionCode, taskCode)) + .thenReturn(dependentWorkflowList); + when(workflowDefinitionMapper.queryByCode(50L)).thenReturn(workflowDefinition); + when(taskDefinitionMapper.queryByCode(999L)).thenReturn(null); // Task definition not found (dirty data) + + // Should return Optional.empty() because all records are orphaned + Optional result = + workflowLineageService.taskDependentMsg(projectCode, workflowDefinitionCode, taskCode); + Assertions.assertFalse(result.isPresent()); + } + + @Test + public void testTaskDependentMsgWithMixedValidAndOrphanedRecords() { + // Test case: Some records are valid, some are orphaned + long projectCode = 1L; + long workflowDefinitionCode = 100L; + long taskCode = 200L; + + List dependentWorkflowList = new ArrayList<>(); + + // Valid lineage record + WorkflowTaskLineage validLineage = new WorkflowTaskLineage(); + validLineage.setId(1); + validLineage.setDeptWorkflowDefinitionCode(50L); + validLineage.setTaskDefinitionCode(300L); + dependentWorkflowList.add(validLineage); + + // Orphaned lineage record (dirty data) + WorkflowTaskLineage orphanedLineage = new WorkflowTaskLineage(); + orphanedLineage.setId(2); + orphanedLineage.setDeptWorkflowDefinitionCode(60L); + orphanedLineage.setTaskDefinitionCode(999L); // This task definition doesn't exist + dependentWorkflowList.add(orphanedLineage); + + WorkflowDefinition workflowDefinition1 = new WorkflowDefinition(); + workflowDefinition1.setCode(50L); + workflowDefinition1.setName("ValidWorkflow"); + + WorkflowDefinition workflowDefinition2 = new WorkflowDefinition(); + workflowDefinition2.setCode(60L); + workflowDefinition2.setName("OrphanedWorkflow"); + + TaskDefinition validTaskDefinition = new TaskDefinition(); + validTaskDefinition.setCode(300L); + validTaskDefinition.setName("ValidTask"); + + when(workflowTaskLineageDao.queryWorkFlowLineageByDept(projectCode, workflowDefinitionCode, taskCode)) + .thenReturn(dependentWorkflowList); + when(workflowDefinitionMapper.queryByCode(50L)).thenReturn(workflowDefinition1); + when(workflowDefinitionMapper.queryByCode(60L)).thenReturn(workflowDefinition2); + when(taskDefinitionMapper.queryByCode(300L)).thenReturn(validTaskDefinition); + when(taskDefinitionMapper.queryByCode(999L)).thenReturn(null); // Orphaned record + + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setCode(taskCode); + taskDefinition.setName("TestTask"); + when(taskDefinitionMapper.queryByCode(taskCode)).thenReturn(taskDefinition); + + // Should return a message with only the valid record, skipping the orphaned one + Optional result = + workflowLineageService.taskDependentMsg(projectCode, workflowDefinitionCode, taskCode); + Assertions.assertTrue(result.isPresent()); + String message = result.get(); + Assertions.assertTrue(message.contains("ValidWorkflow")); + Assertions.assertTrue(message.contains("ValidTask")); + // Orphaned record should be skipped, so it shouldn't appear in the message + Assertions.assertFalse(message.contains("OrphanedWorkflow")); + } + + @Test + public void testTaskDependentMsgWithEmptyListAfterFilteringOrphanedRecords() { + // Test case: All records are orphaned, resulting in empty list + long projectCode = 1L; + long workflowDefinitionCode = 100L; + long taskCode = 0L; // No specific task code + + List dependentWorkflowList = new ArrayList<>(); + WorkflowTaskLineage orphanedLineage = new WorkflowTaskLineage(); + orphanedLineage.setId(1); + orphanedLineage.setDeptWorkflowDefinitionCode(50L); + orphanedLineage.setTaskDefinitionCode(999L); // This task definition doesn't exist + dependentWorkflowList.add(orphanedLineage); + + WorkflowDefinition workflowDefinition = new WorkflowDefinition(); + workflowDefinition.setCode(50L); + workflowDefinition.setName("TestWorkflow"); + + when(workflowTaskLineageDao.queryWorkFlowLineageByDept(projectCode, workflowDefinitionCode, 0L)) + .thenReturn(dependentWorkflowList); + when(workflowDefinitionMapper.queryByCode(50L)).thenReturn(workflowDefinition); + when(taskDefinitionMapper.queryByCode(999L)).thenReturn(null); // Task definition not found + + // Should return Optional.empty() because all records are orphaned + Optional result = + workflowLineageService.taskDependentMsg(projectCode, workflowDefinitionCode, taskCode); + Assertions.assertFalse(result.isPresent()); + } + } From 9d4783f17ed679a8a5e3c54f0dd94066f2dea351 Mon Sep 17 00:00:00 2001 From: luxl Date: Wed, 12 Nov 2025 17:35:33 +0800 Subject: [PATCH 10/22] fix #17638 --- .../api/service/WorkflowLineageService.java | 10 ++- .../impl/WorkflowDefinitionServiceImpl.java | 16 ++-- .../impl/WorkflowLineageServiceImpl.java | 23 ++++- .../WorkflowDefinitionServiceTest.java | 90 +++++++++++++++++++ .../WorkflowTaskLineageServiceTest.java | 83 +++++++++++++++++ 5 files changed, 212 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLineageService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLineageService.java index 2857a38ca17b..f20503b1409e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLineageService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLineageService.java @@ -57,7 +57,15 @@ List queryDependentWorkflowDefinitions(long projectCode, l int createWorkflowLineage(List workflowTaskLineages); - int updateWorkflowLineage(List workflowTaskLineages); + /** + * Replace the lineage of given workflow definition by new lineage list. + * When the list is empty, existing lineage data will be deleted. + * + * @param workflowDefinitionCode workflow definition to update + * @param workflowTaskLineages new lineage list, can be empty + * @return affected rows + */ + int updateWorkflowLineage(long workflowDefinitionCode, List workflowTaskLineages); int deleteWorkflowLineage(List workflowDefinitionCodes); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 998fb9de9259..0ded9013342a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -409,14 +409,16 @@ public void saveWorkflowLineage(long projectCode, long workflowDefinitionCode, int workflowDefinitionVersion, List taskDefinitionLogList) { - List workflowTaskLineageList = - generateWorkflowLineageList(taskDefinitionLogList, workflowDefinitionCode, workflowDefinitionVersion); - if (workflowTaskLineageList.isEmpty()) { - return; - } + List workflowTaskLineageList = generateWorkflowLineageList(taskDefinitionLogList, + workflowDefinitionCode, workflowDefinitionVersion); - int insertWorkflowLineageResult = workflowLineageService.updateWorkflowLineage(workflowTaskLineageList); - if (insertWorkflowLineageResult <= 0) { + int insertWorkflowLineageResult = + workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineageList); + if (CollectionUtils.isEmpty(workflowTaskLineageList)) { + log.info( + "Delete workflow lineage because current lineage is empty, projectCode: {}, workflowDefinitionCode: {}, workflowDefinitionVersion: {}", + projectCode, workflowDefinitionCode, workflowDefinitionVersion); + } else if (insertWorkflowLineageResult <= 0) { log.error( "Save workflow lineage error, projectCode: {}, workflowDefinitionCode: {}, workflowDefinitionVersion: {}", projectCode, workflowDefinitionCode, workflowDefinitionVersion); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 9e56e72ae934..626eb4fc6f0f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -38,6 +38,7 @@ import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -318,8 +319,26 @@ public int createWorkflowLineage(List workflowTaskLineages) } @Override - public int updateWorkflowLineage(List workflowTaskLineages) { - return workflowTaskLineageDao.updateWorkflowTaskLineage(workflowTaskLineages); + public int updateWorkflowLineage(long workflowDefinitionCode, List workflowTaskLineages) { + // Remove existing lineage first to keep data consistent + workflowTaskLineageDao.batchDeleteByWorkflowDefinitionCode( + Collections.singletonList(workflowDefinitionCode)); + + if (CollectionUtils.isEmpty(workflowTaskLineages)) { + return 0; + } + + boolean hasMismatch = workflowTaskLineages.stream() + .anyMatch(lineage -> lineage.getWorkflowDefinitionCode() != workflowDefinitionCode); + if (hasMismatch) { + log.warn("Skip updating lineage due to workflowDefinitionCode mismatch, expected: {}", + workflowDefinitionCode); + throw new IllegalArgumentException( + String.format("All lineage items must belong to workflowDefinitionCode %s", + workflowDefinitionCode)); + } + + return workflowTaskLineageDao.batchInsert(workflowTaskLineages); } @Override diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java index fc788c768406..6602e6f0b611 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; @@ -1390,4 +1391,93 @@ private MultipartFile createMultipartFile(String filePath) throws URISyntaxExcep content); return multipartFile; } + + @Test + public void testSaveWorkflowLineageWithEmptyList() { + // Test case: Empty lineage list should delete historical lineage + long projectCode = 1L; + long workflowDefinitionCode = 100L; + int workflowDefinitionVersion = 1; + List emptyTaskDefinitionLogList = new ArrayList<>(); + + // Mock updateWorkflowLineage to return 0 for empty list + when(workflowLineageService.updateWorkflowLineage(eq(workflowDefinitionCode), anyList())) + .thenReturn(0); + + // Execute - should not throw exception + Assertions.assertDoesNotThrow(() -> { + processDefinitionService.saveWorkflowLineage(projectCode, workflowDefinitionCode, + workflowDefinitionVersion, emptyTaskDefinitionLogList); + }); + + // Verify that updateWorkflowLineage was called with empty list + verify(workflowLineageService).updateWorkflowLineage(eq(workflowDefinitionCode), anyList()); + } + + @Test + public void testSaveWorkflowLineageWithNonEmptyList() { + // Test case: Normal save with non-empty lineage list + long projectCode = 1L; + long workflowDefinitionCode = 100L; + int workflowDefinitionVersion = 1; + + // Create task definition logs with dependent tasks + List taskDefinitionLogList = new ArrayList<>(); + TaskDefinitionLog taskLog = new TaskDefinitionLog(); + taskLog.setCode(200L); + taskLog.setVersion(1); + taskLog.setProjectCode(projectCode); + taskLog.setTaskType("DEPENDENT"); + // Set taskParams with dependent parameters + String taskParams = + "{\"dependence\":{\"dependTaskList\":[{\"dependItemList\":[{\"definitionCode\":50,\"depTaskCode\":300}]}]}}"; + taskLog.setTaskParams(taskParams); + taskDefinitionLogList.add(taskLog); + + // Mock updateWorkflowLineage to return success + when(workflowLineageService.updateWorkflowLineage(eq(workflowDefinitionCode), anyList())) + .thenReturn(1); + + // Execute - should not throw exception + Assertions.assertDoesNotThrow(() -> { + processDefinitionService.saveWorkflowLineage(projectCode, workflowDefinitionCode, + workflowDefinitionVersion, taskDefinitionLogList); + }); + + // Verify that updateWorkflowLineage was called + verify(workflowLineageService).updateWorkflowLineage(eq(workflowDefinitionCode), anyList()); + } + + @Test + public void testSaveWorkflowLineageWithInsertFailure() { + // Test case: Should throw exception when insert fails + long projectCode = 1L; + long workflowDefinitionCode = 100L; + int workflowDefinitionVersion = 1; + + // Create task definition logs + List taskDefinitionLogList = new ArrayList<>(); + TaskDefinitionLog taskLog = new TaskDefinitionLog(); + taskLog.setCode(200L); + taskLog.setVersion(1); + taskLog.setProjectCode(projectCode); + taskLog.setTaskType("DEPENDENT"); + String taskParams = + "{\"dependence\":{\"dependTaskList\":[{\"dependItemList\":[{\"definitionCode\":50,\"depTaskCode\":300}]}]}}"; + taskLog.setTaskParams(taskParams); + taskDefinitionLogList.add(taskLog); + + // Mock updateWorkflowLineage to return 0 (insert failure) + when(workflowLineageService.updateWorkflowLineage(eq(workflowDefinitionCode), anyList())) + .thenReturn(0); + + // Execute and verify exception + ServiceException exception = Assertions.assertThrows(ServiceException.class, () -> { + processDefinitionService.saveWorkflowLineage(projectCode, workflowDefinitionCode, + workflowDefinitionVersion, taskDefinitionLogList); + }); + + Assertions.assertEquals(Status.CREATE_WORKFLOW_LINEAGE_ERROR.getCode(), exception.getCode()); + verify(workflowLineageService).updateWorkflowLineage(eq(workflowDefinitionCode), anyList()); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java index 806225e989d7..21acb5be6dc0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.service; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.api.service.impl.WorkflowLineageServiceImpl; @@ -256,4 +259,84 @@ public void testTaskDependentMsgWithEmptyListAfterFilteringOrphanedRecords() { Assertions.assertFalse(result.isPresent()); } + @Test + public void testUpdateWorkflowLineageWithNonEmptyList() { + // Test case: Normal update with non-empty lineage list + long workflowDefinitionCode = 100L; + List workflowTaskLineages = new ArrayList<>(); + + WorkflowTaskLineage lineage1 = new WorkflowTaskLineage(); + lineage1.setWorkflowDefinitionCode(workflowDefinitionCode); + lineage1.setTaskDefinitionCode(200L); + workflowTaskLineages.add(lineage1); + + WorkflowTaskLineage lineage2 = new WorkflowTaskLineage(); + lineage2.setWorkflowDefinitionCode(workflowDefinitionCode); + lineage2.setTaskDefinitionCode(300L); + workflowTaskLineages.add(lineage2); + + // Mock DAO methods + when(workflowTaskLineageDao.batchDeleteByWorkflowDefinitionCode(anyList())).thenReturn(2); + when(workflowTaskLineageDao.batchInsert(workflowTaskLineages)).thenReturn(2); + + // Execute + int result = workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineages); + + // Verify + Assertions.assertEquals(2, result); + verify(workflowTaskLineageDao) + .batchDeleteByWorkflowDefinitionCode(eq(java.util.Collections.singletonList(workflowDefinitionCode))); + verify(workflowTaskLineageDao).batchInsert(workflowTaskLineages); + } + + @Test + public void testUpdateWorkflowLineageWithEmptyList() { + // Test case: Empty list should delete historical lineage and return 0 + long workflowDefinitionCode = 100L; + List emptyList = new ArrayList<>(); + + // Mock DAO method + when(workflowTaskLineageDao.batchDeleteByWorkflowDefinitionCode(anyList())).thenReturn(1); + + // Execute + int result = workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, emptyList); + + // Verify + Assertions.assertEquals(0, result); + verify(workflowTaskLineageDao) + .batchDeleteByWorkflowDefinitionCode(eq(java.util.Collections.singletonList(workflowDefinitionCode))); + // batchInsert should not be called when list is empty + } + + @Test + public void testUpdateWorkflowLineageWithMismatchedWorkflowDefinitionCode() { + // Test case: Should throw exception when lineage items have different workflowDefinitionCode + long workflowDefinitionCode = 100L; + List workflowTaskLineages = new ArrayList<>(); + + WorkflowTaskLineage lineage1 = new WorkflowTaskLineage(); + lineage1.setWorkflowDefinitionCode(workflowDefinitionCode); + lineage1.setTaskDefinitionCode(200L); + workflowTaskLineages.add(lineage1); + + // Add a lineage with different workflowDefinitionCode + WorkflowTaskLineage lineage2 = new WorkflowTaskLineage(); + lineage2.setWorkflowDefinitionCode(999L); // Different code + lineage2.setTaskDefinitionCode(300L); + workflowTaskLineages.add(lineage2); + + // Mock DAO method for deletion + when(workflowTaskLineageDao.batchDeleteByWorkflowDefinitionCode(anyList())).thenReturn(1); + + // Execute and verify exception + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, () -> { + workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineages); + }); + + Assertions.assertTrue(exception.getMessage().contains(String.valueOf(workflowDefinitionCode))); + verify(workflowTaskLineageDao) + .batchDeleteByWorkflowDefinitionCode(eq(java.util.Collections.singletonList(workflowDefinitionCode))); + // batchInsert should not be called when validation fails + } + } From 2f907cbe9d47c71b5de255b12f12887914078bed Mon Sep 17 00:00:00 2001 From: luxl Date: Wed, 19 Nov 2025 11:09:44 +0800 Subject: [PATCH 11/22] Simplify the logic of WorkflowDefinitionService.saveWorkflowLineage(). --- .../impl/WorkflowDefinitionServiceImpl.java | 17 +-------------- .../impl/WorkflowLineageServiceImpl.java | 18 ++++++++-------- .../WorkflowDefinitionServiceTest.java | 4 ++-- .../WorkflowTaskLineageServiceTest.java | 21 ++++++++----------- 4 files changed, 21 insertions(+), 39 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index 0ded9013342a..f0c060562035 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -412,22 +412,7 @@ public void saveWorkflowLineage(long projectCode, List workflowTaskLineageList = generateWorkflowLineageList(taskDefinitionLogList, workflowDefinitionCode, workflowDefinitionVersion); - int insertWorkflowLineageResult = - workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineageList); - if (CollectionUtils.isEmpty(workflowTaskLineageList)) { - log.info( - "Delete workflow lineage because current lineage is empty, projectCode: {}, workflowDefinitionCode: {}, workflowDefinitionVersion: {}", - projectCode, workflowDefinitionCode, workflowDefinitionVersion); - } else if (insertWorkflowLineageResult <= 0) { - log.error( - "Save workflow lineage error, projectCode: {}, workflowDefinitionCode: {}, workflowDefinitionVersion: {}", - projectCode, workflowDefinitionCode, workflowDefinitionVersion); - throw new ServiceException(Status.CREATE_WORKFLOW_LINEAGE_ERROR); - } else { - log.info( - "Save workflow lineage complete, projectCode: {}, workflowDefinitionCode: {}, workflowDefinitionVersion: {}", - projectCode, workflowDefinitionCode, workflowDefinitionVersion); - } + workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineageList); } private List generateWorkflowLineageList(List taskDefinitionLogList, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java index 626eb4fc6f0f..bfd8f91dd520 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLineageServiceImpl.java @@ -325,20 +325,20 @@ public int updateWorkflowLineage(long workflowDefinitionCode, List lineage.getWorkflowDefinitionCode() != workflowDefinitionCode); - if (hasMismatch) { - log.warn("Skip updating lineage due to workflowDefinitionCode mismatch, expected: {}", - workflowDefinitionCode); - throw new IllegalArgumentException( - String.format("All lineage items must belong to workflowDefinitionCode %s", - workflowDefinitionCode)); + int insertResult = workflowTaskLineageDao.batchInsert(workflowTaskLineages); + if (insertResult <= 0) { + log.error("Save workflow lineage error, workflowDefinitionCode: {}", workflowDefinitionCode); + throw new ServiceException(Status.CREATE_WORKFLOW_LINEAGE_ERROR); } - return workflowTaskLineageDao.batchInsert(workflowTaskLineages); + log.info("Save workflow lineage complete, workflowDefinitionCode: {}, inserted rows: {}", + workflowDefinitionCode, insertResult); + return insertResult; } @Override diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java index 6602e6f0b611..d11f1ef9b5db 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java @@ -1467,9 +1467,9 @@ public void testSaveWorkflowLineageWithInsertFailure() { taskLog.setTaskParams(taskParams); taskDefinitionLogList.add(taskLog); - // Mock updateWorkflowLineage to return 0 (insert failure) + // Mock updateWorkflowLineage to throw exception (insert failure) when(workflowLineageService.updateWorkflowLineage(eq(workflowDefinitionCode), anyList())) - .thenReturn(0); + .thenThrow(new ServiceException(Status.CREATE_WORKFLOW_LINEAGE_ERROR)); // Execute and verify exception ServiceException exception = Assertions.assertThrows(ServiceException.class, () -> { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java index 21acb5be6dc0..c8be7bba6935 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowTaskLineageServiceTest.java @@ -22,6 +22,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.impl.WorkflowLineageServiceImpl; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -309,8 +311,8 @@ public void testUpdateWorkflowLineageWithEmptyList() { } @Test - public void testUpdateWorkflowLineageWithMismatchedWorkflowDefinitionCode() { - // Test case: Should throw exception when lineage items have different workflowDefinitionCode + public void testUpdateWorkflowLineageWithInsertFailure() { + // Test case: Should throw exception when insert fails long workflowDefinitionCode = 100L; List workflowTaskLineages = new ArrayList<>(); @@ -319,24 +321,19 @@ public void testUpdateWorkflowLineageWithMismatchedWorkflowDefinitionCode() { lineage1.setTaskDefinitionCode(200L); workflowTaskLineages.add(lineage1); - // Add a lineage with different workflowDefinitionCode - WorkflowTaskLineage lineage2 = new WorkflowTaskLineage(); - lineage2.setWorkflowDefinitionCode(999L); // Different code - lineage2.setTaskDefinitionCode(300L); - workflowTaskLineages.add(lineage2); - - // Mock DAO method for deletion + // Mock DAO methods when(workflowTaskLineageDao.batchDeleteByWorkflowDefinitionCode(anyList())).thenReturn(1); + when(workflowTaskLineageDao.batchInsert(workflowTaskLineages)).thenReturn(0); // Insert failure // Execute and verify exception - IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, () -> { + ServiceException exception = Assertions.assertThrows(ServiceException.class, () -> { workflowLineageService.updateWorkflowLineage(workflowDefinitionCode, workflowTaskLineages); }); - Assertions.assertTrue(exception.getMessage().contains(String.valueOf(workflowDefinitionCode))); + Assertions.assertEquals(Status.CREATE_WORKFLOW_LINEAGE_ERROR.getCode(), exception.getCode()); verify(workflowTaskLineageDao) .batchDeleteByWorkflowDefinitionCode(eq(java.util.Collections.singletonList(workflowDefinitionCode))); - // batchInsert should not be called when validation fails + verify(workflowTaskLineageDao).batchInsert(workflowTaskLineages); } } From 5a63d8e691a06f55f96a3a12076cea0738d77cb0 Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 24 Nov 2025 14:25:05 +0800 Subject: [PATCH 12/22] fix #17704 --- .../plugin/task/http/HttpTask.java | 2 ++ .../plugin/task/http/HttpTaskTest.java | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 2f2cd4b2d39b..6af83639f0a5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -74,6 +74,8 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { OkHttpResponse httpResponse = sendRequest(); validateResponse(httpResponse.getBody(), httpResponse.getStatusCode()); + + taskExecutionContext.setVarPool(httpParameters.getVarPool()); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java index ba211f6bd78c..a48e6dd9be3d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java @@ -181,6 +181,39 @@ public void testAddDefaultOutput() throws Exception { Assertions.assertEquals(response, property.getValue()); } + @Test + public void testHandleSetsVarPoolToTaskExecutionContext() throws Exception { + String responseBody = "{\"status\": \"success\", \"data\": \"test\"}"; + String taskName = "testHttpTask"; + + TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + Mockito.when(taskExecutionContext.getTaskName()).thenReturn(taskName); + + String url = withMockWebServer(DEFAULT_MOCK_PATH, HttpStatus.SC_OK, responseBody); + String paramData = generateHttpParameters(url, HttpRequestMethod.GET, "", + new ArrayList<>(), HttpCheckCondition.STATUS_CODE_DEFAULT, ""); + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(paramData); + + HttpTask httpTask = new HttpTask(taskExecutionContext); + httpTask.init(); + httpTask.handle(null); + + Mockito.verify(taskExecutionContext, Mockito.times(1)).setVarPool(Mockito.anyList()); + + Mockito.verify(taskExecutionContext).setVarPool(Mockito.argThat(varPool -> { + if (varPool == null || varPool.isEmpty()) { + return false; + } + Property property = varPool.get(0); + return property.getProp().equals(taskName + ".response") + && property.getDirect() == Direct.OUT + && property.getType() == DataType.VARCHAR + && property.getValue().contains("status"); + })); + + Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode()); + } + private String withMockWebServer(String path, int actualResponseCode, String actualResponseBody) throws IOException { MockWebServer server = new MockWebServer(); From 78e3bcc1ad2a24f96b728672aad89e9381d66640 Mon Sep 17 00:00:00 2001 From: luxiaolong Date: Wed, 26 Nov 2025 17:11:24 +0800 Subject: [PATCH 13/22] Update dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../apache/dolphinscheduler/plugin/task/http/HttpTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 6af83639f0a5..2ad40adaa46f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -73,9 +73,9 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { OkHttpResponse httpResponse = sendRequest(); - validateResponse(httpResponse.getBody(), httpResponse.getStatusCode()); - taskExecutionContext.setVarPool(httpParameters.getVarPool()); + + validateResponse(httpResponse.getBody(), httpResponse.getStatusCode()); } @Override From cf3b24087efc413e5d2bdd842003c39988701f0f Mon Sep 17 00:00:00 2001 From: luxl Date: Wed, 17 Dec 2025 20:11:13 +0800 Subject: [PATCH 14/22] Backfill support enables dependency --- .../BackfillWorkflowExecutorDelegate.java | 375 ++++++++++++++++- .../BackfillWorkflowExecutorDelegateTest.java | 389 ++++++++++++++++++ 2 files changed, 745 insertions(+), 19 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java index 3f695e810c40..7b13d169448b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java @@ -17,26 +17,47 @@ package org.apache.dolphinscheduler.api.executor.workflow; +import org.apache.dolphinscheduler.api.dto.workflow.WorkflowBackFillRequest; import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.service.WorkflowLineageService; import org.apache.dolphinscheduler.api.validator.workflow.BackfillWorkflowDTO; +import org.apache.dolphinscheduler.api.validator.workflow.BackfillWorkflowRequestTransformer; import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.ExecutionOrder; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.dao.entity.DependentWorkflowDefinition; +import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; -import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionDao; +import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils; import org.apache.dolphinscheduler.extract.base.client.Clients; import org.apache.dolphinscheduler.extract.master.IWorkflowControlClient; import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowBackfillTriggerRequest; import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowBackfillTriggerResponse; +import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval; +import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; +import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; +import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; +import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.process.ProcessService; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Collections; +import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -51,22 +72,38 @@ public class BackfillWorkflowExecutorDelegate implements IExecutorDelegate> { @Autowired - private CommandDao commandDao; + private RegistryClient registryClient; + + @Autowired + private WorkflowLineageService workflowLineageService; + + @Autowired + private WorkflowDefinitionDao workflowDefinitionDao; @Autowired private ProcessService processService; @Autowired - private RegistryClient registryClient; + private BackfillWorkflowRequestTransformer backfillWorkflowRequestTransformer; @Override public List execute(final BackfillWorkflowDTO backfillWorkflowDTO) { // todo: directly call the master api to do backfill + List workflowInstanceIdList; if (backfillWorkflowDTO.getBackfillParams().getRunMode() == RunMode.RUN_MODE_SERIAL) { - return doSerialBackfillWorkflow(backfillWorkflowDTO); + workflowInstanceIdList = doSerialBackfillWorkflow(backfillWorkflowDTO); } else { - return doParallelBackfillWorkflow(backfillWorkflowDTO); + workflowInstanceIdList = doParallelBackfillWorkflow(backfillWorkflowDTO); + } + + // Trigger dependent workflows after all root workflow instances are created + // This ensures dependent workflows are only triggered once, regardless of parallel partitions + final BackfillWorkflowDTO.BackfillParamsDTO backfillParams = backfillWorkflowDTO.getBackfillParams(); + if (backfillParams.getBackfillDependentMode() == ComplementDependentMode.ALL_DEPENDENT) { + doBackfillDependentWorkflow(backfillWorkflowDTO); } + + return workflowInstanceIdList; } private List doSerialBackfillWorkflow(final BackfillWorkflowDTO backfillWorkflowDTO) { @@ -78,9 +115,7 @@ private List doSerialBackfillWorkflow(final BackfillWorkflowDTO backfil Collections.sort(backfillTimeList); } - final Integer workflowInstanceId = doBackfillWorkflow( - backfillWorkflowDTO, - backfillTimeList.stream().map(DateUtils::dateToString).collect(Collectors.toList())); + final Integer workflowInstanceId = doBackfillWorkflow(backfillWorkflowDTO, backfillTimeList); return Lists.newArrayList(workflowInstanceId); } @@ -97,22 +132,25 @@ private List doParallelBackfillWorkflow(final BackfillWorkflowDTO backf log.info("In parallel mode, current expectedParallelismNumber:{}", expectedParallelismNumber); final List workflowInstanceIdList = Lists.newArrayList(); - for (List stringDate : Lists.partition(listDate, expectedParallelismNumber)) { - final Integer workflowInstanceId = doBackfillWorkflow( - backfillWorkflowDTO, - stringDate.stream().map(DateUtils::dateToString).collect(Collectors.toList())); + for (List datePartition : Lists.partition(listDate, expectedParallelismNumber)) { + final Integer workflowInstanceId = doBackfillWorkflow(backfillWorkflowDTO, datePartition); workflowInstanceIdList.add(workflowInstanceId); } return workflowInstanceIdList; } private Integer doBackfillWorkflow(final BackfillWorkflowDTO backfillWorkflowDTO, - final List backfillTimeList) { + final List backfillDateList) { final Server masterServer = registryClient.getRandomServer(RegistryNodeType.MASTER).orElse(null); if (masterServer == null) { throw new ServiceException("no master server available"); } + // Convert ZonedDateTime to String only when needed for RPC call + List backfillTimeList = backfillDateList.stream() + .map(DateUtils::dateToString) + .collect(Collectors.toList()); + final WorkflowDefinition workflowDefinition = backfillWorkflowDTO.getWorkflowDefinition(); final WorkflowBackfillTriggerRequest backfillTriggerRequest = WorkflowBackfillTriggerRequest.builder() .userId(backfillWorkflowDTO.getLoginUser().getId()) @@ -139,15 +177,314 @@ private Integer doBackfillWorkflow(final BackfillWorkflowDTO backfillWorkflowDTO if (!backfillTriggerResponse.isSuccess()) { throw new ServiceException("Backfill workflow failed: " + backfillTriggerResponse.getMessage()); } + return backfillTriggerResponse.getWorkflowInstanceId(); + } + + /** + * Trigger backfill for dependent workflows recursively + * This method finds all downstream dependent workflows and triggers backfill for each of them + * using the same serial/parallel logic as the main workflow + * + * @param backfillWorkflowDTO the backfill workflow DTO + */ + private void doBackfillDependentWorkflow(final BackfillWorkflowDTO backfillWorkflowDTO) { + final WorkflowDefinition workflowDefinition = backfillWorkflowDTO.getWorkflowDefinition(); final BackfillWorkflowDTO.BackfillParamsDTO backfillParams = backfillWorkflowDTO.getBackfillParams(); - if (backfillParams.getBackfillDependentMode() == ComplementDependentMode.ALL_DEPENDENT) { - doBackfillDependentWorkflow(backfillWorkflowDTO, backfillTimeList); + + boolean allLevelDependent = backfillParams.isAllLevelDependent(); + + List allDependentWorkflows = + getAllDependentWorkflows( + workflowDefinition.getCode(), + allLevelDependent); + + if (allDependentWorkflows.isEmpty()) { + log.info("No dependent workflows found for workflow definition code: {}.", + workflowDefinition.getCode()); + return; } - return backfillTriggerResponse.getWorkflowInstanceId(); + + log.info("Found {} dependent workflows for workflow definition code: {}.", + allDependentWorkflows.size(), workflowDefinition.getCode()); + + RunMode runMode = backfillParams.getRunMode(); + + for (DependentWorkflowDefinition dependentWorkflowDefinition : allDependentWorkflows) { + try { + // The backfill dates of dependent workflows are consistent with the main workflow. + // In the future, we can consider calculating the backfill dates of dependent workflows + // based on the dependency cycle. + BackfillWorkflowDTO dependentBackfillDTO = buildDependentBackfillDTO( + backfillWorkflowDTO, dependentWorkflowDefinition); + + // Recursively trigger dependent workflow using the same serial/parallel logic + if (runMode == RunMode.RUN_MODE_SERIAL) { + doSerialBackfillWorkflow(dependentBackfillDTO); + } else { + doParallelBackfillWorkflow(dependentBackfillDTO); + } + } catch (Exception e) { + log.error("Failed to trigger backfill for dependent workflow definition code: {}, error: {}", + dependentWorkflowDefinition.getWorkflowDefinitionCode(), e.getMessage(), e); + } + } + + log.info("All {} dependent workflows have been triggered.", allDependentWorkflows.size()); + } + + /** + * Build BackfillWorkflowDTO for dependent workflow + * Only execution time, execution mode, and dependent mode use the original workflow's parameters. + * Other configurations use the dependent workflow's own configuration. + */ + private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO originalBackfillDTO, + final DependentWorkflowDefinition dependentWorkflowDefinition) { + // Check if the dependent workflow is online + long dependentWorkflowCode = dependentWorkflowDefinition.getWorkflowDefinitionCode(); + WorkflowDefinition dependentWorkflow = workflowDefinitionDao.queryByCode(dependentWorkflowCode).orElse(null); + if (dependentWorkflow == null) { + throw new ServiceException( + "Dependent workflow definition not found, workflowDefinitionCode: " + dependentWorkflowCode); + } + + if (!ReleaseState.ONLINE.equals(dependentWorkflow.getReleaseState())) { + throw new ServiceException( + "Dependent workflow definition is not online, workflowDefinitionCode: " + dependentWorkflowCode); + } + + // Get Schedule for dependent workflow to retrieve configuration + List schedules = + processService.queryReleaseSchedulerListByWorkflowDefinitionCode(dependentWorkflowCode); + Schedule schedule = schedules.isEmpty() ? null : schedules.get(0); + + // If schedule is null, create a default Schedule with default values to avoid NPE + if (schedule == null) { + schedule = Schedule.builder() + .failureStrategy(FailureStrategy.CONTINUE) + .warningType(WarningType.NONE) + .workflowInstancePriority(Priority.MEDIUM) + .tenantCode(originalBackfillDTO.getTenantCode()) + .environmentCode(null) + .build(); + } + + // Get original workflow's parameters + BackfillWorkflowDTO.BackfillParamsDTO originalParams = originalBackfillDTO.getBackfillParams(); + + // Build BackfillTime from originalBackfillDTO's backfillDateList + // Convert ZonedDateTime list to comma-separated date string + String complementScheduleDateList = calculateDependentBackfillDates(originalParams.getBackfillDateList(), + dependentWorkflowDefinition, originalBackfillDTO.getWorkflowDefinition().getCode()).stream() + .map(DateUtils::dateToString) + .collect(Collectors.joining(",")); + + WorkflowBackFillRequest.BackfillTime backfillTime = WorkflowBackFillRequest.BackfillTime.builder() + .complementScheduleDateList(complementScheduleDateList) + .build(); + + // Build WorkflowBackFillRequest for dependent workflow + String workerGroup = WorkerGroupUtils.getWorkerGroupOrDefault( + dependentWorkflowDefinition.getWorkerGroup()); + + WorkflowBackFillRequest dependentBackfillRequest = WorkflowBackFillRequest.builder() + .loginUser(originalBackfillDTO.getLoginUser()) + .workflowDefinitionCode(dependentWorkflowCode) + .startNodes(null) // In backfill scenario, startNodes is null + .failureStrategy(schedule.getFailureStrategy()) + .taskDependType(TaskDependType.TASK_POST) + .execType(originalBackfillDTO.getExecType()) + .warningType(schedule.getWarningType()) + .warningGroupId(dependentWorkflow.getWarningGroupId()) + .workflowInstancePriority(schedule.getWorkflowInstancePriority()) + .workerGroup(workerGroup) + .tenantCode(schedule.getTenantCode()) + .environmentCode(schedule.getEnvironmentCode()) + .startParamList(dependentWorkflow.getGlobalParams()) + .dryRun(Flag.NO) + .backfillRunMode(originalParams.getRunMode()) + .backfillTime(backfillTime) + .expectedParallelismNumber(originalParams.getExpectedParallelismNumber()) + // Disable recursive execution because dependent workflows are pre-extracted via + // getAllDependentWorkflows, which also handles circular dependencies + .backfillDependentMode(ComplementDependentMode.OFF_MODE) + .allLevelDependent(false) + .executionOrder(originalParams.getExecutionOrder()) + .build(); + + return backfillWorkflowRequestTransformer.transform(dependentBackfillRequest); + } + + /** + * Get all dependent workflows (flattened list, no level grouping) + * If allLevelDependent is true, recursively get all downstream workflows + * If allLevelDependent is false, only get Level 1 workflows + * + * @param workflowDefinitionCode the workflow definition code + * @param allLevelDependent whether to trigger all levels of dependencies + * @return list of all dependent workflow definitions + */ + private List getAllDependentWorkflows( + long workflowDefinitionCode, + boolean allLevelDependent) { + List allWorkflows = new ArrayList<>(); + Set processedWorkflowCodes = new HashSet<>(); + + // Level 1: directly dependent on upstream + List level1Workflows = + workflowLineageService.queryDownstreamDependentWorkflowDefinitions(workflowDefinitionCode); + + // Filter out the current workflow itself to avoid self-triggering + level1Workflows = level1Workflows.stream() + .filter(def -> def.getWorkflowDefinitionCode() != workflowDefinitionCode) + .collect(Collectors.toList()); + + if (level1Workflows.isEmpty()) { + return allWorkflows; + } + + // Add Level 1 workflows + for (DependentWorkflowDefinition def : level1Workflows) { + if (processedWorkflowCodes.add(def.getWorkflowDefinitionCode())) { + allWorkflows.add(def); + } + } + + if (!allLevelDependent) { + // Only Level 1 + return allWorkflows; + } + + // For all level dependent, recursively traverse downstream workflows + List currentLevelWorkflows = new ArrayList<>(level1Workflows); + + while (true) { + List nextLevelWorkflows = new ArrayList<>(); + + for (DependentWorkflowDefinition dependentWorkflowDefinition : currentLevelWorkflows) { + List downstreamList = + workflowLineageService.queryDownstreamDependentWorkflowDefinitions( + dependentWorkflowDefinition.getWorkflowDefinitionCode()); + + for (DependentWorkflowDefinition downstream : downstreamList) { + // Duplicate prevention: only add if not already processed + if (downstream.getWorkflowDefinitionCode() != workflowDefinitionCode + && processedWorkflowCodes.add(downstream.getWorkflowDefinitionCode())) { + nextLevelWorkflows.add(downstream); + allWorkflows.add(downstream); + } + } + } + + if (nextLevelWorkflows.isEmpty()) { + break; + } + + currentLevelWorkflows = new ArrayList<>(nextLevelWorkflows); + } + + log.info("Found {} dependent workflows (all levels) for workflow definition code: {}", + allWorkflows.size(), workflowDefinitionCode); + return allWorkflows; + } + + /** + * Calculate dependent backfill dates based on dependency cycle configuration. + * Only includes dates where the calculated dependent date intervals overlap with upstream backfill dates. + * + *

Example: Downstream dependency cycle > upstream cycle + * 

+     * Upstream: daily backfill on 2025-01-13(Mon) ~ 2025-01-20(Mon)
+     * Downstream: depends on upstream with cycle=WEEK, dateValue="lastMonday"
+     * 
+     * Result: Only 2025-01-20 triggers downstream backfill, because its "lastMonday" (2025-01-13)
+     * exists in upstream list. Other dates don't trigger because their "lastMonday" is outside the upstream range.
+     *
+ * + * @param upstreamBackfillDateList upstream workflow's backfill date list + * @param dependentWorkflowDefinition dependent workflow definition containing dependency configuration + * @param upstreamWorkflowCode upstream workflow code to match the specific dependency item + * @return calculated backfill date list for dependent workflow + */ + private List calculateDependentBackfillDates( + List upstreamBackfillDateList, + DependentWorkflowDefinition dependentWorkflowDefinition, + long upstreamWorkflowCode) { + + List dependentBackfillDateList = new ArrayList<>(); + + String dateValue = getDependentDateValue(dependentWorkflowDefinition, upstreamWorkflowCode); + + if (dateValue == null || dateValue.isEmpty()) { + log.debug("No dateValue found, returning empty list"); + return new ArrayList<>(); + } + + for (ZonedDateTime upstreamBackfillDate : upstreamBackfillDateList) { + // Convert ZonedDateTime to Date for DependentUtils + Date upstreamDate = Date.from(upstreamBackfillDate.toInstant()); + + // Use DependentUtils.getDateIntervalList(Date, String) to calculate dependent date intervals + List dateIntervalList = DependentUtils.getDateIntervalList(upstreamDate, dateValue); + + if (dateIntervalList != null && !dateIntervalList.isEmpty()) { + // Check if any date in upstream list falls within the calculated dependent date intervals + boolean foundMatch = false; + for (DateInterval interval : dateIntervalList) { + // Check each upstream date to see if it falls within this interval + for (ZonedDateTime checkDate : upstreamBackfillDateList) { + Date checkDateAsDate = Date.from(checkDate.toInstant()); + + // Check if checkDate is within [interval.startTime, interval.endTime] + if (!checkDateAsDate.before(interval.getStartTime()) + && !checkDateAsDate.after(interval.getEndTime())) { + if (!dependentBackfillDateList.contains(upstreamBackfillDate)) { + // Downstream backfill date matches the dependency cycle with upstream + dependentBackfillDateList.add(upstreamBackfillDate); + } + foundMatch = true; + break; + } + } + if (foundMatch) { + break; + } + } + } + } + + log.debug("Calculated {} dependent backfill dates from {} upstream dates", + dependentBackfillDateList.size(), upstreamBackfillDateList.size()); + return dependentBackfillDateList; } - private void doBackfillDependentWorkflow(final BackfillWorkflowDTO backfillWorkflowDTO, - final List backfillTimeList) { - // todo: + /** + * Get dateValue from dependent workflow definition for the specified upstream workflow + */ + private String getDependentDateValue(DependentWorkflowDefinition dependentWorkflowDefinition, + long upstreamWorkflowCode) { + try { + DependentParameters dependentParameters = + dependentWorkflowDefinition.getDependentParameters(); + + List dependentTaskModelList = + dependentParameters.getDependence().getDependTaskList(); + + for (DependentTaskModel dependentTaskModel : dependentTaskModelList) { + List dependentItemList = + dependentTaskModel.getDependItemList(); + + for (DependentItem dependentItem : dependentItemList) { + if (upstreamWorkflowCode == dependentItem.getDefinitionCode()) { + return dependentItem.getDateValue(); + } + } + } + } catch (Exception e) { + log.warn("Failed to parse dependent parameters for workflow {}: {}", + dependentWorkflowDefinition.getWorkflowDefinitionCode(), e.getMessage()); + } + + return null; } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java new file mode 100644 index 000000000000..701fed70f2b1 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.DependentWorkflowDefinition; +import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; +import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; +import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; + +import java.lang.reflect.Method; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Test for BackfillWorkflowExecutorDelegate + */ +@ExtendWith(MockitoExtension.class) +public class BackfillWorkflowExecutorDelegateTest { + + @InjectMocks + private BackfillWorkflowExecutorDelegate backfillWorkflowExecutorDelegate; + + private Method calculateDependentBackfillDatesMethod; + + @BeforeEach + public void setUp() throws Exception { + // Get private method using reflection + calculateDependentBackfillDatesMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "calculateDependentBackfillDates", + List.class, + DependentWorkflowDefinition.class, + long.class); + calculateDependentBackfillDatesMethod.setAccessible(true); + } + + /** + * Test case: Downstream depends on weekly cycle (lastMonday) + * Upstream backfills last week Mon-Sun + * Expected: Only Sunday should trigger downstream (as it depends on lastMonday which is in the list) + */ + @Test + public void testCalculateDependentBackfillDates_WeeklyCycle_LastMonday() throws Exception { + // Arrange: upstream backfills last week Mon(13th) - Sun(19th) + List upstreamBackfillDateList = new ArrayList<>(); + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-13T00:00:00Z")); // Monday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-14T00:00:00Z")); // Tuesday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-15T00:00:00Z")); // Wednesday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-16T00:00:00Z")); // Thursday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-17T00:00:00Z")); // Friday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-18T00:00:00Z")); // Saturday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-19T00:00:00Z")); // Sunday + upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-20T00:00:00Z")); // Monday + + // Create dependent workflow definition with "lastMonday" dateValue + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "week", "lastMonday"); + + // Act + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Assert: Only Sunday (19th) should be in result + // Because Sunday's lastMonday is Monday 13th, which exists in upstream list + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.size(), + "Only Sunday should trigger downstream as its lastMonday (13th) is in the upstream list"); + Assertions.assertEquals("2025-01-20", result.get(0).toLocalDate().toString()); + } + + /** + * Test case: Downstream depends on monthly cycle (lastMonthBegin) + * Upstream backfills last week (all dates are in current month) + * Expected: No dates should trigger downstream (lastMonthBegin is not in the list) + */ + @Test + public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Exception { + // Arrange: upstream backfills this month (Jan 13-19) + List upstreamBackfillDateList = new ArrayList<>(); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-14 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-16 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-17 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-18 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-19 00:00:00")); + + // Create dependent workflow definition with "lastMonthBegin" dateValue + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "month", "lastMonthBegin"); + + // Act + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Assert: No dates should be in result + // Because lastMonthBegin (Dec 1st) is not in the upstream list + Assertions.assertNotNull(result); + Assertions.assertEquals(0, result.size(), + "No dates should trigger downstream as lastMonthBegin is not in upstream list"); + } + + /** + * Test case: Downstream depends on hourly cycle (last1Hour) + * Upstream backfills multiple hours + * Expected: Hours that have previous hour in list should trigger downstream + */ + @Test + public void testCalculateDependentBackfillDates_HourlyCycle_Last1Hour() throws Exception { + // Arrange: upstream backfills 5 consecutive hours + List upstreamBackfillDateList = new ArrayList<>(); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 10:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 11:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 12:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 13:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 14:00:00")); + + // Create dependent workflow definition with "last1Hour" dateValue + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "hour", "last1Hour"); + + // Act + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Assert: 11:00, 12:00, 13:00, 14:00 should be in result (not 10:00 as its last1Hour is 9:00 which is not in + // list) + Assertions.assertNotNull(result); + Assertions.assertEquals(4, result.size(), + "4 hours should trigger downstream (all except 10:00)"); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 11)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 12)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 13)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 14)); + } + + /** + * Test case: Downstream depends on daily cycle (last1Days) + * Upstream backfills multiple consecutive days + * Expected: Days that have previous day in list should trigger downstream + */ + @Test + public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Exception { + // Arrange: upstream backfills 5 consecutive days + List upstreamBackfillDateList = new ArrayList<>(); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-10 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-11 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-12 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-14 00:00:00")); + + // Create dependent workflow definition with "last1Days" dateValue + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "day", "last1Days"); + + // Act + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Assert: 11th, 12th, 13th, 14th should be in result (not 10th as its last1Days is 9th which is not in list) + Assertions.assertNotNull(result); + Assertions.assertEquals(4, result.size(), + "4 days should trigger downstream (all except Jan 10th)"); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 11)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 12)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 13)); + Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 14)); + } + + /** + * Test case: Empty upstream backfill list + * Expected: Empty result + */ + @Test + public void testCalculateDependentBackfillDates_EmptyUpstreamList() throws Exception { + // Arrange + List upstreamBackfillDateList = new ArrayList<>(); + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "day", "last1Days"); + + // Act + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Assert + Assertions.assertNotNull(result); + Assertions.assertEquals(0, result.size(), "Empty upstream list should return empty result"); + } + + /** + * Test case: Dependent node depends on two upstream workflows simultaneously + * Upstream1 (100L): daily cycle, last1Days + * Upstream2 (101L): hourly cycle, last1Hour + * Expected: Each upstream is processed independently with its own dependency configuration + */ + @Test + public void testCalculateDependentBackfillDates_TwoUpstreams() throws Exception { + // Arrange: Create a dependent workflow that depends on two upstreams + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinitionWithTwoUpstreams( + 100L, 101L, 200L, + "day", "last1Days", // config for upstream1 (100L) + "hour", "last1Hour"); // config for upstream2 (101L) + + // Upstream1 backfill dates: 5 consecutive days + List upstream1BackfillDateList = new ArrayList<>(); + upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-10 00:00:00")); + upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-11 00:00:00")); + upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-12 00:00:00")); + upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-14 00:00:00")); + + // Upstream2 backfill dates: 5 consecutive hours + List upstream2BackfillDateList = new ArrayList<>(); + upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 10:00:00")); + upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 11:00:00")); + upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 12:00:00")); + upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 13:00:00")); + upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 14:00:00")); + + // Act: Calculate backfill dates for upstream1 (100L) + @SuppressWarnings("unchecked") + List result1 = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstream1BackfillDateList, + dependentWorkflowDefinition, + 100L); // Query for upstream1 + + // Act: Calculate backfill dates for upstream2 (101L) + @SuppressWarnings("unchecked") + List result2 = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstream2BackfillDateList, + dependentWorkflowDefinition, + 101L); // Query for upstream2 + + // Assert: Upstream1 should return 4 dates (11-14, as their last1Days is in the list) + Assertions.assertNotNull(result1); + Assertions.assertEquals(4, result1.size(), + "Upstream1 with daily cycle should return 4 dates"); + Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 11)); + Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 12)); + Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 13)); + Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 14)); + + // Assert: Upstream2 should return 4 dates (11:00-14:00, as their last1Hour is in the list) + Assertions.assertNotNull(result2); + Assertions.assertEquals(4, result2.size(), + "Upstream2 with hourly cycle should return 4 dates"); + Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 11)); + Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 12)); + Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 13)); + Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 14)); + } + + /** + * Helper method to create DependentWorkflowDefinition with taskParams + */ + private DependentWorkflowDefinition createDependentWorkflowDefinition( + long upstreamWorkflowCode, + long downstreamWorkflowCode, + String cycle, + String dateValue) { + DependentWorkflowDefinition definition = new DependentWorkflowDefinition(); + definition.setWorkflowDefinitionCode(downstreamWorkflowCode); + definition.setTaskDefinitionCode(1000L); + + // Create DependentItem + DependentItem dependentItem = new DependentItem(); + dependentItem.setDefinitionCode(upstreamWorkflowCode); + dependentItem.setCycle(cycle); + dependentItem.setDateValue(dateValue); + + // Create DependentTaskModel + DependentTaskModel dependentTaskModel = new DependentTaskModel(); + List dependentItemList = new ArrayList<>(); + dependentItemList.add(dependentItem); + dependentTaskModel.setDependItemList(dependentItemList); + + // Create Dependence + DependentParameters.Dependence dependence = new DependentParameters.Dependence(); + List dependTaskList = new ArrayList<>(); + dependTaskList.add(dependentTaskModel); + dependence.setDependTaskList(dependTaskList); + + // Create DependentParameters + DependentParameters dependentParameters = new DependentParameters(); + dependentParameters.setDependence(dependence); + + // Set taskParams as JSON string + definition.setTaskParams(JSONUtils.toJsonString(dependentParameters)); + + return definition; + } + + /** + * Helper method to create DependentWorkflowDefinition with two upstream dependencies + * This simulates a downstream workflow that depends on two different upstream workflows + */ + private DependentWorkflowDefinition createDependentWorkflowDefinitionWithTwoUpstreams( + long upstream1WorkflowCode, + long upstream2WorkflowCode, + long downstreamWorkflowCode, + String cycle1, + String dateValue1, + String cycle2, + String dateValue2) { + DependentWorkflowDefinition definition = new DependentWorkflowDefinition(); + definition.setWorkflowDefinitionCode(downstreamWorkflowCode); + definition.setTaskDefinitionCode(1000L); + + // Create DependentItem for upstream1 + DependentItem dependentItem1 = new DependentItem(); + dependentItem1.setDefinitionCode(upstream1WorkflowCode); + dependentItem1.setCycle(cycle1); + dependentItem1.setDateValue(dateValue1); + + // Create DependentItem for upstream2 + DependentItem dependentItem2 = new DependentItem(); + dependentItem2.setDefinitionCode(upstream2WorkflowCode); + dependentItem2.setCycle(cycle2); + dependentItem2.setDateValue(dateValue2); + + // Create DependentTaskModel with both items + DependentTaskModel dependentTaskModel = new DependentTaskModel(); + List dependentItemList = new ArrayList<>(); + dependentItemList.add(dependentItem1); + dependentItemList.add(dependentItem2); + dependentTaskModel.setDependItemList(dependentItemList); + + // Create Dependence + DependentParameters.Dependence dependence = new DependentParameters.Dependence(); + List dependTaskList = new ArrayList<>(); + dependTaskList.add(dependentTaskModel); + dependence.setDependTaskList(dependTaskList); + + // Create DependentParameters + DependentParameters dependentParameters = new DependentParameters(); + dependentParameters.setDependence(dependence); + + // Set taskParams as JSON string + definition.setTaskParams(JSONUtils.toJsonString(dependentParameters)); + + return definition; + } +} From 12a9fbddd4efcbb774a7aa0e91cbd735ab2fec64 Mon Sep 17 00:00:00 2001 From: luxl Date: Thu, 18 Dec 2025 19:12:42 +0800 Subject: [PATCH 15/22] fix build backfillDto --- .../BackfillWorkflowExecutorDelegate.java | 50 ++-- .../BackfillWorkflowExecutorDelegateTest.java | 226 ++++++++++-------- 2 files changed, 150 insertions(+), 126 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java index 7b13d169448b..d8095a8a2126 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java @@ -257,7 +257,8 @@ private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO processService.queryReleaseSchedulerListByWorkflowDefinitionCode(dependentWorkflowCode); Schedule schedule = schedules.isEmpty() ? null : schedules.get(0); - // If schedule is null, create a default Schedule with default values to avoid NPE + // If schedule is empty, it means the user has not configured it, + // so a default schedule with default values ​​will be created. if (schedule == null) { schedule = Schedule.builder() .failureStrategy(FailureStrategy.CONTINUE) @@ -300,7 +301,7 @@ private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO .tenantCode(schedule.getTenantCode()) .environmentCode(schedule.getEnvironmentCode()) .startParamList(dependentWorkflow.getGlobalParams()) - .dryRun(Flag.NO) + .dryRun(Flag.YES) .backfillRunMode(originalParams.getRunMode()) .backfillTime(backfillTime) .expectedParallelismNumber(originalParams.getExpectedParallelismNumber()) @@ -315,13 +316,11 @@ private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO } /** - * Get all dependent workflows (flattened list, no level grouping) - * If allLevelDependent is true, recursively get all downstream workflows - * If allLevelDependent is false, only get Level 1 workflows + * Get all dependent workflows (flattened list, no level grouping). * - * @param workflowDefinitionCode the workflow definition code - * @param allLevelDependent whether to trigger all levels of dependencies - * @return list of all dependent workflow definitions + * @param workflowDefinitionCode the workflow definition code of the root workflow + * @param allLevelDependent whether to retrieve all levels of dependencies (true) or only Level 1 (false) + * @return list of all dependent workflow definitions (flattened, no level grouping) */ private List getAllDependentWorkflows( long workflowDefinitionCode, @@ -350,7 +349,6 @@ private List getAllDependentWorkflows( } if (!allLevelDependent) { - // Only Level 1 return allWorkflows; } @@ -388,22 +386,33 @@ private List getAllDependentWorkflows( } /** - * Calculate dependent backfill dates based on dependency cycle configuration. - * Only includes dates where the calculated dependent date intervals overlap with upstream backfill dates. + * Calculate the list of dates that need to be backfilled for the downstream workflow. + * Only includes downstream dates whose dependent upstream dates actually exist in the upstream backfill list. * - *

Example: Downstream dependency cycle > upstream cycle + *

Core logic: For each candidate downstream date, calculate its corresponding upstream dependent date + * according to the dependency cycle rule. If that upstream date appears in the upstream backfill list, + * then this downstream date needs to be backfilled. + * + *

Example: Downstream dependency cycle is WEEK with dateValue="lastMonday" * 

-     * Upstream: daily backfill on 2025-01-13(Mon) ~ 2025-01-20(Mon)
-     * Downstream: depends on upstream with cycle=WEEK, dateValue="lastMonday"
+     * Upstream backfill dates: [2025-01-13(Mon), 2025-01-14(Tue), ..., 2025-01-19(Sun), 2025-01-20(Mon)]
+     * Downstream dependency: cycle=WEEK, dateValue="lastMonday"
+     * 
+     * Calculation process:
+     * Candidate date 2025-01-13: Calculate its "lastMonday" → 2025-01-06 (not in upstream list) → Exclude
+     * Candidate date 2025-01-14: Calculate its "lastMonday" → 2025-01-06 (not in upstream list) → Exclude
+     * ...
+     * Candidate date 2025-01-20: Calculate its "lastMonday" → 2025-01-13 (exists in upstream list) → Include
      * 
-     * Result: Only 2025-01-20 triggers downstream backfill, because its "lastMonday" (2025-01-13)
-     * exists in upstream list. Other dates don't trigger because their "lastMonday" is outside the upstream range.
+     * Result: Downstream backfill dates = [2025-01-20]
+     * Reason: Only 2025-01-20's dependent upstream date (2025-01-13) is actually backfilled
      *
* - * @param upstreamBackfillDateList upstream workflow's backfill date list - * @param dependentWorkflowDefinition dependent workflow definition containing dependency configuration + * @param upstreamBackfillDateList the set of dates actually backfilled by the upstream workflow + * @param dependentWorkflowDefinition dependent workflow definition containing dependency cycle configuration + * (e.g., WEEK, MONTH, DAY, etc.) * @param upstreamWorkflowCode upstream workflow code to match the specific dependency item - * @return calculated backfill date list for dependent workflow + * @return list of downstream dates that need to be backfilled, sorted in ascending chronological order */ private List calculateDependentBackfillDates( List upstreamBackfillDateList, @@ -420,17 +429,14 @@ private List calculateDependentBackfillDates( } for (ZonedDateTime upstreamBackfillDate : upstreamBackfillDateList) { - // Convert ZonedDateTime to Date for DependentUtils Date upstreamDate = Date.from(upstreamBackfillDate.toInstant()); // Use DependentUtils.getDateIntervalList(Date, String) to calculate dependent date intervals List dateIntervalList = DependentUtils.getDateIntervalList(upstreamDate, dateValue); if (dateIntervalList != null && !dateIntervalList.isEmpty()) { - // Check if any date in upstream list falls within the calculated dependent date intervals boolean foundMatch = false; for (DateInterval interval : dateIntervalList) { - // Check each upstream date to see if it falls within this interval for (ZonedDateTime checkDate : upstreamBackfillDateList) { Date checkDateAsDate = Date.from(checkDate.toInstant()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java index 701fed70f2b1..7385d7634a42 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.executor.workflow; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.service.WorkflowLineageService; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.DependentWorkflowDefinition; @@ -34,6 +37,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; /** @@ -45,7 +49,11 @@ public class BackfillWorkflowExecutorDelegateTest { @InjectMocks private BackfillWorkflowExecutorDelegate backfillWorkflowExecutorDelegate; + @Mock + private WorkflowLineageService workflowLineageService; + private Method calculateDependentBackfillDatesMethod; + private Method getAllDependentWorkflowsMethod; @BeforeEach public void setUp() throws Exception { @@ -56,16 +64,17 @@ public void setUp() throws Exception { DependentWorkflowDefinition.class, long.class); calculateDependentBackfillDatesMethod.setAccessible(true); + + getAllDependentWorkflowsMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "getAllDependentWorkflows", + long.class, + boolean.class); + getAllDependentWorkflowsMethod.setAccessible(true); } - /** - * Test case: Downstream depends on weekly cycle (lastMonday) - * Upstream backfills last week Mon-Sun - * Expected: Only Sunday should trigger downstream (as it depends on lastMonday which is in the list) - */ @Test public void testCalculateDependentBackfillDates_WeeklyCycle_LastMonday() throws Exception { - // Arrange: upstream backfills last week Mon(13th) - Sun(19th) + // Arrange: upstream backfills last week Mon(13th) - Sun(19th) and next Monday(20th) List upstreamBackfillDateList = new ArrayList<>(); upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-13T00:00:00Z")); // Monday upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-14T00:00:00Z")); // Tuesday @@ -76,7 +85,6 @@ public void testCalculateDependentBackfillDates_WeeklyCycle_LastMonday() throws upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-19T00:00:00Z")); // Sunday upstreamBackfillDateList.add(ZonedDateTime.parse("2025-01-20T00:00:00Z")); // Monday - // Create dependent workflow definition with "lastMonday" dateValue DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "week", "lastMonday"); @@ -88,19 +96,12 @@ public void testCalculateDependentBackfillDates_WeeklyCycle_LastMonday() throws dependentWorkflowDefinition, 100L); - // Assert: Only Sunday (19th) should be in result - // Because Sunday's lastMonday is Monday 13th, which exists in upstream list + // Assert: Only 2025-01-20 should be in result because its lastMonday (2025-01-13) exists in upstream list Assertions.assertNotNull(result); - Assertions.assertEquals(1, result.size(), - "Only Sunday should trigger downstream as its lastMonday (13th) is in the upstream list"); + Assertions.assertEquals(1, result.size()); Assertions.assertEquals("2025-01-20", result.get(0).toLocalDate().toString()); } - /** - * Test case: Downstream depends on monthly cycle (lastMonthBegin) - * Upstream backfills last week (all dates are in current month) - * Expected: No dates should trigger downstream (lastMonthBegin is not in the list) - */ @Test public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Exception { // Arrange: upstream backfills this month (Jan 13-19) @@ -113,7 +114,6 @@ public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Ex upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-18 00:00:00")); upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-19 00:00:00")); - // Create dependent workflow definition with "lastMonthBegin" dateValue DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "month", "lastMonthBegin"); @@ -125,18 +125,11 @@ public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Ex dependentWorkflowDefinition, 100L); - // Assert: No dates should be in result - // Because lastMonthBegin (Dec 1st) is not in the upstream list + // Assert: Empty result because lastMonthBegin (2024-12-01) is not in upstream list Assertions.assertNotNull(result); - Assertions.assertEquals(0, result.size(), - "No dates should trigger downstream as lastMonthBegin is not in upstream list"); + Assertions.assertEquals(0, result.size()); } - /** - * Test case: Downstream depends on hourly cycle (last1Hour) - * Upstream backfills multiple hours - * Expected: Hours that have previous hour in list should trigger downstream - */ @Test public void testCalculateDependentBackfillDates_HourlyCycle_Last1Hour() throws Exception { // Arrange: upstream backfills 5 consecutive hours @@ -147,7 +140,6 @@ public void testCalculateDependentBackfillDates_HourlyCycle_Last1Hour() throws E upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 13:00:00")); upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 14:00:00")); - // Create dependent workflow definition with "last1Hour" dateValue DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "hour", "last1Hour"); @@ -159,22 +151,16 @@ public void testCalculateDependentBackfillDates_HourlyCycle_Last1Hour() throws E dependentWorkflowDefinition, 100L); - // Assert: 11:00, 12:00, 13:00, 14:00 should be in result (not 10:00 as its last1Hour is 9:00 which is not in + // Assert: 11:00, 12:00, 13:00, 14:00 should be in result (10:00 excluded because its last1Hour 9:00 is not in // list) Assertions.assertNotNull(result); - Assertions.assertEquals(4, result.size(), - "4 hours should trigger downstream (all except 10:00)"); + Assertions.assertEquals(4, result.size()); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 11)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 12)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 13)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getHour() == 14)); } - /** - * Test case: Downstream depends on daily cycle (last1Days) - * Upstream backfills multiple consecutive days - * Expected: Days that have previous day in list should trigger downstream - */ @Test public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Exception { // Arrange: upstream backfills 5 consecutive days @@ -185,7 +171,6 @@ public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Ex upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-14 00:00:00")); - // Create dependent workflow definition with "last1Days" dateValue DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "day", "last1Days"); @@ -197,20 +182,15 @@ public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Ex dependentWorkflowDefinition, 100L); - // Assert: 11th, 12th, 13th, 14th should be in result (not 10th as its last1Days is 9th which is not in list) + // Assert: 11th, 12th, 13th, 14th should be in result (10th excluded because its last1Days 9th is not in list) Assertions.assertNotNull(result); - Assertions.assertEquals(4, result.size(), - "4 days should trigger downstream (all except Jan 10th)"); + Assertions.assertEquals(4, result.size()); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 11)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 12)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 13)); Assertions.assertTrue(result.stream().anyMatch(dt -> dt.getDayOfMonth() == 14)); } - /** - * Test case: Empty upstream backfill list - * Expected: Empty result - */ @Test public void testCalculateDependentBackfillDates_EmptyUpstreamList() throws Exception { // Arrange @@ -226,74 +206,112 @@ public void testCalculateDependentBackfillDates_EmptyUpstreamList() throws Excep dependentWorkflowDefinition, 100L); - // Assert + // Assert: Empty result when upstream list is empty Assertions.assertNotNull(result); - Assertions.assertEquals(0, result.size(), "Empty upstream list should return empty result"); + Assertions.assertEquals(0, result.size()); } - /** - * Test case: Dependent node depends on two upstream workflows simultaneously - * Upstream1 (100L): daily cycle, last1Days - * Upstream2 (101L): hourly cycle, last1Hour - * Expected: Each upstream is processed independently with its own dependency configuration - */ @Test - public void testCalculateDependentBackfillDates_TwoUpstreams() throws Exception { - // Arrange: Create a dependent workflow that depends on two upstreams - DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinitionWithTwoUpstreams( - 100L, 101L, 200L, - "day", "last1Days", // config for upstream1 (100L) - "hour", "last1Hour"); // config for upstream2 (101L) - - // Upstream1 backfill dates: 5 consecutive days - List upstream1BackfillDateList = new ArrayList<>(); - upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-10 00:00:00")); - upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-11 00:00:00")); - upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-12 00:00:00")); - upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); - upstream1BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-14 00:00:00")); - - // Upstream2 backfill dates: 5 consecutive hours - List upstream2BackfillDateList = new ArrayList<>(); - upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 10:00:00")); - upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 11:00:00")); - upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 12:00:00")); - upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 13:00:00")); - upstream2BackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-15 14:00:00")); - - // Act: Calculate backfill dates for upstream1 (100L) + public void testGetAllDependentWorkflows_OnlyLevel1() throws Exception { + // Arrange: Root workflow A has Level 1 dependencies B and C, B has Level 2 dependency D + long rootWorkflowCode = 100L; + long level1WorkflowB = 200L; + long level1WorkflowC = 300L; + long level2WorkflowD = 400L; + + DependentWorkflowDefinition level1B = new DependentWorkflowDefinition(); + level1B.setWorkflowDefinitionCode(level1WorkflowB); + DependentWorkflowDefinition level1C = new DependentWorkflowDefinition(); + level1C.setWorkflowDefinitionCode(level1WorkflowC); + DependentWorkflowDefinition level2D = new DependentWorkflowDefinition(); + level2D.setWorkflowDefinitionCode(level2WorkflowD); + + List level1List = new ArrayList<>(); + level1List.add(level1B); + level1List.add(level1C); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) + .thenReturn(level1List); + + List level2List = new ArrayList<>(); + level2List.add(level2D); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowB)) + .thenReturn(level2List); + + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowC)) + .thenReturn(new ArrayList<>()); + + // Act: Get dependencies with allLevelDependent = false @SuppressWarnings("unchecked") - List result1 = (List) calculateDependentBackfillDatesMethod.invoke( - backfillWorkflowExecutorDelegate, - upstream1BackfillDateList, - dependentWorkflowDefinition, - 100L); // Query for upstream1 + List result = (List) getAllDependentWorkflowsMethod + .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, false); - // Act: Calculate backfill dates for upstream2 (101L) + // Assert: Only Level 1 workflows (B and C) should be returned, Level 2 dependency D is excluded + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.size()); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowB)); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowC)); + Assertions.assertFalse(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level2WorkflowD)); + } + + @Test + public void testGetAllDependentWorkflows_AllLevels() throws Exception { + // Arrange: Root workflow A has Level 1 dependencies B and C, B has Level 2 dependency D + long rootWorkflowCode = 100L; + long level1WorkflowB = 200L; + long level1WorkflowC = 300L; + long level2WorkflowD = 400L; + + DependentWorkflowDefinition level1B = new DependentWorkflowDefinition(); + level1B.setWorkflowDefinitionCode(level1WorkflowB); + DependentWorkflowDefinition level1C = new DependentWorkflowDefinition(); + level1C.setWorkflowDefinitionCode(level1WorkflowC); + DependentWorkflowDefinition level2D = new DependentWorkflowDefinition(); + level2D.setWorkflowDefinitionCode(level2WorkflowD); + + List level1List = new ArrayList<>(); + level1List.add(level1B); + level1List.add(level1C); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) + .thenReturn(level1List); + + List level2List = new ArrayList<>(); + level2List.add(level2D); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowB)) + .thenReturn(level2List); + + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowC)) + .thenReturn(new ArrayList<>()); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level2WorkflowD)) + .thenReturn(new ArrayList<>()); + + // Act: Get dependencies with allLevelDependent = true @SuppressWarnings("unchecked") - List result2 = (List) calculateDependentBackfillDatesMethod.invoke( - backfillWorkflowExecutorDelegate, - upstream2BackfillDateList, - dependentWorkflowDefinition, - 101L); // Query for upstream2 - - // Assert: Upstream1 should return 4 dates (11-14, as their last1Days is in the list) - Assertions.assertNotNull(result1); - Assertions.assertEquals(4, result1.size(), - "Upstream1 with daily cycle should return 4 dates"); - Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 11)); - Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 12)); - Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 13)); - Assertions.assertTrue(result1.stream().anyMatch(dt -> dt.getDayOfMonth() == 14)); - - // Assert: Upstream2 should return 4 dates (11:00-14:00, as their last1Hour is in the list) - Assertions.assertNotNull(result2); - Assertions.assertEquals(4, result2.size(), - "Upstream2 with hourly cycle should return 4 dates"); - Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 11)); - Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 12)); - Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 13)); - Assertions.assertTrue(result2.stream().anyMatch(dt -> dt.getHour() == 14)); + List result = (List) getAllDependentWorkflowsMethod + .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, true); + + // Assert: All levels (B, C, D) should be returned in a flattened list + Assertions.assertNotNull(result); + Assertions.assertEquals(3, result.size()); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowB)); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowC)); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level2WorkflowD)); + } + + @Test + public void testGetAllDependentWorkflows_NoDependencies() throws Exception { + // Arrange: Root workflow has no dependencies + long rootWorkflowCode = 100L; + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) + .thenReturn(new ArrayList<>()); + + // Act + @SuppressWarnings("unchecked") + List result = (List) getAllDependentWorkflowsMethod + .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, true); + + // Assert: Empty result when no dependencies exist + Assertions.assertNotNull(result); + Assertions.assertEquals(0, result.size()); } /** From fc21f06f2db9bac6cf9c59c299feff817226026b Mon Sep 17 00:00:00 2001 From: luxl Date: Fri, 19 Dec 2025 12:03:13 +0800 Subject: [PATCH 16/22] Fixed SQL error: "Unknown column 't_max.workflow_instance_id' in 'on clause'" --- .../apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index e33b472fec6c..a5e965a65a7d 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -254,7 +254,9 @@ group by task_code ) t_max - on instance.workflow_instance_id = t_max.workflow_instance_id + -- Fixed: Use #{workflowInstanceId} directly instead of t_max.workflow_instance_id + -- (subquery doesn't select workflow_instance_id, only task_code and max_end_time) + on instance.workflow_instance_id = #{workflowInstanceId} and instance.task_code = t_max.task_code and instance.end_time = t_max.max_end_time From 7d98d6cb9a412700508e3acc6b7af192fc604be4 Mon Sep 17 00:00:00 2001 From: luxl Date: Fri, 19 Dec 2025 12:03:40 +0800 Subject: [PATCH 17/22] Clean up the code --- .../BackfillWorkflowExecutorDelegate.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java index d8095a8a2126..78d702a00736 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java @@ -288,28 +288,29 @@ private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO dependentWorkflowDefinition.getWorkerGroup()); WorkflowBackFillRequest dependentBackfillRequest = WorkflowBackFillRequest.builder() + .backfillTime(backfillTime) + .workerGroup(workerGroup) .loginUser(originalBackfillDTO.getLoginUser()) - .workflowDefinitionCode(dependentWorkflowCode) - .startNodes(null) // In backfill scenario, startNodes is null - .failureStrategy(schedule.getFailureStrategy()) - .taskDependType(TaskDependType.TASK_POST) .execType(originalBackfillDTO.getExecType()) - .warningType(schedule.getWarningType()) - .warningGroupId(dependentWorkflow.getWarningGroupId()) + .dryRun(originalBackfillDTO.getDryRun()) + .failureStrategy(schedule.getFailureStrategy()) .workflowInstancePriority(schedule.getWorkflowInstancePriority()) - .workerGroup(workerGroup) .tenantCode(schedule.getTenantCode()) .environmentCode(schedule.getEnvironmentCode()) + .warningType(schedule.getWarningType()) + .warningGroupId(dependentWorkflow.getWarningGroupId()) + .workflowDefinitionCode(dependentWorkflowCode) .startParamList(dependentWorkflow.getGlobalParams()) - .dryRun(Flag.YES) .backfillRunMode(originalParams.getRunMode()) - .backfillTime(backfillTime) .expectedParallelismNumber(originalParams.getExpectedParallelismNumber()) + .executionOrder(originalParams.getExecutionOrder()) + // In backfill scenario, startNodes is nul + .startNodes(null) + .taskDependType(TaskDependType.TASK_POST) // Disable recursive execution because dependent workflows are pre-extracted via // getAllDependentWorkflows, which also handles circular dependencies .backfillDependentMode(ComplementDependentMode.OFF_MODE) .allLevelDependent(false) - .executionOrder(originalParams.getExecutionOrder()) .build(); return backfillWorkflowRequestTransformer.transform(dependentBackfillRequest); @@ -410,7 +411,6 @@ private List getAllDependentWorkflows( * * @param upstreamBackfillDateList the set of dates actually backfilled by the upstream workflow * @param dependentWorkflowDefinition dependent workflow definition containing dependency cycle configuration - * (e.g., WEEK, MONTH, DAY, etc.) * @param upstreamWorkflowCode upstream workflow code to match the specific dependency item * @return list of downstream dates that need to be backfilled, sorted in ascending chronological order */ @@ -431,7 +431,6 @@ private List calculateDependentBackfillDates( for (ZonedDateTime upstreamBackfillDate : upstreamBackfillDateList) { Date upstreamDate = Date.from(upstreamBackfillDate.toInstant()); - // Use DependentUtils.getDateIntervalList(Date, String) to calculate dependent date intervals List dateIntervalList = DependentUtils.getDateIntervalList(upstreamDate, dateValue); if (dateIntervalList != null && !dateIntervalList.isEmpty()) { @@ -458,8 +457,6 @@ private List calculateDependentBackfillDates( } } - log.debug("Calculated {} dependent backfill dates from {} upstream dates", - dependentBackfillDateList.size(), upstreamBackfillDateList.size()); return dependentBackfillDateList; } From 2f77c881ebed9a93c1f950e06a44c76e5ccf0cf1 Mon Sep 17 00:00:00 2001 From: luxl Date: Fri, 19 Dec 2025 12:10:47 +0800 Subject: [PATCH 18/22] spotless --- .../executor/workflow/BackfillWorkflowExecutorDelegate.java | 3 +-- .../apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java index 78d702a00736..9548ea5361b6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegate.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.ExecutionOrder; import org.apache.dolphinscheduler.common.enums.FailureStrategy; -import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; @@ -305,7 +304,7 @@ private BackfillWorkflowDTO buildDependentBackfillDTO(final BackfillWorkflowDTO .expectedParallelismNumber(originalParams.getExpectedParallelismNumber()) .executionOrder(originalParams.getExecutionOrder()) // In backfill scenario, startNodes is nul - .startNodes(null) + .startNodes(null) .taskDependType(TaskDependType.TASK_POST) // Disable recursive execution because dependent workflows are pre-extracted via // getAllDependentWorkflows, which also handles circular dependencies diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index a5e965a65a7d..c58e5d8ee170 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -254,8 +254,6 @@ group by task_code ) t_max - -- Fixed: Use #{workflowInstanceId} directly instead of t_max.workflow_instance_id - -- (subquery doesn't select workflow_instance_id, only task_code and max_end_time) on instance.workflow_instance_id = #{workflowInstanceId} and instance.task_code = t_max.task_code and instance.end_time = t_max.max_end_time From 743cc4ed38a409190937cd06eec8eb7465501032 Mon Sep 17 00:00:00 2001 From: luxl Date: Fri, 19 Dec 2025 16:06:50 +0800 Subject: [PATCH 19/22] fix ut --- .../BackfillWorkflowExecutorDelegateTest.java | 67 +++---------------- 1 file changed, 11 insertions(+), 56 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java index 7385d7634a42..78ec251e36b7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -232,11 +232,13 @@ public void testGetAllDependentWorkflows_OnlyLevel1() throws Exception { when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) .thenReturn(level1List); + // Mock Level 2 dependency: B has downstream dependency D + // This is crucial to verify that the code correctly stops at Level 1 + // If the code incorrectly recurses, it would find D and the test would fail List level2List = new ArrayList<>(); level2List.add(level2D); when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowB)) .thenReturn(level2List); - when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowC)) .thenReturn(new ArrayList<>()); @@ -246,11 +248,15 @@ public void testGetAllDependentWorkflows_OnlyLevel1() throws Exception { .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, false); // Assert: Only Level 1 workflows (B and C) should be returned, Level 2 dependency D is excluded + // Even though B has downstream dependency D, D should NOT be included because allLevelDependent = false Assertions.assertNotNull(result); - Assertions.assertEquals(2, result.size()); - Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowB)); - Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowC)); - Assertions.assertFalse(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level2WorkflowD)); + Assertions.assertEquals(2, result.size(), "Should only return Level 1 dependencies (B and C)"); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowB), + "Level 1 workflow B should be included"); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level1WorkflowC), + "Level 1 workflow C should be included"); + Assertions.assertFalse(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == level2WorkflowD), + "Level 2 workflow D should be excluded when allLevelDependent = false"); } @Test @@ -353,55 +359,4 @@ private DependentWorkflowDefinition createDependentWorkflowDefinition( return definition; } - - /** - * Helper method to create DependentWorkflowDefinition with two upstream dependencies - * This simulates a downstream workflow that depends on two different upstream workflows - */ - private DependentWorkflowDefinition createDependentWorkflowDefinitionWithTwoUpstreams( - long upstream1WorkflowCode, - long upstream2WorkflowCode, - long downstreamWorkflowCode, - String cycle1, - String dateValue1, - String cycle2, - String dateValue2) { - DependentWorkflowDefinition definition = new DependentWorkflowDefinition(); - definition.setWorkflowDefinitionCode(downstreamWorkflowCode); - definition.setTaskDefinitionCode(1000L); - - // Create DependentItem for upstream1 - DependentItem dependentItem1 = new DependentItem(); - dependentItem1.setDefinitionCode(upstream1WorkflowCode); - dependentItem1.setCycle(cycle1); - dependentItem1.setDateValue(dateValue1); - - // Create DependentItem for upstream2 - DependentItem dependentItem2 = new DependentItem(); - dependentItem2.setDefinitionCode(upstream2WorkflowCode); - dependentItem2.setCycle(cycle2); - dependentItem2.setDateValue(dateValue2); - - // Create DependentTaskModel with both items - DependentTaskModel dependentTaskModel = new DependentTaskModel(); - List dependentItemList = new ArrayList<>(); - dependentItemList.add(dependentItem1); - dependentItemList.add(dependentItem2); - dependentTaskModel.setDependItemList(dependentItemList); - - // Create Dependence - DependentParameters.Dependence dependence = new DependentParameters.Dependence(); - List dependTaskList = new ArrayList<>(); - dependTaskList.add(dependentTaskModel); - dependence.setDependTaskList(dependTaskList); - - // Create DependentParameters - DependentParameters dependentParameters = new DependentParameters(); - dependentParameters.setDependence(dependence); - - // Set taskParams as JSON string - definition.setTaskParams(JSONUtils.toJsonString(dependentParameters)); - - return definition; - } } From 91ab34ea84a18d6465ff52c8a77b9e63f9e547ea Mon Sep 17 00:00:00 2001 From: luxl Date: Fri, 19 Dec 2025 16:55:39 +0800 Subject: [PATCH 20/22] fix ut --- .../workflow/BackfillWorkflowExecutorDelegateTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java index 78ec251e36b7..b7ea4b36e1d7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -39,6 +39,8 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; /** * Test for BackfillWorkflowExecutorDelegate @@ -212,6 +214,7 @@ public void testCalculateDependentBackfillDates_EmptyUpstreamList() throws Excep } @Test + @MockitoSettings(strictness = Strictness.LENIENT) public void testGetAllDependentWorkflows_OnlyLevel1() throws Exception { // Arrange: Root workflow A has Level 1 dependencies B and C, B has Level 2 dependency D long rootWorkflowCode = 100L; From 251326b9952d505a7385839ebaf00ba688c0fcff Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 22 Dec 2025 09:38:43 +0800 Subject: [PATCH 21/22] Improve unit test coverage --- .../BackfillWorkflowExecutorDelegateTest.java | 625 +++++++++++++++++- 1 file changed, 611 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java index b7ea4b36e1d7..b9aaa20b10e8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -17,20 +17,52 @@ package org.apache.dolphinscheduler.api.executor.workflow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.WorkflowLineageService; +import org.apache.dolphinscheduler.api.validator.workflow.BackfillWorkflowDTO; +import org.apache.dolphinscheduler.api.validator.workflow.BackfillWorkflowRequestTransformer; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; +import org.apache.dolphinscheduler.common.enums.ExecutionOrder; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.RunMode; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.DependentWorkflowDefinition; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; +import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionDao; +import org.apache.dolphinscheduler.extract.base.client.Clients; +import org.apache.dolphinscheduler.extract.master.IWorkflowControlClient; +import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowBackfillTriggerRequest; +import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowBackfillTriggerResponse; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; +import org.apache.dolphinscheduler.registry.api.RegistryClient; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; +import org.apache.dolphinscheduler.service.process.ProcessService; import java.lang.reflect.Method; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -38,6 +70,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -54,8 +87,26 @@ public class BackfillWorkflowExecutorDelegateTest { @Mock private WorkflowLineageService workflowLineageService; + @Mock + private RegistryClient registryClient; + + @Mock + private WorkflowDefinitionDao workflowDefinitionDao; + + @Mock + private ProcessService processService; + + @Mock + private BackfillWorkflowRequestTransformer backfillWorkflowRequestTransformer; + private Method calculateDependentBackfillDatesMethod; private Method getAllDependentWorkflowsMethod; + private Method doBackfillWorkflowMethod; + private Method doSerialBackfillWorkflowMethod; + private Method doParallelBackfillWorkflowMethod; + private Method doBackfillDependentWorkflowMethod; + private Method buildDependentBackfillDTOMethod; + private Method getDependentDateValueMethod; @BeforeEach public void setUp() throws Exception { @@ -72,6 +123,39 @@ public void setUp() throws Exception { long.class, boolean.class); getAllDependentWorkflowsMethod.setAccessible(true); + + doBackfillWorkflowMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "doBackfillWorkflow", + BackfillWorkflowDTO.class, + List.class); + doBackfillWorkflowMethod.setAccessible(true); + + doSerialBackfillWorkflowMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "doSerialBackfillWorkflow", + BackfillWorkflowDTO.class); + doSerialBackfillWorkflowMethod.setAccessible(true); + + doParallelBackfillWorkflowMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "doParallelBackfillWorkflow", + BackfillWorkflowDTO.class); + doParallelBackfillWorkflowMethod.setAccessible(true); + + doBackfillDependentWorkflowMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "doBackfillDependentWorkflow", + BackfillWorkflowDTO.class); + doBackfillDependentWorkflowMethod.setAccessible(true); + + buildDependentBackfillDTOMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "buildDependentBackfillDTO", + BackfillWorkflowDTO.class, + DependentWorkflowDefinition.class); + buildDependentBackfillDTOMethod.setAccessible(true); + + getDependentDateValueMethod = BackfillWorkflowExecutorDelegate.class.getDeclaredMethod( + "getDependentDateValue", + DependentWorkflowDefinition.class, + long.class); + getDependentDateValueMethod.setAccessible(true); } @Test @@ -90,7 +174,6 @@ public void testCalculateDependentBackfillDates_WeeklyCycle_LastMonday() throws DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "week", "lastMonday"); - // Act @SuppressWarnings("unchecked") List result = (List) calculateDependentBackfillDatesMethod.invoke( backfillWorkflowExecutorDelegate, @@ -119,7 +202,6 @@ public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Ex DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "month", "lastMonthBegin"); - // Act @SuppressWarnings("unchecked") List result = (List) calculateDependentBackfillDatesMethod.invoke( backfillWorkflowExecutorDelegate, @@ -127,7 +209,7 @@ public void testCalculateDependentBackfillDates_MonthlyCycle_NoMatch() throws Ex dependentWorkflowDefinition, 100L); - // Assert: Empty result because lastMonthBegin (2024-12-01) is not in upstream list + // Empty result because lastMonthBegin (2024-12-01) is not in upstream list Assertions.assertNotNull(result); Assertions.assertEquals(0, result.size()); } @@ -145,7 +227,6 @@ public void testCalculateDependentBackfillDates_HourlyCycle_Last1Hour() throws E DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "hour", "last1Hour"); - // Act @SuppressWarnings("unchecked") List result = (List) calculateDependentBackfillDatesMethod.invoke( backfillWorkflowExecutorDelegate, @@ -176,7 +257,6 @@ public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Ex DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "day", "last1Days"); - // Act @SuppressWarnings("unchecked") List result = (List) calculateDependentBackfillDatesMethod.invoke( backfillWorkflowExecutorDelegate, @@ -195,12 +275,10 @@ public void testCalculateDependentBackfillDates_DailyCycle_Last1Days() throws Ex @Test public void testCalculateDependentBackfillDates_EmptyUpstreamList() throws Exception { - // Arrange List upstreamBackfillDateList = new ArrayList<>(); DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( 100L, 200L, "day", "last1Days"); - // Act @SuppressWarnings("unchecked") List result = (List) calculateDependentBackfillDatesMethod.invoke( backfillWorkflowExecutorDelegate, @@ -245,7 +323,6 @@ public void testGetAllDependentWorkflows_OnlyLevel1() throws Exception { when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(level1WorkflowC)) .thenReturn(new ArrayList<>()); - // Act: Get dependencies with allLevelDependent = false @SuppressWarnings("unchecked") List result = (List) getAllDependentWorkflowsMethod .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, false); @@ -313,7 +390,6 @@ public void testGetAllDependentWorkflows_NoDependencies() throws Exception { when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) .thenReturn(new ArrayList<>()); - // Act @SuppressWarnings("unchecked") List result = (List) getAllDependentWorkflowsMethod .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, true); @@ -323,6 +399,532 @@ public void testGetAllDependentWorkflows_NoDependencies() throws Exception { Assertions.assertEquals(0, result.size()); } + @Test + public void testGetDependentDateValue_Success() throws Exception { + DependentWorkflowDefinition dependentWorkflowDefinition = createDependentWorkflowDefinition( + 100L, 200L, "day", "last1Days"); + + String result = (String) getDependentDateValueMethod.invoke( + backfillWorkflowExecutorDelegate, + dependentWorkflowDefinition, + 100L); + + Assertions.assertNotNull(result); + Assertions.assertEquals("last1Days", result); + } + + @Test + public void testCalculateDependentBackfillDates_NullDateValue() throws Exception { + DependentWorkflowDefinition dependentWorkflowDefinition = new DependentWorkflowDefinition(); + dependentWorkflowDefinition.setWorkflowDefinitionCode(200L); + dependentWorkflowDefinition.setTaskParams("{}"); + + List upstreamBackfillDateList = new ArrayList<>(); + upstreamBackfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + + @SuppressWarnings("unchecked") + List result = (List) calculateDependentBackfillDatesMethod.invoke( + backfillWorkflowExecutorDelegate, + upstreamBackfillDateList, + dependentWorkflowDefinition, + 100L); + + // Should return empty list when dateValue is null + Assertions.assertNotNull(result); + Assertions.assertEquals(0, result.size()); + } + + @Test + public void testGetAllDependentWorkflows_SelfDependency() throws Exception { + long rootWorkflowCode = 100L; + DependentWorkflowDefinition selfDependency = new DependentWorkflowDefinition(); + selfDependency.setWorkflowDefinitionCode(rootWorkflowCode); + + List level1List = new ArrayList<>(); + level1List.add(selfDependency); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(rootWorkflowCode)) + .thenReturn(level1List); + + @SuppressWarnings("unchecked") + List result = (List) getAllDependentWorkflowsMethod + .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, true); + + //Self-dependency should be filtered out + Assertions.assertNotNull(result); + Assertions.assertEquals(0, result.size()); + } + + @Test + public void testGetAllDependentWorkflows_CircularDependency() throws Exception { + long workflowA = 100L; + long workflowB = 200L; + long workflowC = 300L; + + DependentWorkflowDefinition defB = new DependentWorkflowDefinition(); + defB.setWorkflowDefinitionCode(workflowB); + DependentWorkflowDefinition defC = new DependentWorkflowDefinition(); + defC.setWorkflowDefinitionCode(workflowC); + DependentWorkflowDefinition defA = new DependentWorkflowDefinition(); + defA.setWorkflowDefinitionCode(workflowA); + + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(workflowA)) + .thenReturn(Collections.singletonList(defB)); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(workflowB)) + .thenReturn(Collections.singletonList(defC)); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(workflowC)) + .thenReturn(Collections.singletonList(defA)); + + @SuppressWarnings("unchecked") + List result = (List) getAllDependentWorkflowsMethod + .invoke(backfillWorkflowExecutorDelegate, workflowA, true); + + // Circular dependency should be handled (duplicates filtered) + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.size()); // B and C, A is filtered as self-dependency + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == workflowB)); + Assertions.assertTrue(result.stream().anyMatch(w -> w.getWorkflowDefinitionCode() == workflowC)); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoSerialBackfillWorkflow_AscendingOrder() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 3); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(1001); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + @SuppressWarnings("unchecked") + List result = (List) doSerialBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO); + + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(1001, result.get(0)); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoSerialBackfillWorkflow_DescendingOrder() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.DESC_ORDER, 3); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(1001); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + @SuppressWarnings("unchecked") + List result = (List) doSerialBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO); + + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.size()); + // Verify dates are sorted in descending order (checked via reflection of internal state) + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoParallelBackfillWorkflow() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_PARALLEL, ExecutionOrder.ASC_ORDER, 5); + backfillWorkflowDTO.getBackfillParams().setExpectedParallelismNumber(2); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response1 = WorkflowBackfillTriggerResponse.success(1001); + WorkflowBackfillTriggerResponse response2 = WorkflowBackfillTriggerResponse.success(1002); + WorkflowBackfillTriggerResponse response3 = WorkflowBackfillTriggerResponse.success(1003); + + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response1, response2, response3); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + @SuppressWarnings("unchecked") + List result = (List) doParallelBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO); + + // 5 dates with parallelism 2 = 3 partitions (2+2+1) + Assertions.assertNotNull(result); + Assertions.assertEquals(3, result.size()); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoBackfillWorkflow_Success() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + List backfillDateList = Collections.singletonList( + DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(1001); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + Integer result = (Integer) doBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO, backfillDateList); + + Assertions.assertNotNull(result); + Assertions.assertEquals(1001, result); + } + } + + @Test + public void testDoBackfillWorkflow_NoMasterServer() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + List backfillDateList = Collections.singletonList( + DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.empty()); + + Exception exception = Assertions.assertThrows(Exception.class, () -> { + doBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO, backfillDateList); + }); + Assertions.assertTrue(exception.getCause() instanceof ServiceException); + Assertions.assertTrue(exception.getCause().getMessage().contains("no master server available")); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoBackfillWorkflow_FailureResponse() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + List backfillDateList = Collections.singletonList( + DateUtils.stringToZoneDateTime("2025-01-13 00:00:00")); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.fail("Backfill failed"); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + Exception exception = Assertions.assertThrows(Exception.class, () -> { + doBackfillWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO, backfillDateList); + }); + Assertions.assertTrue(exception.getCause() instanceof ServiceException); + Assertions.assertTrue(exception.getCause().getMessage().contains("Backfill workflow failed")); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testExecute_SerialMode_WithoutDependent() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + backfillWorkflowDTO.getBackfillParams().setBackfillDependentMode(ComplementDependentMode.OFF_MODE); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(1001); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + List result = backfillWorkflowExecutorDelegate.execute(backfillWorkflowDTO); + + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals(1001, result.get(0)); + // Verify dependent workflow is not triggered + verify(workflowLineageService, never()).queryDownstreamDependentWorkflowDefinitions(anyLong()); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testExecute_ParallelMode_WithDependent() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_PARALLEL, ExecutionOrder.ASC_ORDER, 2); + backfillWorkflowDTO.getBackfillParams().setBackfillDependentMode(ComplementDependentMode.ALL_DEPENDENT); + backfillWorkflowDTO.getBackfillParams().setExpectedParallelismNumber(2); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(1001); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(anyLong())) + .thenReturn(new ArrayList<>()); + + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + List result = backfillWorkflowExecutorDelegate.execute(backfillWorkflowDTO); + + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.size()); + // Verify dependent workflow lookup was called + verify(workflowLineageService, times(1)) + .queryDownstreamDependentWorkflowDefinitions(anyLong()); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testBuildDependentBackfillDTO_Success() throws Exception { + BackfillWorkflowDTO originalDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + DependentWorkflowDefinition dependentDef = createDependentWorkflowDefinition(100L, 200L, "day", "last1Days"); + + WorkflowDefinition dependentWorkflow = new WorkflowDefinition(); + dependentWorkflow.setCode(200L); + dependentWorkflow.setReleaseState(ReleaseState.ONLINE); + dependentWorkflow.setVersion(1); + dependentWorkflow.setWarningGroupId(1); + + when(workflowDefinitionDao.queryByCode(200L)).thenReturn(Optional.of(dependentWorkflow)); + when(processService.queryReleaseSchedulerListByWorkflowDefinitionCode(200L)) + .thenReturn(new ArrayList<>()); + + BackfillWorkflowDTO transformedDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + when(backfillWorkflowRequestTransformer.transform(any())).thenReturn(transformedDTO); + + BackfillWorkflowDTO result = (BackfillWorkflowDTO) buildDependentBackfillDTOMethod.invoke( + backfillWorkflowExecutorDelegate, originalDTO, dependentDef); + + Assertions.assertNotNull(result); + verify(workflowDefinitionDao, times(1)).queryByCode(200L); + verify(processService, times(1)).queryReleaseSchedulerListByWorkflowDefinitionCode(200L); + } + + @Test + public void testBuildDependentBackfillDTO_WorkflowNotFound() throws Exception { + BackfillWorkflowDTO originalDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + DependentWorkflowDefinition dependentDef = createDependentWorkflowDefinition(100L, 200L, "day", "last1Days"); + + when(workflowDefinitionDao.queryByCode(200L)).thenReturn(Optional.empty()); + + Exception exception = Assertions.assertThrows(Exception.class, () -> { + buildDependentBackfillDTOMethod.invoke( + backfillWorkflowExecutorDelegate, originalDTO, dependentDef); + }); + Assertions.assertTrue(exception.getCause() instanceof ServiceException); + Assertions.assertTrue(exception.getCause().getMessage().contains("Dependent workflow definition not found")); + } + + @Test + public void testBuildDependentBackfillDTO_WorkflowNotOnline() throws Exception { + BackfillWorkflowDTO originalDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + DependentWorkflowDefinition dependentDef = createDependentWorkflowDefinition(100L, 200L, "day", "last1Days"); + + WorkflowDefinition dependentWorkflow = new WorkflowDefinition(); + dependentWorkflow.setCode(200L); + dependentWorkflow.setReleaseState(ReleaseState.OFFLINE); + + when(workflowDefinitionDao.queryByCode(200L)).thenReturn(Optional.of(dependentWorkflow)); + + Exception exception = Assertions.assertThrows(Exception.class, () -> { + buildDependentBackfillDTOMethod.invoke( + backfillWorkflowExecutorDelegate, originalDTO, dependentDef); + }); + Assertions.assertTrue(exception.getCause() instanceof ServiceException); + Assertions.assertTrue(exception.getCause().getMessage().contains("is not online")); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoBackfillDependentWorkflow_NoDependencies() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + backfillWorkflowDTO.getBackfillParams().setAllLevelDependent(false); + + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(anyLong())) + .thenReturn(new ArrayList<>()); + + doBackfillDependentWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO); + + // Should return without error when no dependencies + verify(workflowLineageService, times(1)) + .queryDownstreamDependentWorkflowDefinitions(anyLong()); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testDoBackfillDependentWorkflow_WithDependencies() throws Exception { + BackfillWorkflowDTO backfillWorkflowDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + backfillWorkflowDTO.getBackfillParams().setAllLevelDependent(false); + + DependentWorkflowDefinition dependentDef = createDependentWorkflowDefinition(100L, 200L, "day", "last1Days"); + when(workflowLineageService.queryDownstreamDependentWorkflowDefinitions(100L)) + .thenReturn(Collections.singletonList(dependentDef)); + + WorkflowDefinition dependentWorkflow = new WorkflowDefinition(); + dependentWorkflow.setCode(200L); + dependentWorkflow.setReleaseState(ReleaseState.ONLINE); + dependentWorkflow.setVersion(1); + dependentWorkflow.setWarningGroupId(1); + + when(workflowDefinitionDao.queryByCode(200L)).thenReturn(Optional.of(dependentWorkflow)); + when(processService.queryReleaseSchedulerListByWorkflowDefinitionCode(200L)) + .thenReturn(new ArrayList<>()); + + BackfillWorkflowDTO transformedDTO = createBackfillWorkflowDTO( + RunMode.RUN_MODE_SERIAL, ExecutionOrder.ASC_ORDER, 1); + when(backfillWorkflowRequestTransformer.transform(any())).thenReturn(transformedDTO); + + Server masterServer = new Server(); + masterServer.setHost("localhost"); + masterServer.setPort(5678); + when(registryClient.getRandomServer(RegistryNodeType.MASTER)) + .thenReturn(Optional.of(masterServer)); + + WorkflowBackfillTriggerResponse response = WorkflowBackfillTriggerResponse.success(2001); + try (MockedStatic clientsMock = org.mockito.Mockito.mockStatic(Clients.class)) { + IWorkflowControlClient client = org.mockito.Mockito.mock(IWorkflowControlClient.class); + org.mockito.Mockito.when(client.backfillTriggerWorkflow(any(WorkflowBackfillTriggerRequest.class))) + .thenReturn(response); + @SuppressWarnings("unchecked") + Clients.JdkDynamicRpcClientProxyBuilder builder = org.mockito.Mockito.mock( + Clients.JdkDynamicRpcClientProxyBuilder.class); + org.mockito.Mockito.when(builder.withHost(anyString())).thenReturn(client); + clientsMock.when(() -> Clients.withService(IWorkflowControlClient.class)).thenReturn(builder); + + doBackfillDependentWorkflowMethod.invoke( + backfillWorkflowExecutorDelegate, backfillWorkflowDTO); + + // Should process dependent workflow + verify(workflowDefinitionDao, times(1)).queryByCode(200L); + } + } + + /** + * Helper method to create BackfillWorkflowDTO + */ + private BackfillWorkflowDTO createBackfillWorkflowDTO(RunMode runMode, ExecutionOrder executionOrder, + int dateCount) { + User user = new User(); + user.setId(1); + + WorkflowDefinition workflowDefinition = new WorkflowDefinition(); + workflowDefinition.setCode(100L); + workflowDefinition.setVersion(1); + workflowDefinition.setReleaseState(ReleaseState.ONLINE); + + List backfillDateList = new ArrayList<>(); + for (int i = 0; i < dateCount; i++) { + backfillDateList.add(DateUtils.stringToZoneDateTime("2025-01-" + (13 + i) + " 00:00:00")); + } + + BackfillWorkflowDTO.BackfillParamsDTO backfillParams = BackfillWorkflowDTO.BackfillParamsDTO.builder() + .runMode(runMode) + .backfillDateList(backfillDateList) + .expectedParallelismNumber(null) + .backfillDependentMode(ComplementDependentMode.OFF_MODE) + .allLevelDependent(false) + .executionOrder(executionOrder) + .build(); + + return BackfillWorkflowDTO.builder() + .loginUser(user) + .workflowDefinition(workflowDefinition) + .startNodes(null) + .failureStrategy(FailureStrategy.CONTINUE) + .taskDependType(TaskDependType.TASK_POST) + .execType(CommandType.COMPLEMENT_DATA) + .warningType(WarningType.NONE) + .warningGroupId(null) + .runMode(runMode) + .workflowInstancePriority(Priority.MEDIUM) + .workerGroup("default") + .tenantCode("default") + .environmentCode(null) + .startParamList(null) + .dryRun(Flag.NO) + .backfillParams(backfillParams) + .build(); + } + /** * Helper method to create DependentWorkflowDefinition with taskParams */ @@ -335,29 +937,24 @@ private DependentWorkflowDefinition createDependentWorkflowDefinition( definition.setWorkflowDefinitionCode(downstreamWorkflowCode); definition.setTaskDefinitionCode(1000L); - // Create DependentItem DependentItem dependentItem = new DependentItem(); dependentItem.setDefinitionCode(upstreamWorkflowCode); dependentItem.setCycle(cycle); dependentItem.setDateValue(dateValue); - // Create DependentTaskModel DependentTaskModel dependentTaskModel = new DependentTaskModel(); List dependentItemList = new ArrayList<>(); dependentItemList.add(dependentItem); dependentTaskModel.setDependItemList(dependentItemList); - // Create Dependence DependentParameters.Dependence dependence = new DependentParameters.Dependence(); List dependTaskList = new ArrayList<>(); dependTaskList.add(dependentTaskModel); dependence.setDependTaskList(dependTaskList); - // Create DependentParameters DependentParameters dependentParameters = new DependentParameters(); dependentParameters.setDependence(dependence); - // Set taskParams as JSON string definition.setTaskParams(JSONUtils.toJsonString(dependentParameters)); return definition; From 026a830b9829845d2d0e8980c91265d10b8f9c2a Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 22 Dec 2025 10:12:53 +0800 Subject: [PATCH 22/22] spotless --- .../executor/workflow/BackfillWorkflowExecutorDelegateTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java index b9aaa20b10e8..7d89562939c1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/BackfillWorkflowExecutorDelegateTest.java @@ -449,7 +449,7 @@ public void testGetAllDependentWorkflows_SelfDependency() throws Exception { List result = (List) getAllDependentWorkflowsMethod .invoke(backfillWorkflowExecutorDelegate, rootWorkflowCode, true); - //Self-dependency should be filtered out + // Self-dependency should be filtered out Assertions.assertNotNull(result); Assertions.assertEquals(0, result.size()); }