diff --git a/config/plugins_config b/config/plugins_config index 9297d967e489..670ab7158995 100644 --- a/config/plugins_config +++ b/config/plugins_config @@ -107,4 +107,5 @@ dolphinscheduler-task-spark dolphinscheduler-task-sql dolphinscheduler-task-sqoop dolphinscheduler-task-zeppelin +dolphinscheduler-task-external-system --end-- diff --git a/docs/docs/en/guide/task/external-system.md b/docs/docs/en/guide/task/external-system.md new file mode 100644 index 000000000000..2c82d8b3d054 --- /dev/null +++ b/docs/docs/en/guide/task/external-system.md @@ -0,0 +1,24 @@ +# External System Task + +The External System task is used to call external system APIs and integrate with third-party systems. This task type allows DolphinScheduler to interact with external services, execute remote operations, and retrieve results. + +## Introduction + +External System tasks provide a way to execute operations in external systems through REST APIs calls. These tasks can be configured to authenticate with external systems, send requests, and process responses. + +## Configuration + +- **Task Name**: The name of the task. +- **Task Type**: Select "External System" from the task type list. +- **External System**: Select the external system to connect to. +- **API Endpoint**: The API endpoint to call in the external system. +- **Authentication**: Configure authentication parameters for the external system. +- **Request Method**: The HTTP method to use (GET, POST, PUT, DELETE, etc.). +- **Request Parameters**: Parameters to send with the request. +- **Response Processing**: How to process the response from the external system. + +## Example + +![External System Task](../../../../img/tasks/icons/external_system.png) + +This is an example of how to configure an External System task in the workflow. diff --git a/docs/docs/zh/guide/task/external-system.md b/docs/docs/zh/guide/task/external-system.md new file mode 100644 index 000000000000..8c63b808f1fd --- /dev/null +++ b/docs/docs/zh/guide/task/external-system.md @@ -0,0 +1,24 @@ +# 外部系统任务 + +外部系统任务用于调用外部系统API并与第三方系统集成。这种任务类型允许DolphinScheduler与外部服务交互，执行远程操作并检索结果。 + +## 介绍 + +外部系统任务提供了一种通过REST APIs调用与外部系统交互的方式。这些任务可以配置为对第三方系统进行身份验证，发送请求并处理响应。 + +## 配置 + +- **任务名称**: 任务的名称。 +- **任务类型**: 从任务类型列表中选择"外部系统"。 +- **外部系统**: 选择要连接的外部系统。 +- **API端点**: 在外部系统中调用的API端点。 +- **认证**: 配置外部系统的认证参数。 +- **请求方法**: 要使用的HTTP方法（GET、POST、PUT、DELETE等）。 +- **请求参数**: 与请求一起发送的参数。 +- **响应处理**: 如何处理来自外部系统的响应。 + +## 示例 + +![外部系统任务](../../../../img/tasks/icons/external_system.png) + +这是在工作流程中配置外部系统任务的示例。 diff --git a/docs/img/tasks/icons/external_system.png b/docs/img/tasks/icons/external_system.png new file mode 100644 index 000000000000..a564ebcc33a4 Binary files /dev/null and b/docs/img/tasks/icons/external_system.png differ diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index c43dfc4dbf30..1845abafcd58 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -255,6 +255,11 @@ azure-resourcemanager-datafactory + + com.jayway.jsonpath + json-path + + com.azure azure-identity diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExternalSystemController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExternalSystemController.java new file mode 100644 index 000000000000..f60a6d89c9cd --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExternalSystemController.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_EXTERNAL_SYSTEM_ERROR; + +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.ExternalSystemService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.ExternalSystemTaskQuery; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +@Tag(name = "EXTERNAL_SYSTEM_TAG") +@RestController +@RequestMapping("external-systems") +public class ExternalSystemController extends BaseController { + + @Autowired + private ExternalSystemService externalSystemService; + + @Operation(summary = "queryExternalSystemTasks", description = "QUERY_EXTERNAL_SYSTEM_TASKS_NOTES") + @GetMapping(value = "/queryExternalSystemTasks") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_EXTERNAL_SYSTEM_ERROR) + public Result> queryExternalSystemTasks(@RequestParam("externalSystemId") Integer externalSystemId) { + List result = + externalSystemService.queryExternalSystemTasks(externalSystemId); + return Result.success(result); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 3d24834bd429..6013683874ba 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -73,6 +73,8 @@ public enum Status { VERIFY_DATASOURCE_NAME_FAILURE(10039, "verify datasource name failure", "验证数据源名称失败"), UNAUTHORIZED_DATASOURCE(10040, "unauthorized datasource", "未经授权的数据源"), AUTHORIZED_DATA_SOURCE(10041, "authorized data source", "授权数据源失败"), + QUERY_EXTERNAL_SYSTEM_ERROR(11044, "query external system error", "查询第三方系统错误"), + EXTERNAL_SYSTEM_CONNECT_AUTH_FAILED(11049, "external connect failed", "第三方系统连接失败,检查认证信息"), LOGIN_SUCCESS(10042, "login success", "登录成功"), USER_LOGIN_FAILURE(10043, "user login failure", "用户登录失败"), LIST_WORKERS_ERROR(10044, "list workers error", "查询worker列表错误"), @@ -120,6 +122,7 @@ public enum Status { GRANT_PROJECT_ERROR(10094, "grant project error", "授权项目错误"), GRANT_RESOURCE_ERROR(10095, "grant resource error", "授权资源错误"), GRANT_DATASOURCE_ERROR(10097, "grant datasource error", "授权数据源错误"), + GRANT_EXTERNALSYSTEM_ERROR(100971, "grant external system error", "授权第三方系统错误"), GET_USER_INFO_ERROR(10098, "get user info error", "获取用户信息错误"), USER_LIST_ERROR(10099, "user list error", "查询用户列表错误"), VERIFY_USERNAME_ERROR(10100, "verify username error", "用户名验证错误"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExternalSystemService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExternalSystemService.java new file mode 100644 index 000000000000..a7f571d7f6ec --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExternalSystemService.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.dao.entity.ExternalSystemTaskQuery; + +import java.util.List; + +public interface ExternalSystemService { + + List queryExternalSystemTasks(int externalSystemId); + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExternalSystemServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExternalSystemServiceImpl.java new file mode 100644 index 000000000000..6c130cf23fa3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExternalSystemServiceImpl.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.service.ExternalSystemService; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType; +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders; +import org.apache.dolphinscheduler.common.model.OkHttpResponse; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OkHttpUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ExternalSystemTaskQuery; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.AuthenticationUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.AuthConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.InterfaceInfo; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.RequestParameter; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ResponseParameter; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ThirdPartySystemConnectorConnectionParam; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.externalSystem.ExternalTaskConstants; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.FormBody; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.jayway.jsonpath.JsonPath; + +@Service +@Slf4j +public class ExternalSystemServiceImpl extends BaseServiceImpl implements ExternalSystemService { + + @Autowired + private DataSourceMapper dataSourceMapper; + + private static final String EXTERNAL_TASK_ID = "id"; + private static final String EXTERNAL_TASK_NAME = "name"; + + private OkHttpResponse callSelectInterface(ThirdPartySystemConnectorConnectionParam baseExternalSystemParam, + boolean dbPassword) { + if (baseExternalSystemParam == null || baseExternalSystemParam.getSelectInterface() == null) { + throw new IllegalArgumentException( + "ThirdPartySystemConnectorConnectionParam or SelectInterface cannot be null"); + } + + InterfaceInfo selectConfig = baseExternalSystemParam.getSelectInterface(); + + // Replace parameter placeholders + String url = baseExternalSystemParam.getCompleteUrl(selectConfig.getUrl()); + + Map headeMap = new HashMap<>(); + Map requestBody = new HashMap<>(); + Map requestParams = new HashMap<>(); + String token; + + try { + if (dbPassword) { + // Saved information, retrieve from database and decrypt + AuthConfig authConfig = baseExternalSystemParam.getAuthConfig(); + decodePassword(authConfig); + baseExternalSystemParam.setAuthConfig(authConfig); + token = AuthenticationUtils.authenticateAndGetToken(baseExternalSystemParam); + } else { + if (baseExternalSystemParam.getId() != null) { + DataSource existingSystem = dataSourceMapper.selectById(baseExternalSystemParam.getId()); + if (existingSystem == null) { + // New information test connection + token = AuthenticationUtils.authenticateAndGetToken(baseExternalSystemParam); + } else { + // Update information test connection, if password is not modified, use password saved in + // database for test connection + ThirdPartySystemConnectorConnectionParam oldParams = + JSONUtils.parseObject(existingSystem.getConnectionParams(), + ThirdPartySystemConnectorConnectionParam.class); + AuthConfig authConfig = baseExternalSystemParam.getAuthConfig(); + if (authConfig.getBasicPassword() != null + && authConfig.getBasicPassword().equals(Constants.XXXXXX)) { + authConfig.setBasicPassword(oldParams.getAuthConfig().getBasicPassword()); + } + if (authConfig.getOauth2ClientSecret() != null + && authConfig.getOauth2ClientSecret().equals(Constants.XXXXXX)) { + authConfig.setOauth2ClientSecret(oldParams.getAuthConfig().getOauth2ClientSecret()); + } + if (authConfig.getOauth2Password() != null + && authConfig.getOauth2Password().equals(Constants.XXXXXX)) { + authConfig.setOauth2Password(oldParams.getAuthConfig().getOauth2Password()); + } + if (authConfig.getJwtToken() != null && authConfig.getJwtToken().equals(Constants.XXXXXX)) { + authConfig.setJwtToken(oldParams.getAuthConfig().getJwtToken()); + } + decodePassword(authConfig); + baseExternalSystemParam.setAuthConfig(authConfig); + token = AuthenticationUtils.authenticateAndGetToken(baseExternalSystemParam); + } + } else { + token = AuthenticationUtils.authenticateAndGetToken(baseExternalSystemParam); + } + } + } catch (Exception e) { + log.error("Authentication failed: {}", e.getMessage()); + throw new ServiceException(Status.EXTERNAL_SYSTEM_CONNECT_AUTH_FAILED); + } + + try { + headeMap.put("Authorization", + baseExternalSystemParam.getTokenPrefix(baseExternalSystemParam.getAuthConfig().getHeaderPrefix()) + + token); + + // Process parameters + for (RequestParameter param : selectConfig.getParameters()) { + // todo String value = replaceParameterPlaceholders(param.getParamValue()); + String value = param.getParamValue(); + + switch (param.getLocation().name()) { + case "HEADER": + headeMap.put(param.getParamName(), value); + break; + case "PARAM": + requestParams.put(param.getParamName(), value); + break; + } + } + if (selectConfig.getBody() != null) { + requestBody = JSONUtils.parseObject((selectConfig.getBody()), Map.class); + } + OkHttpRequestHeaders headers = new OkHttpRequestHeaders(); + OkHttpRequestHeaderContentType contentType = getContentType(headeMap); + + headers.setOkHttpRequestHeaderContentType(contentType); + if (!headeMap.isEmpty()) { + headers.setHeaders(headeMap); + } + OkHttpResponse response; + if (InterfaceInfo.HttpMethod.POST.equals(selectConfig.getMethod())) { + if (contentType.equals(OkHttpRequestHeaderContentType.APPLICATION_JSON)) { + response = OkHttpUtils.post(url, headers, requestParams, requestBody, 120000, 120000, 120000); + + } else if (contentType.equals(OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED)) { + FormBody.Builder formBodyBuilder = new FormBody.Builder(); + if (requestBody != null) { + for (Map.Entry entry : requestBody.entrySet()) { + formBodyBuilder.add(entry.getKey(), entry.getValue().toString()); + } + } + response = OkHttpUtils.postFormBody(url, headers, requestParams, formBodyBuilder.build(), 120000, + 120000, 120000); + } else { + log.error("select task failed, OkHttpRequestHeaderContentType not support: {},", contentType); + throw new ServiceException(Status.EXTERNAL_SYSTEM_CONNECT_AUTH_FAILED); + } + } else if (InterfaceInfo.HttpMethod.PUT.equals(selectConfig.getMethod())) { + response = OkHttpUtils.put(url, headers, requestBody, 120000, 120000, 120000); + } else { + response = OkHttpUtils.get(url, headers, requestParams, 120000, 120000, 120000); + } + return response; + + } catch (Exception e) { + log.error("select task failed, id: {}, serviceAddress: {}, dbPassword: {}", + baseExternalSystemParam.getId(), + baseExternalSystemParam.getServiceAddress(), + dbPassword, e); + throw new ServiceException(Status.EXTERNAL_SYSTEM_CONNECT_AUTH_FAILED); + } + } + + private OkHttpRequestHeaderContentType getContentType(Map headers) { + String contentType = headers.get("Content-Type"); + if (contentType != null) { + if (contentType.contains("application/json")) { + return OkHttpRequestHeaderContentType.APPLICATION_JSON; + } else if (contentType.contains("application/x-www-form-urlencoded")) { + return OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED; + } + } + return OkHttpRequestHeaderContentType.APPLICATION_JSON; // 默认值 + } + + private void decodePassword(AuthConfig authConfig) { + if (null != authConfig.getOauth2ClientSecret() && !authConfig.getOauth2ClientSecret().isEmpty()) { + authConfig.setOauth2ClientSecret(PasswordUtils.decodePassword(authConfig.getOauth2ClientSecret())); + } + if (null != authConfig.getOauth2Password() && !authConfig.getOauth2Password().isEmpty()) { + authConfig.setOauth2Password(PasswordUtils.decodePassword(authConfig.getOauth2Password())); + } + if (null != authConfig.getJwtToken() && !authConfig.getJwtToken().isEmpty()) { + authConfig.setJwtToken(PasswordUtils.decodePassword(authConfig.getJwtToken())); + } + if (null != authConfig.getBasicPassword() && !authConfig.getBasicPassword().isEmpty()) { + authConfig.setBasicPassword(PasswordUtils.decodePassword(authConfig.getBasicPassword())); + } + } + + /** + * get hidden password (resolve the security hotspot) + * + * @return hidden password + */ + private String getHiddenPassword() { + return Constants.XXXXXX; + } + + @Override + public List queryExternalSystemTasks(int externalSystemId) { + + DataSource dataSource = dataSourceMapper.selectById(externalSystemId); + ThirdPartySystemConnectorConnectionParam baseExternalSystemParam = + JSONUtils.parseObject(dataSource.getConnectionParams(), ThirdPartySystemConnectorConnectionParam.class); + + // Validate query parameters + String taskIdExpression = ""; + String taskNameExpression = ""; + for (ResponseParameter param : baseExternalSystemParam.getSelectInterface() + .getResponseParameters()) { + if (EXTERNAL_TASK_ID.equals(param.getKey())) { + taskIdExpression = param.getJsonPath(); + } + if (EXTERNAL_TASK_NAME.equals(param.getKey())) { + taskNameExpression = param.getJsonPath(); + } + } + if (taskIdExpression.isEmpty() || taskNameExpression.isEmpty()) { + throw new IllegalStateException("External field mapping for 'id' and 'name' not found"); + } + + OkHttpResponse selectResponse = callSelectInterface(baseExternalSystemParam, true); + if (selectResponse.getStatusCode() != ExternalTaskConstants.RESPONSE_CODE_SUCCESS) { + throw new TaskException("Select task failed: " + selectResponse.getBody()); + } + // 解析响应获取id name + return parseSelectResponse(selectResponse.getBody(), taskIdExpression, taskNameExpression); + + } + + private List parseSelectResponse(String responseBody, String taskIdExpression, + String taskNameExpression) throws TaskException { + List resultList = new ArrayList<>(); + + try { + + List idValues = JsonPath.read(responseBody, taskIdExpression); + List nameValues = JsonPath.read(responseBody, taskNameExpression); + + if (idValues.size() != nameValues.size()) { + throw new TaskException("ID and name lists have different sizes"); + } + + // Create tasks + for (int i = 0; i < idValues.size(); i++) { + ExternalSystemTaskQuery task = new ExternalSystemTaskQuery(); + task.setId(idValues.get(i)); + task.setName(nameValues.get(i)); + resultList.add(task); + } + + } catch (Exception e) { + log.error("Parse select response failed", e); + throw new TaskException("Parse select response failed", e); + } + return resultList; + } + +} diff --git a/dolphinscheduler-api/src/main/resources/task-type-config.yaml b/dolphinscheduler-api/src/main/resources/task-type-config.yaml index 51bbae7e9abd..a9b6f627a01c 100644 --- a/dolphinscheduler-api/src/main/resources/task-type-config.yaml +++ b/dolphinscheduler-api/src/main/resources/task-type-config.yaml @@ -31,6 +31,7 @@ task: - 'FLINK_STREAM' - 'HIVECLI' - 'REMOTESHELL' + - 'EXTERNAL_SYSTEM' cloud: - 'EMR' - 'K8S' diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 3c863e120a4a..cb11bba23da1 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -124,6 +124,7 @@ 0.10.1 2.1.4 0.3.2 + 0.11.5 2.0.1 3.17.3 1.3 @@ -1038,6 +1039,23 @@ jdbc ${dolphindb-jdbc.version} + + + io.jsonwebtoken + jjwt-api + ${io-jsonwebtoken.version} + + + io.jsonwebtoken + jjwt-impl + ${io-jsonwebtoken.version} + + + io.jsonwebtoken + jjwt-jackson + ${io-jsonwebtoken.version} + + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 03742f875f77..74f2653d78cd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -212,6 +212,11 @@ public final class Constants { public static final String DEFAULT = "default"; public static final String PASSWORD = "password"; + public static final String BASICPASSWORD = "basicPassword"; + public static final String OAUTH2CLIENTSECRET = "oauth2ClientSecret"; + public static final String JWTTOKEN = "jwtToken"; + public static final String OAUTH2PASSWORD = "oauth2Password"; + public static final String XXXXXX = "******"; public static final String NULL = "NULL"; public static final String THREAD_NAME_MASTER_SERVER = "Master-Server"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java index d942901974b2..bd2ae77ff6d0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java @@ -99,6 +99,34 @@ public class OkHttpUtils { } } + public static @NonNull OkHttpResponse postFormBody(@NonNull String url, + @Nullable OkHttpRequestHeaders okHttpRequestHeaders, + @Nullable Map requestParamsMap, + @Nullable RequestBody formBody, + int connectTimeout, + int writeTimeout, + int readTimeout) throws IOException { + OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout); + String finalUrl = addUrlParams(requestParamsMap, url); + Request.Builder requestBuilder = new Request.Builder().url(finalUrl); + + if (okHttpRequestHeaders != null) { + addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder); + } + + RequestBody safeFormBody = formBody != null ? formBody + : RequestBody.create("", + MediaType.parse(OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED.getValue())); + Request request = requestBuilder + .post(safeFormBody) + .build(); + try (Response response = client.newCall(request).execute()) { + return new OkHttpResponse(response.code(), getResponseBody(response)); + } catch (Exception e) { + throw new RuntimeException(String.format("Post request execute failed, url: %s", url), e); + } + } + /** * http put request * @param connectTimeout connect timeout in milliseconds diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExternalSystemTaskQuery.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExternalSystemTaskQuery.java new file mode 100644 index 000000000000..c607d57a16a1 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExternalSystemTaskQuery.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import lombok.Data; + +@Data +public class ExternalSystemTaskQuery { + + private String id; + + private String name; + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index 7da30a85affb..fd31e93c3246 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -168,5 +168,10 @@ dolphinscheduler-datasource-dolphindb ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-thirdpartysystemconnector + ${project.version} + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/pom.xml new file mode 100644 index 000000000000..c93f7775b6ac --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/pom.xml @@ -0,0 +1,71 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin + dev-SNAPSHOT + + + dolphinscheduler-datasource-thirdpartysystemconnector + jar + ${project.artifactId} + + + thirdpartysystemconnector + + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-common + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/AuthenticationUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/AuthenticationUtils.java new file mode 100644 index 000000000000..ea9feeb604cb --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/AuthenticationUtils.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType; +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders; +import org.apache.dolphinscheduler.common.model.OkHttpResponse; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OkHttpUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.AuthConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.AuthMapping; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ThirdPartySystemConnectorConnectionParam; + +import java.util.HashMap; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.FormBody; +import okhttp3.RequestBody; + +import com.fasterxml.jackson.databind.JsonNode; + +@Slf4j +public class AuthenticationUtils { + + /** + * Authenticate and get Token + * + * @param thirdPartySystemConnectorConnectionParam configuration + * @return Authenticated Token + * @throws Exception + */ + public static String authenticateAndGetToken(ThirdPartySystemConnectorConnectionParam thirdPartySystemConnectorConnectionParam) throws Exception { + AuthConfig authConfig = thirdPartySystemConnectorConnectionParam.getAuthConfig(); + if (authConfig == null) { + throw new RuntimeException("AuthConfig is not provided"); + } + + switch (authConfig.getAuthType()) { + case BASIC_AUTH: + // Basic authentication + String auth = authConfig.getBasicUsername() + ":" + authConfig.getBasicPassword(); + String encoding = java.util.Base64.getEncoder().encodeToString(auth.getBytes()); + return encoding; + case JWT: + // JWT authentication + return authConfig.getJwtToken(); + case OAUTH2: + // OAuth2 authentication + return getOAuth2Token(thirdPartySystemConnectorConnectionParam); + default: + throw new RuntimeException("Unsupported auth type: " + authConfig.getAuthType()); + } + } + + /** + * Get OAuth2 Token + * + * @param thirdPartySystemConnectorConnectionParam Authentication configuration + * @return OAuth2 Token + * @throws Exception + */ + private static String getOAuth2Token(ThirdPartySystemConnectorConnectionParam thirdPartySystemConnectorConnectionParam) throws Exception { + AuthConfig authConfig = thirdPartySystemConnectorConnectionParam.getAuthConfig(); + try { + OkHttpRequestHeaders headers = new OkHttpRequestHeaders(); + headers.setHeaders(new HashMap<>()); + headers.setOkHttpRequestHeaderContentType(OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED); + + FormBody.Builder formBodyBuilder = new FormBody.Builder() + .add("client_id", authConfig.getOauth2ClientId()) + .add("client_secret", authConfig.getOauth2ClientSecret()) + .add("username", authConfig.getOauth2Username()) + .add("password", authConfig.getOauth2Password()) + .add("grant_type", authConfig.getOauth2GrantType()); + + // Add parameters from authMappings + if (authConfig.getAuthMappings() != null) { + for (AuthMapping authMapping : authConfig.getAuthMappings()) { + formBodyBuilder.add(authMapping.getKey(), authMapping.getValue()); + } + } + + RequestBody formBody = formBodyBuilder.build(); + + OkHttpResponse response = OkHttpUtils.postFormBody( + thirdPartySystemConnectorConnectionParam.getCompleteUrl(authConfig.getOauth2TokenUrl()), + headers, + null, + formBody, + 30000, 30000, 30000); + + if (response.getStatusCode() != 200) { + throw new RuntimeException("Authentication failed: " + response.getBody()); + } + + JsonNode authResult = JSONUtils.parseObject(response.getBody(), JsonNode.class); + if (authResult.has("access_token")) { + log.info("Authentication successful, token obtained"); + return authResult.get("access_token").asText(); + } else { + throw new RuntimeException("Failed to get access token from response"); + } + + } catch (Exception e) { + log.error("Authentication failed", e); + throw new RuntimeException("Authentication failed", e); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorAdHocDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorAdHocDataSourceClient.java new file mode 100644 index 000000000000..77e07bfb03f9 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorAdHocDataSourceClient.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.BaseAdHocDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ThirdPartySystemConnectorAdHocDataSourceClient extends BaseAdHocDataSourceClient { + + public ThirdPartySystemConnectorAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorClientWrapper.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorClientWrapper.java new file mode 100644 index 000000000000..7e5a68103116 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorClientWrapper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.AuthConfig; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ThirdPartySystemConnectorClientWrapper implements AutoCloseable { + + private final String serviceAddress; + private final AuthConfig authConfig; + + public ThirdPartySystemConnectorClientWrapper(String serviceAddress, AuthConfig authConfig) { + this.serviceAddress = serviceAddress; + this.authConfig = authConfig; + } + + public boolean checkConnect() { + log.info("Checking connectivity to third party system at: {}", serviceAddress); + return true; + } + + @Override + public void close() { + log.info("Closing third party system connector client"); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannel.java new file mode 100644 index 000000000000..a7089c2f23dd --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannel.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.PooledDataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class ThirdPartySystemConnectorDataSourceChannel implements DataSourceChannel { + + @Override + public AdHocDataSourceClient createAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("ThirdPartySystemConnector AdHocDataSourceClient is not supported"); + } + + @Override + public PooledDataSourceClient createPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("ThirdPartySystemConnector PooledDataSourceClient is not supported"); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannelFactory.java new file mode 100644 index 000000000000..9eba1357ba66 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorDataSourceChannelFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceChannelFactory.class) +public class ThirdPartySystemConnectorDataSourceChannelFactory implements DataSourceChannelFactory { + + @Override + public String getName() { + return DbType.THIRDPARTY_SYSTEM_CONNECTOR.getName(); + } + + @Override + public DataSourceChannel create() { + return new ThirdPartySystemConnectorDataSourceChannel(); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorPooledDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorPooledDataSourceClient.java new file mode 100644 index 000000000000..8d473a7f7e7d --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/ThirdPartySystemConnectorPooledDataSourceClient.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.BasePooledDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ThirdPartySystemConnectorPooledDataSourceClient extends BasePooledDataSourceClient { + + public ThirdPartySystemConnectorPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthConfig.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthConfig.java new file mode 100644 index 000000000000..b3d8e0c6d591 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import java.util.List; + +import lombok.Data; + +@Data +public class AuthConfig { + + private AuthType authType; // authType：BASIC, JWT, OAUTH2 + private String headerPrefix; // headerPrefix：Basic, Bearer + + // === （Basic Auth） === + private String basicUsername; + private String basicPassword; + + // === JWT === + private String jwtToken; + + // === OAuth2 === + private String oauth2TokenUrl; + private String oauth2ClientId; + private String oauth2ClientSecret; + private String oauth2GrantType; // e.g., "client_credentials", "password" + private String oauth2Username; // only password mode + private String oauth2Password; // only password mode + + // === other authMappings === + private List authMappings; + + public enum AuthType { + BASIC_AUTH, JWT, OAUTH2 + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthMapping.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthMapping.java new file mode 100644 index 000000000000..c8c5a3a49a6e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/AuthMapping.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; + +@Data +public class AuthMapping { + + private String key; + private String value; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/InterfaceInfo.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/InterfaceInfo.java new file mode 100644 index 000000000000..a0d679f607ea --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/InterfaceInfo.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import java.util.List; + +import lombok.Data; + +@Data +public class InterfaceInfo { + + private String url; + private HttpMethod method; + private String body; + private List parameters; + private List responseParameters; + + public enum HttpMethod { + GET, POST, PUT + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingFailureConfig.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingFailureConfig.java new file mode 100644 index 000000000000..ad7016ae888b --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingFailureConfig.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; + +@Data +public class PollingFailureConfig { + + private String failureField; + private String failureValue; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingInterfaceInfo.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingInterfaceInfo.java new file mode 100644 index 000000000000..c83bcf9635c6 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingInterfaceInfo.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class PollingInterfaceInfo extends InterfaceInfo { + + private PollingSuccessConfig pollingSuccessConfig; + private PollingFailureConfig pollingFailureConfig; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingSuccessConfig.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingSuccessConfig.java new file mode 100644 index 000000000000..11f7874785b3 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/PollingSuccessConfig.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; + +@Data +public class PollingSuccessConfig { + + private String successField; + private String successValue; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/RequestParameter.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/RequestParameter.java new file mode 100644 index 000000000000..e7ad0c0730fd --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/RequestParameter.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; + +@Data +public class RequestParameter { + + private String paramName; + private String paramValue; + private ParamLocation location; // header,param,body + + public enum ParamLocation { + HEADER, PARAM, BODY + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ResponseParameter.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ResponseParameter.java new file mode 100644 index 000000000000..939cda72b973 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ResponseParameter.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import lombok.Data; + +@Data +public class ResponseParameter { + + private String key; + private String jsonPath; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorConnectionParam.java new file mode 100644 index 000000000000..b8e3728e0a41 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorConnectionParam.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j +public class ThirdPartySystemConnectorConnectionParam implements ConnectionParam { + + private Integer id; // System ID + + private String serviceAddress; + + private AuthConfig authConfig; + + private InterfaceInfo selectInterface; + private InterfaceInfo submitInterface; + private PollingInterfaceInfo pollStatusInterface; + private InterfaceInfo stopInterface; + + private int interfaceTimeout = 120000; + + public String getCompleteUrl(String url) { + if (url == null || !url.startsWith("http")) { + if (serviceAddress == null) { + log.warn("Service address is not set."); + return url; + } + return serviceAddress + url; + } + return url; + } + + public String getTokenPrefix(String headerPrefix) { + if (null == headerPrefix || headerPrefix.isEmpty()) { + return ""; + } else { + return headerPrefix.trim() + " "; + } + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceParamDTO.java new file mode 100644 index 000000000000..d10f12572f19 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceParamDTO.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ThirdPartySystemConnectorDataSourceParamDTO extends BaseDataSourceParamDTO { + + private String serviceAddress; + + private AuthConfig authConfig; + + private InterfaceInfo selectInterface; + private InterfaceInfo submitInterface; + private PollingInterfaceInfo pollStatusInterface; + private InterfaceInfo stopInterface; + + private int interfaceTimeout = 120000; + + @Override + public DbType getType() { + return DbType.THIRDPARTY_SYSTEM_CONNECTOR; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceProcessor.java new file mode 100644 index 000000000000..231d723bb481 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/src/main/java/org/apache/dolphinscheduler/plugin/datasource/thirdpartysystemconnector/param/ThirdPartySystemConnectorDataSourceProcessor.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.AuthenticationUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.text.MessageFormat; + +import lombok.extern.slf4j.Slf4j; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceProcessor.class) +@Slf4j +public class ThirdPartySystemConnectorDataSourceProcessor extends AbstractDataSourceProcessor { + + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, ThirdPartySystemConnectorDataSourceParamDTO.class); + } + + @Override + public void checkDatasourceParam(BaseDataSourceParamDTO datasourceParamDTO) { + ThirdPartySystemConnectorDataSourceParamDTO thirdPartySystemConnectorParamDTO = + (ThirdPartySystemConnectorDataSourceParamDTO) datasourceParamDTO; + + if (StringUtils.isEmpty(thirdPartySystemConnectorParamDTO.getServiceAddress())) { + throw new IllegalArgumentException("third party system connector datasource param is not valid"); + } + checkExternalSystemParam(thirdPartySystemConnectorParamDTO); + } + + @Override + public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { + ThirdPartySystemConnectorConnectionParam baseConnectionParam = + (ThirdPartySystemConnectorConnectionParam) connectionParam; + return MessageFormat.format("{0}@{1}", dbType.getName(), baseConnectionParam.getServiceAddress()); + } + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + ThirdPartySystemConnectorConnectionParam connectionParams = + (ThirdPartySystemConnectorConnectionParam) createConnectionParams(connectionJson); + + ThirdPartySystemConnectorDataSourceParamDTO thirdPartySystemConnectorDataSourceParamDTO = + new ThirdPartySystemConnectorDataSourceParamDTO(); + + thirdPartySystemConnectorDataSourceParamDTO.setServiceAddress(connectionParams.getServiceAddress()); + thirdPartySystemConnectorDataSourceParamDTO.setAuthConfig(connectionParams.getAuthConfig()); + thirdPartySystemConnectorDataSourceParamDTO.setSelectInterface(connectionParams.getSelectInterface()); + thirdPartySystemConnectorDataSourceParamDTO.setSubmitInterface(connectionParams.getSubmitInterface()); + thirdPartySystemConnectorDataSourceParamDTO.setPollStatusInterface(connectionParams.getPollStatusInterface()); + thirdPartySystemConnectorDataSourceParamDTO.setStopInterface(connectionParams.getStopInterface()); + thirdPartySystemConnectorDataSourceParamDTO.setInterfaceTimeout(connectionParams.getInterfaceTimeout()); + + return thirdPartySystemConnectorDataSourceParamDTO; + } + + @Override + public ThirdPartySystemConnectorConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + ThirdPartySystemConnectorDataSourceParamDTO thirdPartySystemConnectorDataSourceParamDTO = + (ThirdPartySystemConnectorDataSourceParamDTO) datasourceParam; + + ThirdPartySystemConnectorConnectionParam thirdPartySystemConnectorConnectionParam = + new ThirdPartySystemConnectorConnectionParam(); + + thirdPartySystemConnectorConnectionParam.setServiceAddress( + thirdPartySystemConnectorDataSourceParamDTO.getServiceAddress()); + thirdPartySystemConnectorConnectionParam.setAuthConfig( + thirdPartySystemConnectorDataSourceParamDTO.getAuthConfig()); + thirdPartySystemConnectorConnectionParam.setSelectInterface( + thirdPartySystemConnectorDataSourceParamDTO.getSelectInterface()); + thirdPartySystemConnectorConnectionParam.setSubmitInterface( + thirdPartySystemConnectorDataSourceParamDTO.getSubmitInterface()); + thirdPartySystemConnectorConnectionParam.setPollStatusInterface( + thirdPartySystemConnectorDataSourceParamDTO.getPollStatusInterface()); + thirdPartySystemConnectorConnectionParam.setStopInterface( + thirdPartySystemConnectorDataSourceParamDTO.getStopInterface()); + thirdPartySystemConnectorConnectionParam.setInterfaceTimeout( + thirdPartySystemConnectorDataSourceParamDTO.getInterfaceTimeout()); + + return thirdPartySystemConnectorConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, ThirdPartySystemConnectorConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ""; + } + + @Override + public String getValidationQuery() { + return ""; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + return ""; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) { + return null; + } + + @Override + public boolean checkDataSourceConnectivity(ConnectionParam connectionParam) { + ThirdPartySystemConnectorConnectionParam baseConnectionParam = + (ThirdPartySystemConnectorConnectionParam) connectionParam; + try { + String token = AuthenticationUtils.authenticateAndGetToken(baseConnectionParam); + return token != null; + } catch (Exception e) { + log.error("connect error, e:{}", e.getMessage()); + return false; + } + } + + @Override + public DbType getDbType() { + return DbType.THIRDPARTY_SYSTEM_CONNECTOR; + } + + @Override + public DataSourceProcessor create() { + return new ThirdPartySystemConnectorDataSourceProcessor(); + } + + private void checkExternalSystemParam(ThirdPartySystemConnectorDataSourceParamDTO paramDTO) { + + // Check system name + if (paramDTO.getName() == null || paramDTO.getName().trim().isEmpty()) { + throw new IllegalArgumentException("system name cannot be empty"); + } + String systemName = paramDTO.getName().trim(); + if (systemName.length() > 64) { + throw new IllegalArgumentException("system name length cannot exceed 64 characters"); + } + paramDTO.setName(systemName); + + // Check service address + if (paramDTO.getServiceAddress() == null || paramDTO.getServiceAddress().trim().isEmpty()) { + throw new IllegalArgumentException("service address cannot be empty"); + } + paramDTO.setServiceAddress(paramDTO.getServiceAddress().trim()); + + // Check authentication configuration + AuthConfig authConfig = paramDTO.getAuthConfig(); + if (authConfig == null) { + throw new IllegalArgumentException("auth config cannot be empty"); + } + if (authConfig.getAuthType() == null) { + throw new IllegalArgumentException("auth type cannot be empty"); + } + + // 根据认证类型进行具体校验 + switch (authConfig.getAuthType()) { + case BASIC_AUTH: + if (authConfig.getBasicUsername() == null || authConfig.getBasicUsername().trim().isEmpty()) { + throw new IllegalArgumentException("basic auth username cannot be empty"); + } + authConfig.setBasicUsername(authConfig.getBasicUsername().trim()); + if (authConfig.getBasicPassword() == null || authConfig.getBasicPassword().trim().isEmpty()) { + throw new IllegalArgumentException("basic auth password cannot be empty"); + } + authConfig.setBasicPassword(authConfig.getBasicPassword().trim()); + break; + case JWT: + if (authConfig.getJwtToken() == null || authConfig.getJwtToken().trim().isEmpty()) { + throw new IllegalArgumentException("JWT token cannot be empty"); + } + authConfig.setJwtToken(authConfig.getJwtToken().trim()); + break; + case OAUTH2: + if (authConfig.getOauth2TokenUrl() == null || authConfig.getOauth2TokenUrl().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 token URL cannot be empty"); + } + authConfig.setOauth2TokenUrl(authConfig.getOauth2TokenUrl().trim()); + if (authConfig.getOauth2ClientId() == null || authConfig.getOauth2ClientId().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 client ID cannot be empty"); + } + authConfig.setOauth2ClientId(authConfig.getOauth2ClientId().trim()); + if (authConfig.getOauth2ClientSecret() == null || authConfig.getOauth2ClientSecret().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 client secret cannot be empty"); + } + authConfig.setOauth2ClientSecret(authConfig.getOauth2ClientSecret().trim()); + if (authConfig.getOauth2GrantType() == null || authConfig.getOauth2GrantType().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 grant type cannot be empty"); + } + authConfig.setOauth2GrantType(authConfig.getOauth2GrantType().trim()); + if (authConfig.getOauth2GrantType().equals("password")) { + if (authConfig.getOauth2Username() == null || authConfig.getOauth2Username().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 username cannot be empty"); + } + authConfig.setOauth2Username(authConfig.getOauth2Username().trim()); + if (authConfig.getOauth2Password() == null || authConfig.getOauth2Password().trim().isEmpty()) { + throw new IllegalArgumentException("OAuth2 password cannot be empty"); + } + authConfig.setOauth2Password(authConfig.getOauth2Password().trim()); + } + break; + default: + throw new IllegalArgumentException("unsupported auth type"); + } + + // Check interface configuration + if (paramDTO.getSelectInterface() == null) { + throw new IllegalArgumentException("select interface config cannot be empty"); + } + if (paramDTO.getSubmitInterface() == null) { + throw new IllegalArgumentException("submit interface config cannot be empty"); + } + if (paramDTO.getPollStatusInterface() == null) { + throw new IllegalArgumentException("poll status interface config cannot be empty"); + } + if (paramDTO.getStopInterface() == null) { + throw new IllegalArgumentException("stop interface config cannot be empty"); + } + + // Check interface configuration URL and method + checkInterfaceConfig(paramDTO.getSelectInterface()); + checkInterfaceConfig(paramDTO.getSubmitInterface()); + checkInterfaceConfig(paramDTO.getPollStatusInterface()); + checkInterfaceConfig(paramDTO.getStopInterface()); + } + + private void checkInterfaceConfig(InterfaceInfo interfaceInfo) { + if (interfaceInfo.getUrl() == null || interfaceInfo.getUrl().trim().isEmpty()) { + throw new IllegalArgumentException("interface URL cannot be empty"); + } + interfaceInfo.setUrl(interfaceInfo.getUrl().trim()); + if (interfaceInfo.getMethod() == null) { + throw new IllegalArgumentException("interface method cannot be empty"); + } + } + +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 66be77400e67..22ad429f4c84 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -58,6 +58,7 @@ dolphinscheduler-datasource-hana dolphinscheduler-datasource-aliyunserverlessspark dolphinscheduler-datasource-dolphindb + dolphinscheduler-datasource-thirdpartysystemconnector diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 6abe98db3cad..2e33e0addbb0 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -61,7 +61,8 @@ public enum DbType { K8S(26, "k8s", "k8s"), ALIYUN_SERVERLESS_SPARK(27, "aliyun_serverless_spark", "aliyun serverless spark"), - DOLPHINDB(28, "dolphindb", "dolphindb"); + DOLPHINDB(28, "dolphindb", "dolphindb"), + THIRDPARTY_SYSTEM_CONNECTOR(29, "thirdparty_system_connector", "thirdparty system connector"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index 8b616170b395..0fb60306dd89 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -216,6 +216,11 @@ dolphinscheduler-task-aliyunserverlessspark ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-task-external-system + ${project.version} + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/pom.xml new file mode 100644 index 000000000000..0c62577249e7 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/pom.xml @@ -0,0 +1,103 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-task-plugin + dev-SNAPSHOT + + + dolphinscheduler-task-external-system + jar + + + task.external-system + + + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-all + ${project.version} + + + + org.apache.httpcomponents + httpcore + + + + com.squareup.okhttp3 + mockwebserver + test + + + junit + junit + + + + + + com.jayway.jsonpath + json-path + 2.7.0 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemParameters.java new file mode 100644 index 000000000000..5aaccfe8a66e --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemParameters.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ThirdPartySystemConnectorConnectionParam; +import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; + +import org.jetbrains.annotations.NotNull; + +public class ExternalSystemParameters extends AbstractParameters { + + private int datasource; + + private String authenticationToken; + + private String externalTaskId; + private String externalTaskName; + + public int getDatasource() { + return datasource; + } + + public void setDatasource(int datasource) { + this.datasource = datasource; + } + + public String getAuthenticationToken() { + return authenticationToken; + } + + public void setAuthenticationToken(String authenticationToken) { + this.authenticationToken = authenticationToken; + } + + public String getExternalTaskId() { + return externalTaskId; + } + + public void setExternalTaskId(String externalTaskId) { + this.externalTaskId = externalTaskId; + } + + public String getExternalTaskName() { + return externalTaskName; + } + + public void setExternalTaskName(String externalTaskName) { + this.externalTaskName = externalTaskName; + } + + @Override + public ResourceParametersHelper getResources() { + ResourceParametersHelper resources = super.getResources(); + resources.put(ResourceType.DATASOURCE, datasource); + return resources; + } + + @Override + public boolean checkParameters() { + // Add validation logic here + return true; + } + + public ThirdPartySystemConnectorConnectionParam generateExtendedContext(@NotNull ResourceParametersHelper parametersHelper) { + DataSourceParameters externalSystemResourceParameters = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, + datasource); + ThirdPartySystemConnectorConnectionParam baseExternalSystemParams = + JSONUtils.parseObject(externalSystemResourceParameters.getConnectionParams(), + ThirdPartySystemConnectorConnectionParam.class); + return baseExternalSystemParams; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTask.java new file mode 100644 index 000000000000..8f1f0a830bda --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTask.java @@ -0,0 +1,516 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaderContentType; +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders; +import org.apache.dolphinscheduler.common.model.OkHttpResponse; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OkHttpUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.AuthenticationUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.InterfaceInfo; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingFailureConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingInterfaceInfo; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingSuccessConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.RequestParameter; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ResponseParameter; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ThirdPartySystemConnectorConnectionParam; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.FormBody; + +import com.jayway.jsonpath.JsonPath; + +@Slf4j +public class ExternalSystemTask extends AbstractTask { + + private Boolean traceEnabled = true; + + private ExternalSystemParameters externalSystemParameters; + private ThirdPartySystemConnectorConnectionParam baseExternalSystemParams; + private TaskExecutionContext taskExecutionContext; + private String accessToken; + private Map parameterMap = new HashMap<>(); + private Set successStatusCache = new HashSet<>(); + private Set failureStatusCache = new HashSet<>(); + + private boolean isTimeout = false; + private long taskStartTime; + + public ExternalSystemTask(TaskExecutionContext taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; + this.externalSystemParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ExternalSystemParameters.class); + baseExternalSystemParams = + externalSystemParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + try { + accessToken = baseExternalSystemParams.getAuthConfig().getHeaderPrefix() + " " + + AuthenticationUtils.authenticateAndGetToken(baseExternalSystemParams); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void init() { + externalSystemParameters = JSONUtils.parseObject( + taskExecutionContext.getTaskParams(), + ExternalSystemParameters.class); + + if (externalSystemParameters == null || !externalSystemParameters.checkParameters()) { + throw new RuntimeException("external system task params is not valid"); + } + + log.info("Initialize external system task with externalSystemId: {}, externalTaskId: {}, externalTaskName: {}", + externalSystemParameters.getDatasource(), + externalSystemParameters.getExternalTaskId(), + externalSystemParameters.getExternalTaskName()); + + // Initialize parameter mapping + initParameterMap(); + initStatusCache(); + } + + @Override + public void handle(TaskCallBack taskCallBack) throws TaskException { + try { + taskStartTime = System.currentTimeMillis(); + submitExternalTask(); + TimeUnit.SECONDS.sleep(10); + trackExternalTaskStatus(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TaskException("Task interrupted", e); + } catch (Exception e) { + throw new TaskException("Execute external system task failed", e); + } + } + + @Override + public void cancel() throws TaskException { + try { + log.info("cancel external system task"); + cancelTaskInstance(); + } catch (Exception e) { + throw new TaskException("cancel external system task error", e); + } finally { + // Only set to failure status when timeout failure strategy is enabled and it actually timed out + log.info("External task timeout check isTimeoutFailureEnabled:{}", isTimeoutFailureEnabled()); + if (isTimeoutFailureEnabled()) { + long currentTime = System.currentTimeMillis(); + long usedTimeMillis = currentTime - taskStartTime; + long usedTime = (usedTimeMillis + 59999) / 60000; // Round up to minutes (ceiling division) + log.info( + "External task timeout check, used time: {}m, timeout: {}m, currentTime: {}, taskStartTime: {}", + usedTime, taskExecutionContext.getTaskTimeout() / 60, currentTime, taskStartTime); + if (usedTime >= taskExecutionContext.getTaskTimeout() / 60) { + isTimeout = true; + log.warn("External task timeout, used time: {}m, timeout: {}m", + usedTime, taskExecutionContext.getTaskTimeout() / 60); + } + } + if (isTimeout) { + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + log.info("External task cancelled due to timeout, set status to FAILED"); + } else { + setExitStatusCode(TaskConstants.EXIT_CODE_KILL); + log.info("External task cancelled manually or timeout not enabled, set status to KILLED"); + } + } + } + + private void submitExternalTask() throws TaskException { + try { + InterfaceInfo submitConfig = baseExternalSystemParams.getSubmitInterface(); + String url = replaceParameterPlaceholders(baseExternalSystemParams.getCompleteUrl(submitConfig.getUrl())); + Map headers = new HashMap<>(); + buildAuthHeader(accessToken, headers); + buildHeaders(submitConfig, headers); + Map requestBody = buildRequestBody(submitConfig); + Map requestParams = buildRequestParams(submitConfig); + + int interfaceTimeout = baseExternalSystemParams.getInterfaceTimeout(); + log.info("Using interface timeout value: {} milliseconds", interfaceTimeout); + OkHttpResponse response = + executeRequestWithoutRetry(submitConfig.getMethod(), url, headers, requestParams, requestBody, + interfaceTimeout, interfaceTimeout, interfaceTimeout); + log.info("Submit task response:{}", response); + if (response.getStatusCode() != 200) { + throw new TaskException("Submit task failed: " + response.getBody()); + } + + parseSubmitResponse(submitConfig.getResponseParameters(), response.getBody()); + } catch (Exception e) { + log.error("Submit task failed:{}", e); + throw new TaskException("Submit task failed", e); + } + } + + private void trackExternalTaskStatus() throws TaskException { + try { + String status; + do { + // Only check timeout when timeout failure strategy is enabled + if (isTimeoutFailureEnabled()) { + long currentTime = System.currentTimeMillis(); + long usedTimeMillis = currentTime - taskStartTime; + long usedTime = (usedTimeMillis + 59999) / 60000; // Round up to minutes (ceiling division) + if (usedTime >= taskExecutionContext.getTaskTimeout()) { + isTimeout = true; + log.error("External task timeout, used time: {}m, timeout: {}m", + usedTime, taskExecutionContext.getTaskTimeout() / 60); + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + cancelTaskInstance(); + return; + } + } + + status = pollTaskStatus(); + + if (successStatusCache.contains(status)) { + setExitStatusCode(TaskConstants.EXIT_CODE_SUCCESS); + log.info("External task completed successfully with status: {}", status); + return; + } else if (failureStatusCache.contains(status)) { + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + log.error("External task failed with status: {}", status); + return; + } + + TimeUnit.SECONDS.sleep(10); + } while (traceEnabled); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TaskException("Task status tracking interrupted", e); + } catch (Exception e) { + log.error("Track task status failed", e); + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + throw new TaskException("Track task status failed", e); + } + } + + private String pollTaskStatus() throws TaskException { + try { + PollingInterfaceInfo pollConfig = + baseExternalSystemParams.getPollStatusInterface(); + String url = replaceParameterPlaceholders(baseExternalSystemParams.getCompleteUrl(pollConfig.getUrl())); + Map headers = new HashMap<>(); + buildAuthHeader(accessToken, headers); + buildHeaders(pollConfig, headers); + Map requestBody = buildRequestBody(pollConfig); + Map requestParams = buildRequestParams(pollConfig); + + int interfaceTimeout = baseExternalSystemParams.getInterfaceTimeout(); + OkHttpResponse response = + executeRequestWithRetry(pollConfig.getMethod(), url, headers, requestParams, requestBody, + interfaceTimeout, interfaceTimeout, interfaceTimeout, 3); + log.info("poll task status response:{}", response); + + if (response.getStatusCode() != 200) { + throw new TaskException("polling task failed: " + response.getBody()); + } + + String successField = pollConfig.getPollingSuccessConfig().getSuccessField(); + String failureField = pollConfig.getPollingFailureConfig().getFailureField(); + + Object successStatusObj = JsonPath.read(response.getBody(), successField); + Object failureStatusObj = JsonPath.read(response.getBody(), failureField); + + log.info("PollTaskStatus successfully, external task instance success status: {}, failure status: {}", + successStatusObj != null ? successStatusObj.toString() : "null", + failureStatusObj != null ? failureStatusObj.toString() : "null"); + + if (successStatusObj != null) { + return successStatusObj.toString(); + } else if (failureStatusObj != null) { + return failureStatusObj.toString(); + } else { + log.warn("successStatusObj and failureStatusObj are null"); + return "UNKNOWN"; + } + } catch (Exception e) { + log.error("Poll task status failed", e); + throw new TaskException("Poll task status failed", e); + } + } + + private void cancelTaskInstance() throws TaskException { + try { + traceEnabled = false; + InterfaceInfo stopConfig = baseExternalSystemParams.getStopInterface(); + log.info("start cancel External System TaskInstance"); + String url = replaceParameterPlaceholders(baseExternalSystemParams.getCompleteUrl(stopConfig.getUrl())); + Map headers = new HashMap<>(); + buildAuthHeader(accessToken, headers); + buildHeaders(stopConfig, headers); + Map requestBody = buildRequestBody(stopConfig); + Map requestParams = buildRequestParams(stopConfig); + + int interfaceTimeout = baseExternalSystemParams.getInterfaceTimeout(); + OkHttpResponse response = + executeRequestWithRetry(stopConfig.getMethod(), url, headers, requestParams, requestBody, + interfaceTimeout, interfaceTimeout, interfaceTimeout, 3); + log.info("cancel task response:{}", response); + + if (response.getStatusCode() != 200) { + throw new TaskException("Cancel task failed: " + response.getBody()); + } + log.info("Cancel task result: {}", response.getBody()); + } catch (Exception e) { + log.error("Cancel task failed", e); + throw new TaskException("Cancel task failed", e); + } + } + + private OkHttpResponse executeRequestWithoutRetry(InterfaceInfo.HttpMethod method, String url, + Map headers, Map requestParams, + Map requestBody, int connectTimeout, + int readTimeout, + int writeTimeout) throws TaskException { + return executeRequestWithRetry(method, url, headers, requestParams, requestBody, connectTimeout, readTimeout, + writeTimeout, 0); + } + + private OkHttpResponse executeRequestWithRetry(InterfaceInfo.HttpMethod method, String url, + Map headers, Map requestParams, + Map requestBody, int connectTimeout, int readTimeout, + int writeTimeout, int maxRetries) throws TaskException { + int retryCount = 0; + while (retryCount <= maxRetries) { + OkHttpRequestHeaders okHttpRequestHeaders = new OkHttpRequestHeaders(); + okHttpRequestHeaders.setHeaders(headers); + OkHttpRequestHeaderContentType contentType = getContentType(headers); + okHttpRequestHeaders.setOkHttpRequestHeaderContentType(getContentType(headers)); + try { + switch (method) { + case POST: + if (contentType.equals(OkHttpRequestHeaderContentType.APPLICATION_JSON)) { + return OkHttpUtils.post(url, okHttpRequestHeaders, requestParams, requestBody, + connectTimeout, + readTimeout, writeTimeout); + } + if (contentType.equals(OkHttpRequestHeaderContentType.APPLICATION_FORM_URLENCODED)) { + FormBody.Builder formBodyBuilder = new FormBody.Builder(); + if (requestBody != null) { + for (Map.Entry entry : requestBody.entrySet()) { + formBodyBuilder.add(entry.getKey(), entry.getValue().toString()); + } + } + return OkHttpUtils.postFormBody(url, okHttpRequestHeaders, requestParams, + formBodyBuilder.build(), connectTimeout, + readTimeout, writeTimeout); + } + case PUT: + return OkHttpUtils.put(url, okHttpRequestHeaders, requestBody, connectTimeout, readTimeout, + writeTimeout); + case GET: + return OkHttpUtils.get(url, okHttpRequestHeaders, requestParams, connectTimeout, readTimeout, + writeTimeout); + default: + throw new TaskException("Unsupported HTTP method: " + method); + } + } catch (Exception e) { + retryCount++; + if (maxRetries > 0) { + log.warn("Request failed, retrying... (attempt {}/{})", retryCount, maxRetries, e); + if (retryCount > maxRetries) { + throw new TaskException("Request failed after " + maxRetries + " retries", e); + } + try { + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new TaskException("Request retry interrupted", ie); + } + } else { + throw new TaskException("Request failed without retry", e); + } + } + } + return null; // This line should never be reached + } + + private void buildAuthHeader(String accessToken, Map headers) { + headers.put("Authorization", accessToken); + } + + private Map buildHeaders(InterfaceInfo config, + Map requestParams) { + for (RequestParameter param : config.getParameters()) { + if (param.getLocation().equals(RequestParameter.ParamLocation.HEADER)) { + requestParams.put(param.getParamName(), replaceParameterPlaceholders(param.getParamValue())); + } + } + return requestParams; + } + + private Map buildRequestBody(InterfaceInfo config) { + Map requestBody = new HashMap<>(); + if (config.getBody() != null) { + requestBody = JSONUtils.parseObject(replaceParameterPlaceholders(config.getBody()), Map.class); + } + return requestBody; + } + + private Map buildRequestParams(InterfaceInfo config) { + Map requestParams = new HashMap<>(); + for (RequestParameter param : config.getParameters()) { + if (param.getLocation().equals(RequestParameter.ParamLocation.PARAM)) { + requestParams.put(param.getParamName(), replaceParameterPlaceholders(param.getParamValue())); + } + } + return requestParams; + } + + private String replaceParameterPlaceholders(String template) { + if (StringUtils.isEmpty(template)) { + return template; + } + + StringBuilder result = new StringBuilder(template); + for (Map.Entry entry : parameterMap.entrySet()) { + String placeholder = "${" + entry.getKey() + "}"; + int index; + while ((index = result.indexOf(placeholder)) != -1) { + result.replace(index, index + placeholder.length(), entry.getValue()); + } + } + String resultString = ParameterUtils.convertParameterPlaceholders(result.toString(), parameterMap); + log.info("after replaceParameterPlaceholders:{}", resultString); + return resultString; + } + + private void parseSubmitResponse(List responseParameters, + String responseBody) throws TaskException { + try { + for (ResponseParameter param : responseParameters) { + String jsonPath = param.getJsonPath(); + String key = param.getKey(); + Object value = JsonPath.read(responseBody, jsonPath); + + if (value == null) { + log.warn("Response parameter {} not found in response body", key); + continue; + } + + parameterMap.put(key, value.toString().replace("\"", "")); + log.info("Parsed parameter {}: {}", key, value != null ? value.toString() : "null"); + + } + } catch (Exception e) { + log.error("Parse submit response failed", e); + throw new TaskException("Parse submit response failed", e); + } + } + + @Override + public AbstractParameters getParameters() { + return this.externalSystemParameters; + } + + /** + * Initialize parameter mapping + */ + private void initParameterMap() { + Map prepareParamsMap = taskExecutionContext.getPrepareParamsMap(); + if (prepareParamsMap != null) { + for (Map.Entry entry : prepareParamsMap.entrySet()) { + parameterMap.put(entry.getKey(), entry.getValue().getValue()); + } + } + if (externalSystemParameters.getExternalTaskId() != null) { + parameterMap.put(ExternalTaskConstants.EXTERNAL_TASK_ID, externalSystemParameters.getExternalTaskId()); + parameterMap.put(ExternalTaskConstants.EXTERNAL_TASK_NAME, externalSystemParameters.getExternalTaskName()); + } + } + + private void initStatusCache() { + PollingSuccessConfig successConfig = + baseExternalSystemParams.getPollStatusInterface().getPollingSuccessConfig(); + + if (successConfig != null && successConfig.getSuccessValue() != null) { + try { + String successValueString = successConfig.getSuccessValue(); + String[] successValues = successValueString.split(","); + for (String successValue : successValues) { + successStatusCache.add(successValue); + } + log.info("trackExternalTaskStatus successValues is :{}", successStatusCache); + + } catch (NullPointerException e) { + log.error("Error: successValue is null"); + } + } + PollingFailureConfig failureConfig = + baseExternalSystemParams.getPollStatusInterface().getPollingFailureConfig(); + if (failureConfig != null && failureConfig.getFailureField() != null) { + try { + String failureValueString = failureConfig.getFailureValue(); + String[] failureValues = failureValueString.split(","); + for (String failureValue : failureValues) { + failureStatusCache.add(failureValue); + } + log.info("trackExternalTaskStatus failureValues is :{}", failureStatusCache); + } catch (NullPointerException e) { + log.error("Error: failureStatus is null"); + } + } + } + + /** + * Check if timeout failure strategy is enabled + */ + private boolean isTimeoutFailureEnabled() { + return taskExecutionContext.getTaskTimeoutStrategy() != null + && taskExecutionContext.getTaskTimeout() > 0 + && taskExecutionContext.getTaskTimeout() < Integer.MAX_VALUE + && (taskExecutionContext.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED + || taskExecutionContext.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED); + } + + private OkHttpRequestHeaderContentType getContentType(Map headers) { + if (headers == null || (!headers.containsKey(ExternalTaskConstants.CONTENT_TYPE) + && !headers.containsKey(ExternalTaskConstants.CONTENT_TYPE_LOWERCASE))) { + return OkHttpRequestHeaderContentType.APPLICATION_JSON; + } + String contentType = headers.get(ExternalTaskConstants.CONTENT_TYPE); + return OkHttpRequestHeaderContentType.fromValue(contentType) != null + ? OkHttpRequestHeaderContentType.fromValue(contentType) + : OkHttpRequestHeaderContentType.APPLICATION_JSON; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannel.java new file mode 100644 index 000000000000..ac87a9ccca08 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannel.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; + +public class ExternalSystemTaskChannel implements TaskChannel { + + @Override + public AbstractTask createTask(TaskExecutionContext taskRequest) { + return new ExternalSystemTask(taskRequest); + } + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ExternalSystemParameters.class); + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannelFactory.java new file mode 100644 index 000000000000..8a521403a365 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskChannelFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class ExternalSystemTaskChannelFactory implements TaskChannelFactory { + + @Override + public String getName() { + return "EXTERNAL_SYSTEM"; + } + + @Override + public TaskChannel create() { + return new ExternalSystemTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalTaskConstants.java new file mode 100644 index 000000000000..d62df2207c6e --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/main/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalTaskConstants.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ExternalTaskConstants { + + public static final String CONTENT_TYPE = "Content-Type"; + public static final String CONTENT_TYPE_LOWERCASE = "content-type"; + + public static final String EXTERNAL_TASK_ID = "id"; + public static final String EXTERNAL_TASK_NAME = "name"; + public static final int RESPONSE_CODE_SUCCESS = 200; +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/test/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/test/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskTest.java new file mode 100644 index 000000000000..ee91296d8bdd --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-external-system/src/test/java/org/apache/dolphinscheduler/plugin/task/externalSystem/ExternalSystemTaskTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.externalSystem; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.common.model.OkHttpRequestHeaders; +import org.apache.dolphinscheduler.common.model.OkHttpResponse; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OkHttpUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.AuthenticationUtils; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.AuthConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.InterfaceInfo; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingFailureConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingInterfaceInfo; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.PollingSuccessConfig; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ResponseParameter; +import org.apache.dolphinscheduler.plugin.datasource.thirdpartysystemconnector.param.ThirdPartySystemConnectorConnectionParam; +import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class ExternalSystemTaskTest { + + private TaskExecutionContext taskExecutionContext; + private ExternalSystemParameters externalSystemParameters; + private ThirdPartySystemConnectorConnectionParam thirdPartyParams; + private ExternalSystemTask externalSystemTask; + + @BeforeEach + public void setUp() throws Exception { + taskExecutionContext = mock(TaskExecutionContext.class); + externalSystemParameters = new ExternalSystemParameters(); + externalSystemParameters.setDatasource(1); + externalSystemParameters.setExternalTaskId("task-123"); + externalSystemParameters.setExternalTaskName("Test Task"); + + thirdPartyParams = new ThirdPartySystemConnectorConnectionParam(); + AuthConfig authConfig = new AuthConfig(); + authConfig.setHeaderPrefix("Bearer"); + thirdPartyParams.setAuthConfig(authConfig); + + InterfaceInfo submitInterface = new InterfaceInfo(); + submitInterface.setUrl("http://test.com/api/tasks"); + submitInterface.setMethod(InterfaceInfo.HttpMethod.POST); + submitInterface.setBody("{\"taskName\":\"${externalTaskName}\"}"); + submitInterface.setParameters(Collections.emptyList()); + submitInterface.setResponseParameters(Arrays.asList( + createResponseParameter("taskId", "$.taskId"), + createResponseParameter("status", "$.status"))); + thirdPartyParams.setSubmitInterface(submitInterface); + + PollingInterfaceInfo pollInterface = new PollingInterfaceInfo(); + pollInterface.setUrl("http://test.com/api/tasks/${taskId}/status"); + pollInterface.setMethod(InterfaceInfo.HttpMethod.GET); + pollInterface.setParameters(Collections.emptyList()); + pollInterface.setResponseParameters(Collections.emptyList()); + + PollingSuccessConfig successConfig = new PollingSuccessConfig(); + successConfig.setSuccessField("$.status"); + successConfig.setSuccessValue("COMPLETED,SUCCESS"); + pollInterface.setPollingSuccessConfig(successConfig); + + PollingFailureConfig failureConfig = new PollingFailureConfig(); + failureConfig.setFailureField("$.status"); + failureConfig.setFailureValue("FAILED,ERROR"); + pollInterface.setPollingFailureConfig(failureConfig); + + thirdPartyParams.setPollStatusInterface(pollInterface); + + InterfaceInfo stopInterface = new InterfaceInfo(); + stopInterface.setUrl("http://test.com/api/tasks/${taskId}/stop"); + stopInterface.setMethod(InterfaceInfo.HttpMethod.POST); + stopInterface.setParameters(Collections.emptyList()); + thirdPartyParams.setStopInterface(stopInterface); + + String paramsJson = JSONUtils.toJsonString(externalSystemParameters); + Mockito.lenient().when(taskExecutionContext.getTaskParams()).thenReturn(paramsJson); + Mockito.lenient().when(taskExecutionContext.getTaskTimeout()).thenReturn(300000); + Mockito.lenient().when(taskExecutionContext.getTaskTimeoutStrategy()).thenReturn(TaskTimeoutStrategy.FAILED); + + org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper resourceHelper = + mock(org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper.class); + + org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters dataSourceParams = + mock(org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters.class); + Mockito.lenient().when(dataSourceParams.getConnectionParams()) + .thenReturn(JSONUtils.toJsonString(thirdPartyParams)); + + Mockito.lenient().when(resourceHelper.getResourceParameters( + org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType.DATASOURCE, + externalSystemParameters.getDatasource())).thenReturn(dataSourceParams); + + Mockito.lenient().when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceHelper); + + try (MockedStatic mockedAuth = Mockito.mockStatic(AuthenticationUtils.class)) { + mockedAuth + .when(() -> AuthenticationUtils + .authenticateAndGetToken(any(ThirdPartySystemConnectorConnectionParam.class))) + .thenReturn("mocked-token"); + + externalSystemTask = new ExternalSystemTask(taskExecutionContext); + + java.lang.reflect.Field parametersField = + ExternalSystemTask.class.getDeclaredField("externalSystemParameters"); + parametersField.setAccessible(true); + parametersField.set(externalSystemTask, externalSystemParameters); + + java.lang.reflect.Field baseParamsField = + ExternalSystemTask.class.getDeclaredField("baseExternalSystemParams"); + baseParamsField.setAccessible(true); + baseParamsField.set(externalSystemTask, thirdPartyParams); + + externalSystemTask.init(); + } + } + + private ResponseParameter createResponseParameter(String key, String jsonPath) { + ResponseParameter param = new ResponseParameter(); + param.setKey(key); + param.setJsonPath(jsonPath); + return param; + } + + @Test + public void testInit() { + try (MockedStatic mockedJsonUtils = Mockito.mockStatic(JSONUtils.class)) { + mockedJsonUtils.when(() -> JSONUtils.parseObject(any(String.class), eq(ExternalSystemParameters.class))) + .thenReturn(externalSystemParameters); + + externalSystemTask.init(); + + Assertions.assertNotNull(externalSystemTask.getParameters()); + } + } + + @Test + public void testSubmitExternalTask() throws Exception { + try (MockedStatic mockedOkHttp = Mockito.mockStatic(OkHttpUtils.class)) { + OkHttpResponse submitResponse = mock(OkHttpResponse.class); + when(submitResponse.getStatusCode()).thenReturn(200); + when(submitResponse.getBody()) + .thenReturn("{\"taskId\": \"12345\", \"status\": \"SUBMITTED\", \"name\": \"Test Task\"}"); + + OkHttpResponse runningResponse = mock(OkHttpResponse.class); + when(runningResponse.getStatusCode()).thenReturn(200); + when(runningResponse.getBody()).thenReturn("{\"status\": \"RUNNING\"}"); + + OkHttpResponse completedResponse = mock(OkHttpResponse.class); + when(completedResponse.getStatusCode()).thenReturn(200); + when(completedResponse.getBody()).thenReturn("{\"status\": \"COMPLETED\"}"); + + mockedOkHttp.when(() -> OkHttpUtils.post(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), any(Map.class), anyInt(), anyInt(), anyInt())) + .thenReturn(submitResponse); + + mockedOkHttp.when(() -> OkHttpUtils.get(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), anyInt(), anyInt(), anyInt())) + .thenReturn(runningResponse) + .thenReturn(runningResponse) + .thenReturn(completedResponse); + + TaskCallBack taskCallBack = mock(TaskCallBack.class); + + Assertions.assertDoesNotThrow(() -> { + externalSystemTask.handle(taskCallBack); + }); + + mockedOkHttp.verify(() -> OkHttpUtils.get(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), anyInt(), anyInt(), anyInt()), times(3)); + } + } + + @Test + public void testSubmitExternalTaskWithSuccessStatus() throws Exception { + try (MockedStatic mockedOkHttp = Mockito.mockStatic(OkHttpUtils.class)) { + OkHttpResponse submitResponse = mock(OkHttpResponse.class); + when(submitResponse.getStatusCode()).thenReturn(200); + when(submitResponse.getBody()).thenReturn( + "{\"taskId\": \"12345\", \"taskInstanceId\": \"instance-123\", \"status\": \"SUBMITTED\", \"name\": \"Test Task\"}"); + + OkHttpResponse runningResponse1 = mock(OkHttpResponse.class); + when(runningResponse1.getStatusCode()).thenReturn(200); + when(runningResponse1.getBody()).thenReturn("{\"status\": \"RUNNING\"}"); + + OkHttpResponse runningResponse2 = mock(OkHttpResponse.class); + when(runningResponse2.getStatusCode()).thenReturn(200); + when(runningResponse2.getBody()).thenReturn("{\"status\": \"RUNNING\"}"); + + OkHttpResponse successResponse = mock(OkHttpResponse.class); + when(successResponse.getStatusCode()).thenReturn(200); + when(successResponse.getBody()).thenReturn("{\"status\": \"SUCCESS\"}"); + + mockedOkHttp.when(() -> OkHttpUtils.post(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), any(Map.class), anyInt(), anyInt(), anyInt())) + .thenReturn(submitResponse); + + mockedOkHttp.when(() -> OkHttpUtils.get(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), anyInt(), anyInt(), anyInt())) + .thenReturn(runningResponse1) + .thenReturn(runningResponse2) + .thenReturn(successResponse); + + TaskCallBack taskCallBack = mock(TaskCallBack.class); + + Assertions.assertDoesNotThrow(() -> { + externalSystemTask.handle(taskCallBack); + }); + + mockedOkHttp.verify(() -> OkHttpUtils.get(any(String.class), any(OkHttpRequestHeaders.class), + any(Map.class), anyInt(), anyInt(), anyInt()), times(3)); + + Assertions.assertEquals(TaskConstants.EXIT_CODE_SUCCESS, externalSystemTask.getExitStatusCode()); + } + } + + @Test + public void testTimeoutCheckEnabled() { + try { + java.lang.reflect.Method method = ExternalSystemTask.class.getDeclaredMethod("isTimeoutFailureEnabled"); + method.setAccessible(true); + boolean result = (boolean) method.invoke(externalSystemTask); + Assertions.assertTrue(result); + } catch (Exception e) { + Assertions.fail("Failed to invoke isTimeoutFailureEnabled method: " + e.getMessage()); + } + } + + @Test + public void testReplaceParameterPlaceholders() { + Map parameterMap = new HashMap<>(); + parameterMap.put("externalTaskId", "task-123"); + parameterMap.put("externalTaskName", "Test Task"); + + try { + java.lang.reflect.Field field = ExternalSystemTask.class.getDeclaredField("parameterMap"); + field.setAccessible(true); + field.set(externalSystemTask, parameterMap); + } catch (Exception e) { + Assertions.fail("Failed to set parameterMap field"); + } + + String template = "http://test.com/api/tasks/${externalTaskId}/${externalTaskName}"; + try { + java.lang.reflect.Method method = + ExternalSystemTask.class.getDeclaredMethod("replaceParameterPlaceholders", String.class); + method.setAccessible(true); + String result = (String) method.invoke(externalSystemTask, template); + + Assertions.assertEquals("http://test.com/api/tasks/task-123/Test Task", result); + } catch (Exception e) { + Assertions.fail("Failed to invoke replaceParameterPlaceholders method: " + e.getMessage()); + } + } +} diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index 6f3d727d59b9..aeb23258b062 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -62,6 +62,7 @@ dolphinscheduler-task-datafactory dolphinscheduler-task-remoteshell dolphinscheduler-task-aliyunserverlessspark + dolphinscheduler-task-external-system diff --git a/dolphinscheduler-ui/public/images/task-icons/external_system.png b/dolphinscheduler-ui/public/images/task-icons/external_system.png new file mode 100644 index 000000000000..a564ebcc33a4 Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/external_system.png differ diff --git a/dolphinscheduler-ui/public/images/task-icons/external_system_hover.png b/dolphinscheduler-ui/public/images/task-icons/external_system_hover.png new file mode 100644 index 000000000000..d379f195216f Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/external_system_hover.png differ diff --git a/dolphinscheduler-ui/src/locales/en_US/index.ts b/dolphinscheduler-ui/src/locales/en_US/index.ts index 0e289642de67..08c9d265d8aa 100644 --- a/dolphinscheduler-ui/src/locales/en_US/index.ts +++ b/dolphinscheduler-ui/src/locales/en_US/index.ts @@ -29,6 +29,7 @@ import project from '@/locales/en_US/project' import resource from '@/locales/en_US/resource' import security from '@/locales/en_US/security' import theme from '@/locales/en_US/theme' +import thirdparty_api_source from '@/locales/en_US/thirdparty-api-source' import user_dropdown from '@/locales/en_US/user-dropdown' import ui_setting from '@/locales/en_US/ui_setting' import about from '@/locales/en_US/about' @@ -48,6 +49,7 @@ export default { project, security, datasource, + thirdparty_api_source, crontab, ui_setting, input_search diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 916893b02a0c..d033b7eda13f 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -547,6 +547,8 @@ export default { target_task_name: 'Target Task Name', datasource_type: 'Datasource types', datasource_instances: 'Datasource instances', + external_systems: 'External Systems', + external_system_tasks: 'External System Tasks', sql_type: 'SQL Type', sql_type_query: 'Query', sql_type_non_query: 'Non Query', diff --git a/dolphinscheduler-ui/src/locales/en_US/security.ts b/dolphinscheduler-ui/src/locales/en_US/security.ts index 15e1c1fc2c42..e23ec17cfeac 100644 --- a/dolphinscheduler-ui/src/locales/en_US/security.ts +++ b/dolphinscheduler-ui/src/locales/en_US/security.ts @@ -153,12 +153,14 @@ export default { file_resource: 'File Resource', datasource: 'Datasource', namespace: 'Namespace', + thirdparty: 'Thirdparty', revoke_auth: 'Revoke', grant_read: 'Grant Read', grant_all: 'Grant All', authorize_project: 'Project Authorize', authorize_namespace: 'Namespace Authorize', authorize_datasource: 'Datasource Authorize', + authorize_thirdparty: 'Thirdparty Authorize', username: 'Username', username_exists: 'The username already exists', username_tips: 'Please enter username', diff --git a/dolphinscheduler-ui/src/locales/en_US/thirdparty-api-source.ts b/dolphinscheduler-ui/src/locales/en_US/thirdparty-api-source.ts new file mode 100644 index 000000000000..03185fbc3494 --- /dev/null +++ b/dolphinscheduler-ui/src/locales/en_US/thirdparty-api-source.ts @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { + // Route titles + thirdparty_api_source: 'Thirdparty API Source', + create_thirdparty_api_source: 'Create Thirdparty API Source', + edit_thirdparty_api_source: 'Edit Thirdparty API Source', + + // Basic Information + basic_info: 'Basic Information', + id: 'ID', + system_name: 'System Name', + system_name_tips: 'Please enter system name', + service_address: 'Service Address', + service_address_tips: 'Please enter service address', + field_mapping: 'Field Mapping', + external_field: 'External Field', + internal_field: 'Internal Field', + create_time: 'Create Time', + update_time: 'Update Time', + + // Authentication Configuration + auth_config: 'Authentication Configuration', + auth_type: 'Authentication Type', + auth_type_tips: 'Please select authentication type', + basic_auth: 'Basic Authentication', + oauth2: 'OAuth2 Authentication', + jwt: 'JWT Authentication', + username: 'Username', + username_tips: 'Please enter username', + password: 'Password', + password_tips: 'Please enter password', + jwt_token: 'JWT Token', + jwt_token_tips: 'Please enter JWT token', + oauth2_token_url: 'Token URL', + oauth2_token_url_tips: 'Please enter Token URL', + oauth2_client_id: 'Client ID', + oauth2_client_id_tips: 'Please enter Client ID', + oauth2_client_secret: 'Client Secret', + oauth2_client_secret_tips: 'Please enter Client Secret', + oauth2_grant_type: 'Grant Type', + oauth2_grant_type_tips: 'Please enter Grant Type', + oauth2_username: 'OAuth2 Username', + oauth2_username_tips: 'Please enter OAuth2 username', + oauth2_password: 'OAuth2 Password', + oauth2_password_tips: 'Please enter OAuth2 password', + additional_params: 'Additional Parameters', + add_param: 'Add Param', + add_extract_field: 'Add Extract Field', + key: 'key', + value: 'value', + + // Interface Configuration + interface_config: 'Interface Configuration', + input_interface: 'Input Interface', + input_interface_tips: 'Please enter interface address', + submit_interface: 'Submit Interface', + submit_interface_tips: 'Please enter interface address', + query_interface: 'Query Interface', + query_interface_tips: 'Please enter interface address', + stop_interface: 'Stop Interface', + stop_interface_tips: 'Please enter interface address', + parameters: 'Parameters', + param_location: 'Parameter Location', + param_location_tips: 'Please select parameter location', + param_name_tips: 'Please enter parameter name', + param_value_tips: 'Please enter parameter value', + extract_response_data: 'Please enter response data jsonPath', + extract_field: 'Please enter extract field', + json_path_list: 'Please enter, e.g: $.data[*].id', + json_path: 'Please enter, e.g: $.data.taskInstanceId', + header_prefix: 'Authorization Token Prefix', + header_prefix_tips: 'Authorization header before token,e.g.Bearer', + system_field_tips: 'Please select system field', + request_body: 'Request Body', + request_body_placeholder: 'Please enter JSON format request body', + header: 'Header', + query: 'Query', + get: 'GET', + post: 'POST', + put: 'PUT', + interface_timeout: 'Interface Timeout', + interface_timeout_tips: 'Please enter interface timeout', + millisecond: 'millisecond', + + interface_timeout_description: + 'Set the interface request timeout, default is 120000 milliseconds (2 minutes)', + + // Polling Configuration + polling_config: 'Polling Configuration', + success_condition: 'Success Condition', + success_field: 'Success Field', + success_field_tips: 'Please enter success field JSONPath,e.g.$.data.status', + success_value: 'Success Value', + success_value_tips: + 'Please enter all enum values for success,e.g.SUCCESS,FINISHED', + failure_condition: 'Failure Condition', + failure_field: 'Failure Field', + failure_field_tips: 'Please enter failure field JSONPath,e.g.$.data.status', + failure_value: 'Failure Value', + failure_value_tips: + 'Please enter all enum values for failure,e.g.CANCELED, FAILED', + + // Buttons and Operations + cancel: 'Cancel', + submit: 'Submit', + test: 'Test', + search: 'Search', + create: 'Create', + edit: 'Edit', + delete: 'Delete', + update: 'Update', + + // Message Tips + create_success: 'Create successful', + edit_success: 'Edit successful', + delete_success: 'Delete successful', + test_success: 'Test successful', + create_failed: 'Create failed', + edit_failed: 'Edit failed', + delete_failed: 'Delete failed', + test_failed: 'Test failed', + submit_failed: 'Submit failed, please check form content', + + // Form validation messages + system_name_required: 'System name is required', + service_address_required: 'Service address is required', + auth_type_required: 'Authentication type is required', + username_required: 'Username is required', + password_required: 'Password is required', + jwt_token_required: 'JWT token is required', + oauth2_token_url_required: 'Token URL is required', + oauth2_client_id_required: 'Client ID is required', + oauth2_client_secret_required: 'Client Secret is required', + oauth2_grant_type_required: 'Grant Type is required', + input_interface_url_required: 'Input interface address is required', + submit_interface_url_required: 'Submit interface address is required', + query_interface_url_required: 'Query interface address is required', + stop_interface_url_required: 'Stop interface address is required', + success_condition_required: + 'Success condition field and value cannot be empty', + failure_condition_required: + 'Failure condition field and value cannot be empty', + + external_system_required: 'External system is required', + external_system_task_required: 'External system task is required', + id_jsonpath_required: 'ID field and JSONPath is required', + name_jsonpath_required: 'Name field and JSONPath is required', + taskinstanceid_jsonpath_required: + 'TaskInstanceId field and JSONPath is required' +} diff --git a/dolphinscheduler-ui/src/locales/zh_CN/index.ts b/dolphinscheduler-ui/src/locales/zh_CN/index.ts index fdd36e1a5cb0..15de405c6a99 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/index.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/index.ts @@ -29,6 +29,7 @@ import project from '@/locales/zh_CN/project' import resource from '@/locales/zh_CN/resource' import security from '@/locales/zh_CN/security' import theme from '@/locales/zh_CN/theme' +import thirdparty_api_source from '@/locales/zh_CN/thirdparty-api-source' import user_dropdown from '@/locales/zh_CN/user-dropdown' import ui_setting from '@/locales/zh_CN/ui_setting' import about from '@/locales/zh_CN/about' @@ -48,6 +49,7 @@ export default { project, security, datasource, + thirdparty_api_source, crontab, ui_setting, input_search diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 46fd12e348fd..27bbac05ec05 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -530,6 +530,8 @@ export default { target_task_name: '目标任务名', datasource_type: '数据源类型', datasource_instances: '数据源实例', + external_systems: '第三方系统列表', + external_system_tasks: '第三方系统任务列表', sql_type: 'SQL类型', sql_type_query: '查询', sql_type_non_query: '非查询', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/security.ts b/dolphinscheduler-ui/src/locales/zh_CN/security.ts index fc0a00f5e1a1..0efc75747e08 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/security.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/security.ts @@ -151,12 +151,14 @@ export default { file_resource: '文件资源', datasource: '数据源', namespace: '命名空间', + thirdparty: '第三方系统', revoke_auth: '撤销权限', grant_read: '授予读权限', grant_all: '授予所有权限', authorize_project: '项目授权', authorize_namespace: '命名空间授权', authorize_datasource: '数据源授权', + authorize_thirdparty: '第三方系统授权', username: '用户名', username_exists: '用户名已存在', username_tips: '请输入用户名', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/thirdparty-api-source.ts b/dolphinscheduler-ui/src/locales/zh_CN/thirdparty-api-source.ts new file mode 100644 index 000000000000..04e58317c1fa --- /dev/null +++ b/dolphinscheduler-ui/src/locales/zh_CN/thirdparty-api-source.ts @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { + thirdparty_api_source: '第三方系统API', + create_thirdparty_api_source: '创建第三方系统API', + edit_thirdparty_api_source: '编辑第三方系统API', + + basic_info: '基本信息', + id: 'ID', + system_name: '系统名称', + system_name_tips: '请输入系统名称', + service_address: '服务地址', + service_address_tips: '请输入服务地址', + field_mapping: '字段映射', + internal_field: '系统字段', + external_field: '注册系统字段', + create_time: '创建时间', + update_time: '更新时间', + + auth_config: '认证配置', + auth_type: '认证类型', + auth_type_tips: '请选择认证类型', + basic_auth: '基础认证', + oauth2: 'OAuth2认证', + jwt: 'JWT认证', + username: '用户名', + username_tips: '请输入用户名', + password: '密码', + password_tips: '请输入密码', + jwt_token: 'JWT串', + jwt_token_tips: '请输入JWT串', + oauth2_token_url: 'Token URL', + oauth2_token_url_tips: '请输入Token URL', + oauth2_client_id: 'Client ID', + oauth2_client_id_tips: '请输入Client ID', + oauth2_client_secret: 'Client Secret', + oauth2_client_secret_tips: '请输入Client Secret', + oauth2_grant_type: 'Grant Type', + oauth2_grant_type_tips: '请输入Grant Type', + oauth2_username: 'OAuth2用户名', + oauth2_username_tips: '请输入OAuth2用户名', + oauth2_password: 'OAuth2密码', + oauth2_password_tips: '请输入OAuth2密码', + additional_params: '补充参数', + add_param: '添加参数', + add_extract_field: '添加提取参数', + key: '键名', + value: '键值', + + interface_config: '接口配置', + input_interface: '查询任务列表接口', + input_interface_tips: '请输入接口地址', + submit_interface: '启动任务接口', + submit_interface_tips: '请输入接口地址', + query_interface: '查询任务状态接口', + query_interface_tips: '请输入接口地址', + stop_interface: '停止任务接口', + stop_interface_tips: '请输入接口地址', + parameters: '参数', + param_location: '参数位置', + param_location_tips: '请选择参数位置', + param_name_tips: '请输入参数名', + param_value_tips: '请输入参数值', + extract_response_data: '提取响应并存储变量', + extract_field: '请输入提取参数', + json_path_list: '请输入，例如：$.data[*].id', + json_path: '请输入，例如：$.data.taskInstanceId', + header_prefix: 'Authorization Token前缀', + header_prefix_tips: 'Authorization Token前缀,e.g.Bearer', + system_field_tips: '请选择系统字段', + request_body: '请求Body', + request_body_placeholder: '请输入JSON格式的请求体', + header: '请求头', + query: '查询参数', + get: 'GET', + post: 'POST', + put: 'PUT', + interface_timeout: '接口超时时间', + interface_timeout_tips: '请输入接口超时时间', + millisecond: '毫秒', + interface_timeout_description: + '设置接口请求超时时间，默认为120000毫秒（2分钟）', + + polling_config: '轮询配置', + success_condition: '成功条件', + success_field: '成功字段', + success_field_tips: '成功字段JSONPath:$.data.status', + success_value: '成功值', + success_value_tips: '成功值枚举值:SUCCESS,FINISHED', + failure_condition: '失败条件', + failure_field: '失败字段', + failure_field_tips: '失败字段JSONPath:$.data.status', + failure_value: '失败值', + failure_value_tips: '失败值所有枚举值:CANCELED,FAILED', + + cancel: '取消', + submit: '确定', + test: '测试连接', + search: '搜索', + create: '新建', + update: '更新', + edit: '编辑', + delete: '删除', + + create_success: '创建成功', + edit_success: '编辑成功', + delete_success: '删除成功', + test_success: '测试成功', + create_failed: '创建失败', + edit_failed: '编辑失败', + delete_failed: '删除失败', + test_failed: '测试失败', + submit_failed: '提交失败，请检查表单内容', + + system_name_required: '系统名称为必填项', + service_address_required: '服务地址为必填项', + auth_type_required: '认证类型为必选项', + username_required: '用户名为必填项', + password_required: '密码为必填项', + jwt_token_required: 'JWT串为必填项', + oauth2_token_url_required: 'Token URL为必填项', + oauth2_client_id_required: 'Client ID为必填项', + oauth2_client_secret_required: 'Client Secret为必填项', + oauth2_grant_type_required: 'Grant Type为必填项', + input_interface_url_required: '入参接口地址为必填项', + submit_interface_url_required: '提交接口地址为必填项', + query_interface_url_required: '查询接口地址为必填项', + stop_interface_url_required: '停止接口地址为必填项', + success_condition_required: '成功条件字段路径和值不能为空', + failure_condition_required: '失败条件字段路径和值不能为空', + + external_system_required: '第三方系统不能为空', + external_system_task_required: '第三方系统任务不能为空', + id_jsonpath_required: '字段id以及JSONPath为必填项', + name_jsonpath_required: '字段name以及JSONPath为必填项' +} diff --git a/dolphinscheduler-ui/src/service/modules/data-source/index.ts b/dolphinscheduler-ui/src/service/modules/data-source/index.ts index f629384dac43..dfa805b25ac2 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/index.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/index.ts @@ -167,3 +167,13 @@ export function getDatasourceTableColumnsById( } }) } + +export function queryExternalSystemTasks(externalSystemId: number): any { + return axios({ + url: '/external-systems/queryExternalSystemTasks', + method: 'get', + params: { + externalSystemId + } + }) +} diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index cd6badccb4c6..9460dff3a6ca 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -44,6 +44,7 @@ type IDataBase = | 'K8S' | 'ALIYUN_SERVERLESS_SPARK' | 'DOLPHINDB' + | 'THIRDPARTY_SYSTEM_CONNECTOR' type IDataBaseLabel = | 'MYSQL' @@ -101,6 +102,94 @@ interface IDataSource { accessKeySecret?: string regionId?: string endpoint?: string + // THIRDPARTY_SYSTEM_CONNECTOR fields + serviceAddress?: string + interfaceTimeout?: number + authConfig?: { + authType: string + basicUsername?: string + basicPassword?: string + jwtToken?: string + oauth2TokenUrl?: string + oauth2ClientId?: string + oauth2ClientSecret?: string + oauth2GrantType?: string + oauth2Username?: string + oauth2Password?: string + headerPrefix?: string + authMappings?: { + key: string + value: string + }[] + } + selectInterface?: { + url: string + method: string + parameters?: { + paramName: string + paramValue: string + location: string + }[] + body: string + responseParameters?: { + key: string + jsonPath: string + disabled?: boolean + }[] + } + submitInterface?: { + url: string + method: string + parameters?: { + paramName: string + paramValue: string + location: string + }[] + body: string + responseParameters?: { + key: string + jsonPath: string + disabled?: boolean + }[] + } + pollStatusInterface?: { + url: string + method: string + parameters?: { + paramName: string + paramValue: string + location: string + }[] + body: string + pollingSuccessConfig: { + successField: string + successValue: string + } + pollingFailureConfig: { + failureField: string + failureValue: string + } + responseParameters?: { + key: string + jsonPath: string + disabled?: boolean + }[] + } + stopInterface?: { + url: string + method: string + parameters?: { + paramName: string + paramValue: string + location: string + }[] + body: string + responseParameters?: { + key: string + jsonPath: string + disabled?: boolean + }[] + } } interface ListReq { diff --git a/dolphinscheduler-ui/src/store/project/task-type.ts b/dolphinscheduler-ui/src/store/project/task-type.ts index b6de8e6e5a23..77d1ba616cd9 100644 --- a/dolphinscheduler-ui/src/store/project/task-type.ts +++ b/dolphinscheduler-ui/src/store/project/task-type.ts @@ -126,6 +126,10 @@ export const TASK_TYPES_MAP = { helperLinkDisable: true, taskExecuteType: 'STREAM' }, + EXTERNAL_SYSTEM: { + alias: 'EXTERNAL_SYSTEM', + helperLinkDisable: true + }, HIVECLI: { alias: 'HIVECLI', helperLinkDisable: true diff --git a/dolphinscheduler-ui/src/store/project/types.ts b/dolphinscheduler-ui/src/store/project/types.ts index ba7773fa5b14..bd71c9a160d3 100644 --- a/dolphinscheduler-ui/src/store/project/types.ts +++ b/dolphinscheduler-ui/src/store/project/types.ts @@ -49,6 +49,7 @@ type TaskType = | 'SAGEMAKER' | 'CHUNJUN' | 'FLINK_STREAM' + | 'EXTERNAL_SYSTEM' | 'HIVECLI' | 'DMS' | 'DATASYNC' diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx index 6f12176f7940..acfb94c86ea0 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx +++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx @@ -116,6 +116,25 @@ const DetailModal = defineComponent({ props.show && props.id && setFieldsValue(await queryById(props.id)) } ) + // Monitor authType change, update headerPrefix + watch( + () => state.detailForm.authConfig?.authType, + (newAuthType) => { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig + ) { + if (newAuthType === 'BASIC_AUTH') { + state.detailForm.authConfig.headerPrefix = 'Basic' + } else if (newAuthType === 'JWT' || newAuthType === 'OAUTH2') { + state.detailForm.authConfig.headerPrefix = 'Bearer' + } else { + state.detailForm.authConfig.headerPrefix = '' + } + } + }, + { immediate: true } + ) watch( () => props.selectType, @@ -375,7 +394,7 @@ const DetailModal = defineComponent({ placeholder={t('datasource.krb5_conf_tips')} /> - {/* 验证条件选择 */} + {/* validation */} + {/* THIRDPARTY_SYSTEM_CONNECTOR */} + {detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && ( + <> + + + + + + {{ + suffix: () => t('thirdparty_api_source.millisecond') + }} + + + + { + if (detailForm.authConfig) { + detailForm.authConfig.authType = value + } + }} + options={[ + { + label: t('thirdparty_api_source.basic_auth'), + value: 'BASIC_AUTH' + }, + { + label: t('thirdparty_api_source.oauth2'), + value: 'OAUTH2' + }, + { + label: t('thirdparty_api_source.jwt'), + value: 'JWT' + } + ]} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.headerPrefix = value + } + }} + placeholder={t( + 'thirdparty_api_source.header_prefix_tips' + )} + /> + + {detailForm.authConfig?.authType === 'BASIC_AUTH' && ( + <> + + { + if (detailForm.authConfig) { + detailForm.authConfig.basicUsername = value + } + }} + placeholder={t( + 'thirdparty_api_source.username_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.basicPassword = value + } + }} + type='password' + showPasswordOn='click' + placeholder={t( + 'thirdparty_api_source.password_tips' + )} + /> + + + )} + {detailForm.authConfig?.authType === 'OAUTH2' && ( + <> + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2TokenUrl = value + } + }} + placeholder={t( + 'thirdparty_api_source.oauth2_token_url_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2ClientId = value + } + }} + placeholder={t( + 'thirdparty_api_source.oauth2_client_id_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2ClientSecret = value + } + }} + placeholder={t( + 'thirdparty_api_source.oauth2_client_secret_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2GrantType = value + } + }} + placeholder={t( + 'thirdparty_api_source.oauth2_grant_type_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2Username = value + } + }} + placeholder={t( + 'thirdparty_api_source.oauth2_username_tips' + )} + /> + + + { + if (detailForm.authConfig) { + detailForm.authConfig.oauth2Password = value + } + }} + type='password' + showPasswordOn='click' + placeholder={t( + 'thirdparty_api_source.oauth2_password_tips' + )} + /> + + + )} + {detailForm.authConfig?.authType === 'JWT' && ( + + { + if (detailForm.authConfig) { + detailForm.authConfig.jwtToken = value + } + }} + placeholder={t( + 'thirdparty_api_source.jwt_token_tips' + )} + /> + + )} + {/* additional params */} + +
+ {/* add button */} + { + if (!detailForm.authConfig) { + detailForm.authConfig = { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] + } + } + if (!detailForm.authConfig.authMappings) { + detailForm.authConfig.authMappings = [] + } + detailForm.authConfig.authMappings.push({ + key: '', + value: '' + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_param')} + + + {/* param list */} + {detailForm.authConfig?.authMappings && + detailForm.authConfig.authMappings.map( + ( + param: { key: string; value: string }, + index: number + ) => ( +
+ + (param.key = value) + } + placeholder={t('thirdparty_api_source.key')} + style={{ width: '40%' }} + /> + + (param.value = value) + } + placeholder={t('thirdparty_api_source.value')} + style={{ width: '40%', marginLeft: '10px' }} + /> + { + detailForm.authConfig?.authMappings?.splice( + index, + 1 + ) + }} + style={{ width: '20%', marginLeft: '10px' }} + size='small' + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + +
+ { + if (detailForm.selectInterface) { + detailForm.selectInterface.url = value + } + }} + placeholder={t( + 'thirdparty_api_source.input_interface_tips' + )} + style={{ flex: 1 }} + /> + { + if (detailForm.selectInterface) { + detailForm.selectInterface.method = value + } + }} + options={[ + { + label: t('thirdparty_api_source.get'), + value: 'GET' + }, + { + label: t('thirdparty_api_source.post'), + value: 'POST' + }, + { + label: t('thirdparty_api_source.put'), + value: 'PUT' + } + ]} + style={{ width: '120px', marginLeft: '10px' }} + /> +
+ + +
+ {/* add Button*/} + { + if (!detailForm.selectInterface) { + detailForm.selectInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [] + } + } + if (!detailForm.selectInterface.parameters) { + detailForm.selectInterface.parameters = [] + } + detailForm.selectInterface.parameters.push({ + paramName: '', + paramValue: '', + location: 'HEADER' + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_param')} + + + {/* parameter list */} + {detailForm.selectInterface?.parameters && + detailForm.selectInterface.parameters.map( + ( + param: { + paramName: string + paramValue: string + location: string + }, + index: number + ) => ( +
+ + (param.location = value) + } + options={[ + { label: 'Header', value: 'HEADER' }, + { label: 'Param', value: 'PARAM' } + ]} + placeholder={t( + 'thirdparty_api_source.param_location_tips' + )} + style={{ width: '120px' }} + /> + + (param.paramName = value) + } + placeholder={t( + 'thirdparty_api_source.param_name_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + + (param.paramValue = value) + } + placeholder={t( + 'thirdparty_api_source.param_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.selectInterface?.parameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + {(detailForm.selectInterface?.method === 'POST' || + detailForm.selectInterface?.method === 'PUT') && ( + + { + if (detailForm.selectInterface) { + detailForm.selectInterface.body = value + } + }} + type='textarea' + autosize={{ + minRows: 4, + maxRows: 10 + }} + placeholder='请输入JSON格式的请求体' + /> + + )} + +
+ {/* add Button */} + { + if (!detailForm.selectInterface) { + detailForm.selectInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [] + } + } + if ( + !detailForm.selectInterface.responseParameters + ) { + detailForm.selectInterface.responseParameters = [] + } + detailForm.selectInterface.responseParameters.push({ + key: '', + jsonPath: '', + disabled: false + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_extract_field')} + + + {/* responseParameters */} + {detailForm.selectInterface?.responseParameters && + detailForm.selectInterface.responseParameters.map( + ( + param: { + key: string + jsonPath: string + disabled?: boolean + }, + index: number + ) => ( +
+ + (param.key = value) + } + placeholder={t( + 'thirdparty_api_source.extract_field' + )} + style={{ flex: 1 }} + disabled={param.disabled} + /> + + (param.jsonPath = value) + } + placeholder={t( + 'thirdparty_api_source.json_path_list' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.selectInterface?.responseParameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + +
+ { + if (detailForm.submitInterface) { + detailForm.submitInterface.url = value + } + }} + placeholder={t( + 'thirdparty_api_source.submit_interface_tips' + )} + style={{ flex: 1 }} + /> + { + if (detailForm.submitInterface) { + detailForm.submitInterface.method = value + } + }} + options={[ + { + label: t('thirdparty_api_source.get'), + value: 'GET' + }, + { + label: t('thirdparty_api_source.post'), + value: 'POST' + }, + { + label: t('thirdparty_api_source.put'), + value: 'PUT' + } + ]} + style={{ width: '120px', marginLeft: '10px' }} + /> +
+ + +
+ {/* add button */} + { + if (!detailForm.submitInterface) { + detailForm.submitInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } + if (!detailForm.submitInterface.parameters) { + detailForm.submitInterface.parameters = [] + } + detailForm.submitInterface.parameters.push({ + paramName: '', + paramValue: '', + location: 'HEADER' + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_param')} + + + {/* parameter list */} + {detailForm.submitInterface?.parameters && + detailForm.submitInterface.parameters.map( + ( + param: { + paramName: string + paramValue: string + location: string + }, + index: number + ) => ( +
+ + (param.location = value) + } + options={[ + { label: 'Header', value: 'HEADER' }, + { label: 'Param', value: 'PARAM' } + ]} + placeholder={t( + 'thirdparty_api_source.param_location_tips' + )} + style={{ width: '120px' }} + /> + + (param.paramName = value) + } + placeholder={t( + 'thirdparty_api_source.param_name_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + + (param.paramValue = value) + } + placeholder={t( + 'thirdparty_api_source.param_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.submitInterface?.parameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + {(detailForm.submitInterface?.method === 'POST' || + detailForm.submitInterface?.method === 'PUT') && ( + + { + if (detailForm.submitInterface) { + detailForm.submitInterface.body = value + } + }} + type='textarea' + autosize={{ + minRows: 4, + maxRows: 10 + }} + placeholder='请输入JSON格式的请求体' + /> + + )} + +
+ {/* add button */} + { + if (!detailForm.submitInterface) { + detailForm.submitInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } + if ( + !detailForm.submitInterface.responseParameters + ) { + detailForm.submitInterface.responseParameters = [] + } + detailForm.submitInterface.responseParameters.push({ + key: '', + jsonPath: '', + disabled: false + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_extract_field')} + + + {/* param list */} + {detailForm.submitInterface?.responseParameters && + detailForm.submitInterface.responseParameters.map( + ( + param: { + key: string + jsonPath: string + disabled?: boolean + }, + index: number + ) => ( +
+ + (param.key = value) + } + placeholder={t( + 'thirdparty_api_source.extract_field' + )} + style={{ width: '180px' }} + disabled={param.disabled} + /> + + (param.jsonPath = value) + } + placeholder={t( + 'thirdparty_api_source.json_path' + )} + style={{ width: '180px' }} + disabled={param.disabled} + /> + { + detailForm.submitInterface?.responseParameters?.splice( + index, + 1 + ) + }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + +
+ { + if (detailForm.pollStatusInterface) { + detailForm.pollStatusInterface.url = value + } + }} + placeholder={t( + 'thirdparty_api_source.query_interface_tips' + )} + style={{ flex: 1 }} + /> + { + if (detailForm.pollStatusInterface) { + detailForm.pollStatusInterface.method = value + } + }} + options={[ + { + label: t('thirdparty_api_source.get'), + value: 'GET' + }, + { + label: t('thirdparty_api_source.post'), + value: 'POST' + }, + { + label: t('thirdparty_api_source.put'), + value: 'PUT' + } + ]} + style={{ width: '120px', marginLeft: '10px' }} + /> +
+ + +
+ {/* add button */} + { + if (!detailForm.pollStatusInterface) { + detailForm.pollStatusInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + }, + responseParameters: [] + } + } + if (!detailForm.pollStatusInterface.parameters) { + detailForm.pollStatusInterface.parameters = [] + } + detailForm.pollStatusInterface.parameters.push({ + paramName: '', + paramValue: '', + location: 'HEADER' + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_param')} + + + {/* param list */} + {detailForm.pollStatusInterface?.parameters && + detailForm.pollStatusInterface.parameters.map( + ( + param: { + paramName: string + paramValue: string + location: string + }, + index: number + ) => ( +
+ + (param.location = value) + } + options={[ + { label: 'Header', value: 'HEADER' }, + { label: 'Param', value: 'PARAM' } + ]} + placeholder={t( + 'thirdparty_api_source.param_location_tips' + )} + style={{ width: '120px' }} + /> + + (param.paramName = value) + } + placeholder={t( + 'thirdparty_api_source.param_name_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + + (param.paramValue = value) + } + placeholder={t( + 'thirdparty_api_source.param_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.pollStatusInterface?.parameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + {(detailForm.pollStatusInterface?.method === 'POST' || + detailForm.pollStatusInterface?.method === 'PUT') && ( + + { + if (detailForm.pollStatusInterface) { + detailForm.pollStatusInterface.body = value + } + }} + type='textarea' + autosize={{ + minRows: 4, + maxRows: 10 + }} + placeholder='请输入JSON格式的请求体' + /> + + )} + +
+ {/* add button */} + { + if (!detailForm.pollStatusInterface) { + detailForm.pollStatusInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + }, + responseParameters: [] + } + } + if ( + !detailForm.pollStatusInterface.responseParameters + ) { + detailForm.pollStatusInterface.responseParameters = + [] + } + detailForm.pollStatusInterface.responseParameters.push( + { key: '', jsonPath: '', disabled: false } + ) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_extract_field')} + + + {/* param list */} + {detailForm.pollStatusInterface?.responseParameters && + detailForm.pollStatusInterface.responseParameters.map( + ( + param: { + key: string + jsonPath: string + disabled?: boolean + }, + index: number + ) => ( +
+ + (param.key = value) + } + placeholder={t( + 'thirdparty_api_source.extract_field' + )} + style={{ flex: 1 }} + disabled={param.disabled} + /> + + (param.jsonPath = value) + } + placeholder={t( + 'thirdparty_api_source.json_path' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.pollStatusInterface?.responseParameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + +
+ { + if ( + detailForm.pollStatusInterface + ?.pollingSuccessConfig + ) { + detailForm.pollStatusInterface.pollingSuccessConfig.successField = + value + } + }} + placeholder={t( + 'thirdparty_api_source.success_field_tips' + )} + style={{ flex: 1 }} + /> + { + if ( + detailForm.pollStatusInterface + ?.pollingSuccessConfig + ) { + detailForm.pollStatusInterface.pollingSuccessConfig.successValue = + value + } + }} + placeholder={t( + 'thirdparty_api_source.success_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> +
+ + +
+ { + if ( + detailForm.pollStatusInterface + ?.pollingFailureConfig + ) { + detailForm.pollStatusInterface.pollingFailureConfig.failureField = + value + } + }} + placeholder={t( + 'thirdparty_api_source.failure_field_tips' + )} + style={{ flex: 1 }} + /> + { + if ( + detailForm.pollStatusInterface + ?.pollingFailureConfig + ) { + detailForm.pollStatusInterface.pollingFailureConfig.failureValue = + value + } + }} + placeholder={t( + 'thirdparty_api_source.failure_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> +
+ + +
+ { + if (detailForm.stopInterface) { + detailForm.stopInterface.url = value + } + }} + placeholder={t( + 'thirdparty_api_source.stop_interface_tips' + )} + style={{ flex: 1 }} + /> + { + if (detailForm.stopInterface) { + detailForm.stopInterface.method = value + } + }} + options={[ + { + label: t('thirdparty_api_source.get'), + value: 'GET' + }, + { + label: t('thirdparty_api_source.post'), + value: 'POST' + }, + { + label: t('thirdparty_api_source.put'), + value: 'PUT' + } + ]} + style={{ width: '120px', marginLeft: '10px' }} + /> +
+ + +
+ {/* add button */} + { + if (!detailForm.stopInterface) { + detailForm.stopInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } + if (!detailForm.stopInterface.parameters) { + detailForm.stopInterface.parameters = [] + } + detailForm.stopInterface.parameters.push({ + paramName: '', + paramValue: '', + location: 'HEADER' + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_param')} + + + {/* param list */} + {detailForm.stopInterface?.parameters && + detailForm.stopInterface.parameters.map( + ( + param: { + paramName: string + paramValue: string + location: string + }, + index: number + ) => ( +
+ + (param.location = value) + } + options={[ + { label: 'Header', value: 'HEADER' }, + { label: 'Param', value: 'PARAM' } + ]} + placeholder={t( + 'thirdparty_api_source.param_location_tips' + )} + style={{ width: '120px' }} + /> + + (param.paramName = value) + } + placeholder={t( + 'thirdparty_api_source.param_name_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + + (param.paramValue = value) + } + placeholder={t( + 'thirdparty_api_source.param_value_tips' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.stopInterface?.parameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + {(detailForm.stopInterface?.method === 'POST' || + detailForm.stopInterface?.method === 'PUT') && ( + + { + if (detailForm.stopInterface) { + detailForm.stopInterface.body = value + } + }} + type='textarea' + autosize={{ + minRows: 4, + maxRows: 10 + }} + placeholder='请输入JSON格式的请求体' + /> + + )} + +
+ {/* add button */} + { + if (!detailForm.stopInterface) { + detailForm.stopInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } + if (!detailForm.stopInterface.responseParameters) { + detailForm.stopInterface.responseParameters = [] + } + detailForm.stopInterface.responseParameters.push({ + key: '', + jsonPath: '', + disabled: false + }) + }} + style={{ marginBottom: '10px' }} + > + {t('thirdparty_api_source.add_extract_field')} + + + {/* param list */} + {detailForm.stopInterface?.responseParameters && + detailForm.stopInterface.responseParameters.map( + ( + param: { + key: string + jsonPath: string + disabled?: boolean + }, + index: number + ) => ( +
+ + (param.key = value) + } + placeholder={t( + 'thirdparty_api_source.extract_field' + )} + style={{ flex: 1 }} + disabled={param.disabled} + /> + + (param.jsonPath = value) + } + placeholder={t( + 'thirdparty_api_source.json_path' + )} + style={{ flex: 1, marginLeft: '10px' }} + /> + { + detailForm.stopInterface?.responseParameters?.splice( + index, + 1 + ) + }} + style={{ marginLeft: '10px' }} + > + {t('thirdparty_api_source.delete')} + +
+ ) + )} +
+ + + )} ), diff --git a/dolphinscheduler-ui/src/views/datasource/list/types.ts b/dolphinscheduler-ui/src/views/datasource/list/types.ts index a3dc8b3c40e9..8dbc7c9cfceb 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/types.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/types.ts @@ -22,8 +22,72 @@ import type { import type { TableColumns } from 'naive-ui/es/data-table/src/interface' import type { SelectBaseOption } from 'naive-ui/es/select/src/interface' +// THIRDPARTY_SYSTEM_CONNECTOR +interface AuthMapping { + key: string + value: string +} + +interface AuthConfig { + authType: string + basicUsername?: string + basicPassword?: string + jwtToken?: string + oauth2TokenUrl?: string + oauth2ClientId?: string + oauth2ClientSecret?: string + oauth2GrantType?: string + oauth2Username?: string + oauth2Password?: string + headerPrefix?: string + authMappings?: AuthMapping[] +} + +interface InterfaceParameter { + paramName: string + paramValue: string + location: string +} + +interface ResponseParameter { + key: string + jsonPath: string + disabled?: boolean +} + +interface InterfaceConfig { + url: string + method: string + parameters: InterfaceParameter[] + body: string + responseParameters?: ResponseParameter[] +} + +interface PollingSuccessConfig { + successField: string + successValue: string +} + +interface PollingFailureConfig { + failureField: string + failureValue: string +} + +interface PollStatusInterfaceConfig extends InterfaceConfig { + pollingSuccessConfig: PollingSuccessConfig + pollingFailureConfig: PollingFailureConfig +} + interface IDataSourceDetail extends Omit { other?: string + // THIRDPARTY_SYSTEM_CONNECTOR + serviceAddress?: string + interfaceTimeout?: number + authConfig?: AuthConfig + selectInterface?: InterfaceConfig + submitInterface?: InterfaceConfig + pollStatusInterface?: PollStatusInterfaceConfig + stopInterface?: InterfaceConfig } interface IDataBaseOption extends SelectBaseOption { diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-detail.ts b/dolphinscheduler-ui/src/views/datasource/list/use-detail.ts index 3f533d00e719..9e66478485b3 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-detail.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-detail.ts @@ -38,10 +38,171 @@ export function useDetail(getFieldsValue: Function) { const formatParams = (): IDataSource => { const values = getFieldsValue() - return { + const params: IDataSource = { ...values, other: values.other ? JSON.parse(values.other) : null } + + if (values.type === 'THIRDPARTY_SYSTEM_CONNECTOR') { + if (params.authConfig?.authMappings) { + params.authConfig.authMappings = + params.authConfig?.authMappings?.filter( + (mapping: { key: string; value: string }) => + mapping.key || mapping.value + ) + } + + if (!params.selectInterface) { + params.selectInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + } + } else { + if (params.selectInterface.parameters) { + params.selectInterface.parameters = + params.selectInterface.parameters.filter( + (param: { paramName: string; paramValue: string }) => + param.paramName || param.paramValue + ) + } + + if (params.selectInterface.responseParameters) { + params.selectInterface.responseParameters = + params.selectInterface.responseParameters.filter( + (param: { key: string; jsonPath: string }) => + param.key || param.jsonPath + ) + } + } + + if (!params.submitInterface) { + params.submitInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [{ key: 'taskInstanceId', jsonPath: '' }] + } + } else { + if (params.submitInterface.parameters) { + params.submitInterface.parameters = + params.submitInterface.parameters.filter( + (param: { paramName: string; paramValue: string }) => + param.paramName || param.paramValue + ) + } + + if (params.submitInterface.responseParameters) { + params.submitInterface.responseParameters = + params.submitInterface.responseParameters.filter( + (param: { key: string; jsonPath: string }) => + param.key || param.jsonPath + ) + } + } + + if (!params.pollStatusInterface) { + params.pollStatusInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + }, + responseParameters: [] + } + } else { + if (params.pollStatusInterface.parameters) { + params.pollStatusInterface.parameters = + params.pollStatusInterface.parameters.filter( + (param: { paramName: string; paramValue: string }) => + param.paramName || param.paramValue + ) + } + + if (!params.pollStatusInterface.pollingSuccessConfig) { + params.pollStatusInterface.pollingSuccessConfig = { + successField: '', + successValue: '' + } + } + + if (!params.pollStatusInterface.pollingFailureConfig) { + params.pollStatusInterface.pollingFailureConfig = { + failureField: '', + failureValue: '' + } + } + + if (params.pollStatusInterface.responseParameters) { + params.pollStatusInterface.responseParameters = + params.pollStatusInterface.responseParameters.filter( + (param: { key: string; jsonPath: string }) => + param.key || param.jsonPath + ) + } + } + + if (!params.stopInterface) { + params.stopInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } else { + if (params.stopInterface.parameters) { + params.stopInterface.parameters = + params.stopInterface.parameters.filter( + (param: { paramName: string; paramValue: string }) => + param.paramName || param.paramValue + ) + } + + if (params.stopInterface.responseParameters) { + params.stopInterface.responseParameters = + params.stopInterface.responseParameters.filter( + (param: { key: string; jsonPath: string }) => + param.key || param.jsonPath + ) + } + } + + if (!params.authConfig) { + params.authConfig = { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] + } + } + + delete params.userName + delete params.password + } + + return params } const queryById = async (id: number) => { diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts index 9701c79381a8..4ead40b0fff5 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts @@ -51,7 +51,63 @@ export function useForm(id?: number) { endpoint: '', MSIClientId: '', dbUser: '', - datawarehouse: '' + datawarehouse: '', + // THIRDPARTY_SYSTEM_CONNECTOR + serviceAddress: 'http://', + interfaceTimeout: 120000, + authConfig: { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] as { key: string; value: string }[] + }, + selectInterface: { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + }, + submitInterface: { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [{ key: 'taskInstanceId', jsonPath: '' }] + }, + pollStatusInterface: { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [], + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + } + }, + stopInterface: { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } } as IDataSourceDetail const state = reactive({ @@ -68,13 +124,14 @@ export function useForm(id?: number) { showMode: false, showDataBaseName: true, showJDBCConnectParameters: true, - showPrivateKey: false, + showPublicKey: false, showNamespace: false, showKubeConfig: false, showAccessKeyId: false, showAccessKeySecret: false, showRegionId: false, showEndpoint: false, + showPrivateKey: false, rules: { name: { trigger: ['input'], @@ -126,12 +183,26 @@ export function useForm(id?: number) { !state.detailForm.userName && state.detailForm.type !== 'AZURESQL' && state.detailForm.type !== 'K8S' && - state.detailForm.type !== 'ALIYUN_SERVERLESS_SPARK' + state.detailForm.type !== 'ALIYUN_SERVERLESS_SPARK' && + state.detailForm.type !== 'THIRDPARTY_SYSTEM_CONNECTOR' ) { return new Error(t('datasource.user_name_tips')) } } }, + password: { + trigger: ['input'], + validator() { + if ( + !state.detailForm.password && + state.detailForm.type !== 'K8S' && + state.detailForm.type !== 'ALIYUN_SERVERLESS_SPARK' && + state.detailForm.type !== 'THIRDPARTY_SYSTEM_CONNECTOR' + ) { + return new Error(t('datasource.user_password_tips')) + } + } + }, awsRegion: { trigger: ['input'], validator() { @@ -196,15 +267,183 @@ export function useForm(id?: number) { return new Error(t('datasource.IAM-accessKey')) } } + }, + // THIRDPARTY_SYSTEM_CONNECTOR check rule + serviceAddress: { + trigger: ['input'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.serviceAddress + ) { + return new Error( + t('thirdparty_api_source.service_address_required') + ) + } + } + }, + 'authConfig.authType': { + trigger: ['change'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.authConfig?.authType + ) { + return new Error(t('thirdparty_api_source.auth_type_required')) + } + } + }, + 'authConfig.basicUsername': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'BASIC_AUTH' && + !state.detailForm.authConfig?.basicUsername + ) { + return new Error(t('thirdparty_api_source.username_required')) + } + return true + } + }, + 'authConfig.basicPassword': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'BASIC_AUTH' && + !state.detailForm.authConfig?.basicPassword + ) { + return new Error(t('thirdparty_api_source.password_required')) + } + return true + } + }, + 'authConfig.oauth2TokenUrl': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'OAUTH2' && + !state.detailForm.authConfig?.oauth2TokenUrl + ) { + return new Error( + t('thirdparty_api_source.oauth2_token_url_required') + ) + } + return true + } + }, + 'authConfig.oauth2ClientId': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'OAUTH2' && + !state.detailForm.authConfig?.oauth2ClientId + ) { + return new Error( + t('thirdparty_api_source.oauth2_client_id_required') + ) + } + return true + } + }, + 'authConfig.oauth2ClientSecret': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'OAUTH2' && + !state.detailForm.authConfig?.oauth2ClientSecret + ) { + return new Error( + t('thirdparty_api_source.oauth2_client_secret_required') + ) + } + return true + } + }, + 'authConfig.oauth2GrantType': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'OAUTH2' && + !state.detailForm.authConfig?.oauth2GrantType + ) { + return new Error( + t('thirdparty_api_source.oauth2_grant_type_required') + ) + } + return true + } + }, + 'authConfig.jwtToken': { + trigger: ['blur'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + state.detailForm.authConfig?.authType === 'JWT' && + !state.detailForm.authConfig?.jwtToken + ) { + return new Error(t('thirdparty_api_source.jwt_token_required')) + } + return true + } + }, + 'selectInterface.url': { + trigger: ['blur', 'change'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.selectInterface?.url + ) { + return new Error( + t('thirdparty_api_source.input_interface_url_required') + ) + } + } + }, + 'submitInterface.url': { + trigger: ['blur', 'change'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.submitInterface?.url + ) { + return new Error( + t('thirdparty_api_source.submit_interface_url_required') + ) + } + } + }, + 'pollStatusInterface.url': { + trigger: ['blur', 'change'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.pollStatusInterface?.url + ) { + return new Error( + t('thirdparty_api_source.query_interface_url_required') + ) + } + } + }, + 'stopInterface.url': { + trigger: ['blur', 'change'], + validator() { + if ( + state.detailForm.type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.stopInterface?.url + ) { + return new Error( + t('thirdparty_api_source.stop_interface_url_required') + ) + } + } } - // databaseUserName: { - // trigger: ['input'], - // validator() { - // if (!state.detailForm.userName) { - // return new Error(t('datasource.user_name_tips')) - // } - // } - // }, } as FormRules, modeOptions: [ { @@ -277,18 +516,19 @@ export function useForm(id?: number) { type === 'SAGEMAKER' || type === 'K8S' || type === 'ALIYUN_SERVERLESS_SPARK' || - type === 'DOLPHINDB' + type === 'DOLPHINDB' || + type === 'THIRDPARTY_SYSTEM_CONNECTOR' ) { state.showDataBaseName = false state.requiredDataBase = false state.showJDBCConnectParameters = false - state.showPrivateKey = false + state.showPublicKey = false if (type === 'DOLPHINDB') { state.showJDBCConnectParameters = true - state.showPrivateKey = false + state.showPublicKey = false } if (type === 'SSH') { - state.showPrivateKey = true + state.showPublicKey = true } if (type === 'ZEPPELIN') { state.showHost = false @@ -300,7 +540,8 @@ export function useForm(id?: number) { if ( type === 'SAGEMAKER' || type === 'K8S' || - type == 'ALIYUN_SERVERLESS_SPARK' + type == 'ALIYUN_SERVERLESS_SPARK' || + type === 'THIRDPARTY_SYSTEM_CONNECTOR' ) { state.showHost = false state.showPort = false @@ -323,11 +564,60 @@ export function useForm(id?: number) { state.showRegionId = false state.showEndpoint = false } + // 处理THIRDPARTY_SYSTEM_CONNECTOR特殊字段显示 + if (type === 'THIRDPARTY_SYSTEM_CONNECTOR') { + state.showHost = false + state.showPort = false + state.showJDBCConnectParameters = false + state.showDataBaseName = false + state.requiredDataBase = false + + // make sure authConfig \ authMappings inited + if (!state.detailForm.authConfig) { + state.detailForm.authConfig = { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] + } + } else if (!state.detailForm.authConfig?.authMappings) { + state.detailForm.authConfig.authMappings = [] + } + } + + // init THIRDPARTY_SYSTEM_CONNECTOR authConfig + if ( + type === 'THIRDPARTY_SYSTEM_CONNECTOR' && + !state.detailForm.authConfig + ) { + state.detailForm.authConfig = { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] + } + } } else { state.showDataBaseName = true state.requiredDataBase = true state.showJDBCConnectParameters = true - state.showPrivateKey = false + state.showPublicKey = false state.showRestEndpoint = false state.showNamespace = false state.showKubeConfig = false @@ -349,10 +639,201 @@ export function useForm(id?: number) { } const setFieldsValue = (values: IDataSource) => { + const processedValues = { + ...values, + other: values.other ? JSON.stringify(values.other) : values.other, + selectInterface: values.selectInterface + ? { + ...values.selectInterface, + parameters: values.selectInterface.parameters || [], + responseParameters: values.selectInterface.responseParameters || [] + } + : undefined, + submitInterface: values.submitInterface + ? { + ...values.submitInterface, + parameters: values.submitInterface.parameters || [], + responseParameters: values.submitInterface.responseParameters || [] + } + : undefined, + pollStatusInterface: values.pollStatusInterface + ? { + ...values.pollStatusInterface, + parameters: values.pollStatusInterface.parameters || [], + responseParameters: + values.pollStatusInterface.responseParameters || [] + } + : undefined, + stopInterface: values.stopInterface + ? { + ...values.stopInterface, + parameters: values.stopInterface.parameters || [], + responseParameters: values.stopInterface.responseParameters || [] + } + : undefined + } + state.detailForm = { ...state.detailForm, - ...values, - other: values.other ? JSON.stringify(values.other) : values.other + ...processedValues + } + + // check THIRDPARTY_SYSTEM_CONNECTOR has right authConfig + if (values.type === 'THIRDPARTY_SYSTEM_CONNECTOR') { + if (!state.detailForm.authConfig) { + state.detailForm.authConfig = { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] + } + } else { + state.detailForm.authConfig = { + authType: values.authConfig?.authType || 'BASIC_AUTH', + basicUsername: values.authConfig?.basicUsername || '', + basicPassword: values.authConfig?.basicPassword || '', + jwtToken: values.authConfig?.jwtToken || '', + oauth2TokenUrl: values.authConfig?.oauth2TokenUrl || '', + oauth2ClientId: values.authConfig?.oauth2ClientId || '', + oauth2ClientSecret: values.authConfig?.oauth2ClientSecret || '', + oauth2GrantType: values.authConfig?.oauth2GrantType || '', + oauth2Username: values.authConfig?.oauth2Username || '', + oauth2Password: values.authConfig?.oauth2Password || '', + headerPrefix: values.authConfig?.headerPrefix || 'Basic', + authMappings: values.authConfig?.authMappings + ? [...values.authConfig.authMappings] + : [] + } + } + + if (!state.detailForm.selectInterface) { + state.detailForm.selectInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + responseParameters: [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + } + } else { + state.detailForm.selectInterface = { + url: values.selectInterface?.url || '', + method: values.selectInterface?.method || 'GET', + parameters: values.selectInterface?.parameters + ? [...values.selectInterface.parameters] + : [], + body: values.selectInterface?.body || '', + responseParameters: values.selectInterface?.responseParameters + ? [...values.selectInterface.responseParameters] + : [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + } + } + + if (!state.detailForm.submitInterface) { + state.detailForm.submitInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [{ key: 'taskInstanceId', jsonPath: '' }] + } + } else { + state.detailForm.submitInterface = { + url: values.submitInterface?.url || '', + method: values.submitInterface?.method || 'POST', + parameters: values.submitInterface?.parameters + ? [...values.submitInterface.parameters] + : [], + body: values.submitInterface?.body || '', + responseParameters: values.submitInterface?.responseParameters + ? [...values.submitInterface.responseParameters] + : [{ key: 'taskInstanceId', jsonPath: '' }] + } + } + + if (!state.detailForm.pollStatusInterface) { + state.detailForm.pollStatusInterface = { + url: '', + method: 'GET', + parameters: [], + body: '', + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + }, + responseParameters: [] + } + } else { + state.detailForm.pollStatusInterface = { + url: values.pollStatusInterface?.url || '', + method: values.pollStatusInterface?.method || 'GET', + parameters: values.pollStatusInterface?.parameters + ? [...values.pollStatusInterface.parameters] + : [], + body: values.pollStatusInterface?.body || '', + pollingSuccessConfig: { + successField: + values.pollStatusInterface?.pollingSuccessConfig?.successField || + '', + successValue: + values.pollStatusInterface?.pollingSuccessConfig?.successValue || + '' + }, + pollingFailureConfig: { + failureField: + values.pollStatusInterface?.pollingFailureConfig?.failureField || + '', + failureValue: + values.pollStatusInterface?.pollingFailureConfig?.failureValue || + '' + }, + responseParameters: values.pollStatusInterface?.responseParameters + ? [...values.pollStatusInterface.responseParameters] + : [] + } + } + + if (!state.detailForm.stopInterface) { + state.detailForm.stopInterface = { + url: '', + method: 'POST', + parameters: [], + body: '', + responseParameters: [] + } + } else { + state.detailForm.stopInterface = { + url: values.stopInterface?.url || '', + method: values.stopInterface?.method || 'POST', + parameters: values.stopInterface?.parameters + ? [...values.stopInterface.parameters] + : [], + body: values.stopInterface?.body || '', + responseParameters: values.stopInterface?.responseParameters + ? [...values.stopInterface.responseParameters] + : [] + } + } + + delete state.detailForm.userName + delete state.detailForm.password } } @@ -508,6 +989,11 @@ export const datasourceType: IDataBaseOptionKeys = { value: 'DOLPHINDB', label: 'DOLPHINDB', defaultPort: 8848 + }, + THIRDPARTY_SYSTEM_CONNECTOR: { + value: 'THIRDPARTY_SYSTEM_CONNECTOR', + label: 'THIRDPARTY_SYSTEM_CONNECTOR', + defaultPort: 80 } } diff --git a/dolphinscheduler-ui/src/views/projects/components/dependencies/dependencies-modal.tsx b/dolphinscheduler-ui/src/views/projects/components/dependencies/dependencies-modal.tsx index 207673a2a837..e405f720a346 100644 --- a/dolphinscheduler-ui/src/views/projects/components/dependencies/dependencies-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/components/dependencies/dependencies-modal.tsx @@ -59,7 +59,7 @@ export default defineComponent({ } const cancelToHandle = () => { - ctx.emit('update:show', showRef) + ctx.emit('update:show', showRef.value) } const renderDownstreamDependencies = () => { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts index 43c1ed66d138..946965aedf88 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts @@ -51,6 +51,7 @@ export { useMainJar } from './use-main-jar' export { useResources } from './use-resources' export { useTaskDefinition } from './use-task-definition' export { useJavaTaskMainJar } from './use-java-task-main-jar' +export { useExternalSystem } from './use-external-system' export { useShell } from './use-shell' export { useSpark } from './use-spark' diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts index 950c7418eaa2..bfaaf8ef22cd 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts @@ -34,7 +34,9 @@ export function useDatasource( const { t } = useI18n() const options = ref([] as { label: string; value: string }[]) - const datasourceOptions = ref([] as { label: string; value: number }[]) + const datasourceOptions = ref( + [] as { label: string; value: number | string }[] + ) const datasourceTypes = [ { @@ -166,6 +168,11 @@ export function useDatasource( id: 28, code: 'DOLPHINDB', disabled: false + }, + { + id: 29, + code: 'THIRDPARTY_SYSTEM_CONNECTOR', + disabled: false } ] diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-external-system.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-external-system.ts new file mode 100644 index 000000000000..0a11ba7edd03 --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-external-system.ts @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ref, onMounted, nextTick, Ref } from 'vue' +import { useI18n } from 'vue-i18n' +import { + queryDataSourceList, + queryExternalSystemTasks +} from '@/service/modules/data-source' +import { find } from 'lodash' +import type { IJsonItem } from '../types' +import type { + TypeReq, + IDataBase as unusedIDataBase +} from '@/service/modules/data-source/types' + +export function useExternalSystem( + model: { [field: string]: any }, + params: { + externalSystemField?: string + taskField?: string + span?: Ref | number + } = {} +): IJsonItem[] { + const { t } = useI18n() + + const datasourceOptions = ref([] as { label: string; value: string }[]) + const taskOptions = ref([] as { label: string; value: string }[]) + + const getDataSources = async () => { + try { + const parameters = { + type: 'THIRDPARTY_SYSTEM_CONNECTOR' + } as TypeReq + + const res = await queryDataSourceList(parameters) + datasourceOptions.value = res.map((item: any) => ({ + label: item.name, + value: String(item.id) + })) + } catch (error) { + // Error handling is done by the calling function + } + } + + const refreshTasks = async () => { + const datasourceId = model[params.externalSystemField || 'datasource'] + if (!datasourceId) return + + try { + const res = await queryExternalSystemTasks(datasourceId) + taskOptions.value = res.map((item: any) => ({ + label: item.name, + value: String(item.id) + })) + } catch (error) { + // Error handling is done by the calling function + } + + const taskField = params.taskField || 'task' + if (!taskOptions.value.length && model[taskField]) model[taskField] = null + if (taskOptions.value.length && model[taskField]) { + const item = find(taskOptions.value, { value: model[taskField] }) + if (!item) { + model[taskField] = null + } + } + } + + const onChange = () => { + taskOptions.value = [] + const taskField = params.taskField || 'externalTaskId' + model[taskField] = null + model.externalTaskName = '' + refreshTasks() + } + + const onTaskChange = (value: string) => { + if (value) { + const taskItem = taskOptions.value.find((item) => item.value === value) + if (taskItem) { + model.externalTaskName = taskItem.label // Set the name based on the selected task + } + } else { + model.externalTaskName = '' + } + } + + onMounted(async () => { + await getDataSources() + await nextTick() + refreshTasks() + }) + + return [ + { + type: 'select', + field: params.externalSystemField || 'datasource', + span: params.span || 24, + name: t('project.node.datasource_instances'), + props: { 'on-update:value': onChange }, + options: datasourceOptions, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(unuse: any, value) { + if (!value) { + return new Error(t('project.node.datasource_instances_required')) + } + } + } + }, + { + type: 'select', + field: params.taskField || 'externalTaskId', + span: params.span || 24, + name: t('project.node.external_system_tasks'), + props: { 'on-update:value': onTaskChange }, + options: taskOptions, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(unuse: any, value) { + if (!value) { + return new Error( + t('thirdparty_api_source.external_system_task_required') + ) + } + } + } + } + ] +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 8fc67906775c..17a5f76a817f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -115,6 +115,12 @@ export function formatParams(data: INodeData): { taskParams.connectTimeout = data.connectTimeout taskParams.socketTimeout = data.socketTimeout } + if (data.taskType === 'EXTERNAL_SYSTEM') { + taskParams.type = data.type + taskParams.datasource = data.datasource + taskParams.externalTaskId = data.externalTaskId + taskParams.externalTaskName = data.externalTaskName + } if (data.taskType === 'SQOOP') { taskParams.jobType = data.isCustomTask ? 'CUSTOM' : 'TEMPLATE' diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts index 3db6bb96ea5b..3380678cf1fa 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts @@ -31,6 +31,7 @@ import { useSeaTunnel } from './use-sea-tunnel' import { useSwitch } from './use-switch' import { useConditions } from './use-conditions' import { useDataX } from './use-datax' +import { useExternalSystem } from './use-external-system' import { useDependent } from './use-dependent' import { useEmr } from './use-emr' import { useZeppelin } from './use-zeppelin' @@ -80,6 +81,7 @@ export default { SAGEMAKER: userSagemaker, CHUNJUN: useChunjun, FLINK_STREAM: useFlinkStream, + EXTERNAL_SYSTEM: useExternalSystem, JAVA: useJava, HIVECLI: useHiveCli, DMS: useDms, diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-external-system.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-external-system.ts new file mode 100644 index 000000000000..52a0b2b435cd --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-external-system.ts @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive } from 'vue' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData, ITaskData } from '../types' + +export function useExternalSystem({ + projectCode, + from = 0, + readonly, + data +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData +}) { + const model = reactive({ + taskType: 'EXTERNAL_SYSTEM', + name: '', + flag: 'YES', + description: '', + timeoutFlag: false, + timeoutNotifyStrategy: ['WARN'], + timeout: 30, + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + cpuQuota: -1, + memoryMax: -1, + delayTime: 0, + datasource: '', + type: 'THIRDPARTY_SYSTEM_CONNECTOR', + externalTaskId: '', + externalTaskName: '' + } as INodeData) + + return { + json: [ + Fields.useName(from), + ...Fields.useTaskDefinition({ projectCode, from, readonly, data, model }), + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(projectCode), + Fields.useEnvironmentName(model, !data?.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + ...Fields.useResourceLimit(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useExternalSystem(model), + Fields.usePreTasks() + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 76dcde23ee2d..e69122c507b6 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -302,6 +302,8 @@ interface ITaskParams { connectTimeout?: number socketTimeout?: number type?: string + externalTaskName?: string + externalTaskId?: string datasource?: string sql?: string sqlType?: string diff --git a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts index ce0db268f4f8..79045898fcff 100644 --- a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts @@ -51,6 +51,7 @@ export type TaskType = | 'DATA_FACTORY' | 'REMOTESHELL' | 'ALIYUN_SERVERLESS_SPARK' + | 'EXTERNAL_SYSTEM' export type TaskExecuteType = 'STREAM' | 'BATCH' @@ -155,6 +156,10 @@ export const TASK_TYPES_MAP = { helperLinkDisable: true, taskExecuteType: 'STREAM' }, + EXTERNAL_SYSTEM: { + alias: 'EXTERNAL_SYSTEM', + helperLinkDisable: true + }, HIVECLI: { alias: 'HIVECLI', helperLinkDisable: true diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss index 809bc6fae3cf..d2b5e81f93b6 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss @@ -117,6 +117,9 @@ $bgLight: #ffffff; &.icon-flink_stream { background-image: url('/images/task-icons/flink.png'); } + &.icon-external_system { + background-image: url('/images/task-icons/external_system.png'); + } &.icon-mr { background-image: url('/images/task-icons/mr.png'); } @@ -227,6 +230,9 @@ $bgLight: #ffffff; &.icon-flink_stream { background-image: url('/images/task-icons/flink_hover.png'); } + &.icon-external_system { + background-image: url('/images/task-icons/external_system_hover.png'); + } &.icon-mr { background-image: url('/images/task-icons/mr_hover.png'); } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx index 26d6540eb6b3..8bf9fa90f27a 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx @@ -157,6 +157,13 @@ export default defineComponent({ taskType: 'FLINK_STREAM', color: '#d68f5b', image: `${import.meta.env.BASE_URL}images/task-icons/flink.png` + }, + { + taskType: 'EXTERNAL_SYSTEM', + color: '#d68f5b', + image: `${ + import.meta.env.BASE_URL + }images/task-icons/external_system.png` } ]) diff --git a/dolphinscheduler-ui/src/views/thirdparty-api-source/index.module.scss b/dolphinscheduler-ui/src/views/thirdparty-api-source/index.module.scss new file mode 100644 index 000000000000..d32b36e059c6 --- /dev/null +++ b/dolphinscheduler-ui/src/views/thirdparty-api-source/index.module.scss @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.thirdparty-modal { + position: relative; + width: 700px; + max-height: 100vh; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.thirdparty-modal :global(.n-modal__close), +.thirdparty-modal :global(.n-modal-header__close), +.thirdparty-modal :global(.n-modal-header .n-button), +.thirdparty-modal :global(.n-modal-header .n-icon), +.thirdparty-modal :global(.n-modal-header .n-base-icon), +.thirdparty-modal :global(.n-modal-header .n-button--icon-only), +.thirdparty-modal :global(.n-base-close), +.thirdparty-modal :global(.n-base-close--absolute), +.thirdparty-modal :global(.n-card-header__close) { + display: none !important; +} + +.modal-content { + flex: 1; + max-height: calc(90vh - 120px); + overflow-y: auto; + overflow-x: hidden; + width: 100%; + padding: 0 16px; + box-sizing: border-box; +} + +.param-location { + width: 100px; + min-width: 100px; +} + +.param-name { + width: 150px; + min-width: 150px; +} + +.param-value { + width: 180px; + min-width: 180px; +} + +.method-select, +.auth-type-select { + width: 120px; + min-width: 120px; +} + +.condition-field { + width: calc(50% - 4px); + min-width: 200px; +} + +.condition-value { + width: calc(50% - 4px); + min-width: 200px; + margin-left: 8px; +} + +.submit-url { + flex: 1; + min-width: 300px; +} + +.submit-method { + width: 120px; + min-width: 120px; +} + +.timeout-input { + width: 200px; +} + +.timeout-description { + font-size: 12px; + color: #999; + margin-top: 4px; + line-height: 1.4; +} + +.key-input, +.value-input { + width: 200px; + min-width: 150px; +} + +.modal-footer { + position: sticky; + bottom: 0; + background: #fff; + padding-top: 16px; + z-index: 10; + display: flex; + justify-content: flex-end; + border-top: 1px solid #f0f0f0; + margin-top: 16px; + box-sizing: border-box; + width: 100%; +} + +.modal-footer :global(.n-space) { + margin-right: 15px; + gap: 8px; +} + +.modal-footer :global(.n-button) { + min-width: 80px; + height: 32px; + padding: 0 14px; +} + +@media screen and (min-width: 1920px) { + .thirdparty-modal { + width: 700px; + } + + .modal-content { + max-height: calc(90vh - 120px); + } +} \ No newline at end of file diff --git a/dolphinscheduler-ui/src/views/thirdparty-api-source/modal.tsx b/dolphinscheduler-ui/src/views/thirdparty-api-source/modal.tsx new file mode 100644 index 000000000000..9f7b9dfd2995 --- /dev/null +++ b/dolphinscheduler-ui/src/views/thirdparty-api-source/modal.tsx @@ -0,0 +1,1499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineComponent, PropType, reactive, watch, computed, ref } from 'vue' +import type { FormInst } from 'naive-ui' +import { useI18n } from 'vue-i18n' +import styles from './index.module.scss' +import { + NModal, + NForm, + NFormItem, + NInput, + NInputNumber, + NButton, + NSpace, + NDivider, + NDynamicInput, + NSelect, + NTooltip, + NIcon +} from 'naive-ui' +import MonacoEditor from '@/components/monaco-editor' +import { InfoCircleOutlined } from '@vicons/antd' + +export default defineComponent({ + name: 'ThirdpartyApiSourceModal', + props: { + show: Boolean, + data: { + type: Object as PropType, + default: () => null + }, + operationType: { + type: String as PropType<'create' | 'edit'>, + default: 'create' + } + }, + emits: ['close', 'submit', 'test'], + setup(props, { emit }) { + const { t } = useI18n() + + const authTypeOptions = computed(() => [ + { label: t('thirdparty_api_source.basic_auth'), value: 'BASIC_AUTH' }, + { label: t('thirdparty_api_source.oauth2'), value: 'OAUTH2' }, + { label: t('thirdparty_api_source.jwt'), value: 'JWT' } + ]) + + const methodOptions = computed(() => [ + { label: t('thirdparty_api_source.get'), value: 'GET' }, + { label: t('thirdparty_api_source.post'), value: 'POST' }, + { label: t('thirdparty_api_source.put'), value: 'PUT' } + ]) + + const form = reactive({ + serviceAddress: '', + interfaceTimeout: 120000, // default 2 min + authConfig: { + authType: 'BASIC_AUTH', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + headerPrefix: 'Basic', + authMappings: [] as any[] + }, + selectInterface: { + url: '', + method: 'GET', + parameters: [] as any[], + body: '', + responseParameters: [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + }, + submitInterface: { + url: '', + method: 'POST', + parameters: [] as any[], + body: '', + responseParameters: [{ key: 'taskInstanceId', jsonPath: '' }] + }, + pollStatusInterface: { + url: '', + method: 'GET', + parameters: [] as any[], + body: '', + pollingSuccessConfig: { + successField: '', + successValue: '' + }, + pollingFailureConfig: { + failureField: '', + failureValue: '' + } + }, + stopInterface: { + url: '', + method: 'POST', + parameters: [] as any[], + body: '' + } + }) + + // Form validation rules + const rules = { + serviceAddress: [ + { + required: true, + message: t('thirdparty_api_source.service_address_required'), + trigger: 'blur' + } + ], + 'authConfig.authType': [ + { + required: true, + message: t('thirdparty_api_source.auth_type_required'), + trigger: 'change' + } + ], + 'authConfig.basicUsername': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'BASIC_AUTH' && !value) { + return new Error(t('thirdparty_api_source.username_required')) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.basicPassword': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'BASIC_AUTH' && !value) { + return new Error(t('thirdparty_api_source.password_required')) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.oauth2TokenUrl': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'OAUTH2' && !value) { + return new Error( + t('thirdparty_api_source.oauth2_token_url_required') + ) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.oauth2ClientId': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'OAUTH2' && !value) { + return new Error( + t('thirdparty_api_source.oauth2_client_id_required') + ) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.oauth2ClientSecret': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'OAUTH2' && !value) { + return new Error( + t('thirdparty_api_source.oauth2_client_secret_required') + ) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.oauth2GrantType': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'OAUTH2' && !value) { + return new Error( + t('thirdparty_api_source.oauth2_grant_type_required') + ) + } + return true + }, + trigger: 'blur' + } + ], + 'authConfig.jwtToken': [ + { + validator: (rule: any, value: any) => { + if (form.authConfig.authType === 'JWT' && !value) { + return new Error(t('thirdparty_api_source.jwt_token_required')) + } + return true + }, + trigger: 'blur' + } + ], + 'selectInterface.url': [ + { + required: true, + message: t('thirdparty_api_source.input_interface_url_required'), + trigger: ['blur', 'change'] + } + ], + 'submitInterface.url': [ + { + required: true, + message: t('thirdparty_api_source.submit_interface_url_required'), + trigger: ['blur', 'change'] + } + ], + 'pollStatusInterface.url': [ + { + required: true, + message: t('thirdparty_api_source.query_interface_url_required'), + trigger: ['blur', 'change'] + } + ], + 'stopInterface.url': [ + { + required: true, + message: t('thirdparty_api_source.stop_interface_url_required'), + trigger: ['blur', 'change'] + } + ], + 'pollStatusInterface.pollingSuccessConfig': [ + { + validator: (rule: any, value: any) => { + if (!value.successField || !value.successValue) { + return new Error( + t('thirdparty_api_source.success_condition_required') + ) + } + return true + }, + trigger: ['blur', 'change'] + } + ], + 'pollStatusInterface.pollingFailureConfig': [ + { + validator: (rule: any, value: any) => { + if (!value.failureField || !value.failureValue) { + return new Error( + t('thirdparty_api_source.failure_condition_required') + ) + } + return true + }, + trigger: ['blur', 'change'] + } + ], + 'selectInterface.responseParameters': [ + { + validator: (rule: any, value: any) => { + const idField = value.find((item: any) => item.key === 'id') + const nameField = value.find((item: any) => item.key === 'name') + + if (!idField || !idField.jsonPath) { + return new Error(t('thirdparty_api_source.id_jsonpath_required')) + } + if (!nameField || !nameField.jsonPath) { + return new Error( + t('thirdparty_api_source.name_jsonpath_required') + ) + } + return true + }, + trigger: ['blur', 'change'] + } + ] + } + + const formRef = ref(null) + const isEditMode = computed(() => props.operationType === 'edit') + + // Define the initial state of the form + const getInitialFormState = () => ({ + serviceAddress: 'http://', + interfaceTimeout: 120000, // default 2 minutes + authConfig: { + authType: '', + headerPrefix: '', + basicUsername: '', + basicPassword: '', + jwtToken: '', + oauth2TokenUrl: '', + oauth2ClientId: '', + oauth2ClientSecret: '', + oauth2GrantType: '', + oauth2Username: '', + oauth2Password: '', + authMappings: [] + }, + selectInterface: { + url: '', + method: 'GET', + parameters: [] as any[], + body: '', + responseParameters: [ + { key: 'id', jsonPath: '' }, + { key: 'name', jsonPath: '' } + ] + }, + submitInterface: { + url: '', + method: 'POST', + parameters: [] as any[], + body: '', + responseParameters: [{ key: 'taskInstanceId', jsonPath: '' }] + }, + pollStatusInterface: { + url: '', + method: 'GET', + parameters: [] as any[], + body: '', + pollingSuccessConfig: { successField: '', successValue: '' }, + pollingFailureConfig: { failureField: '', failureValue: '' } + }, + stopInterface: { + url: '', + method: 'POST', + parameters: [] as any[], + body: '' + } + }) + + // Function to reset form data + const resetForm = () => { + const initialState = getInitialFormState() + Object.keys(form).forEach((key) => { + delete (form as any)[key] + }) + Object.assign(form, initialState) + formRef.value?.restoreValidation?.() + } + + // Save original edit data for testing connection + const originalEditData = ref(null) + // Listen for modal visibility and data changes + watch( + [() => props.show, () => props.data, () => props.operationType], + ([show, data, operationType]) => { + if (show) { + if (data && operationType === 'edit') { + originalEditData.value = JSON.parse(JSON.stringify(data)) + resetForm() + const editData = originalEditData.value + // Use data returned from backend completely + Object.assign(form, editData) + } else { + originalEditData.value = null + resetForm() + // Only set default values in create mode + form.authConfig.authType = 'BASIC_AUTH' + form.authConfig.headerPrefix = 'Basic' + } + } + }, + { immediate: true } + ) + + watch( + () => form.authConfig.authType, + (newAuthType) => { + // Only automatically set headerPrefix in create mode + if (!isEditMode.value) { + if (newAuthType === 'BASIC_AUTH') { + form.authConfig.headerPrefix = 'Basic' + } else if (newAuthType === 'JWT' || newAuthType === 'OAUTH2') { + form.authConfig.headerPrefix = 'Bearer' + } else { + form.authConfig.headerPrefix = '' + } + } + } + ) + + const handleClose = () => { + resetForm() + emit('close') + } + + const handleSubmit = () => { + ;(formRef.value as any)?.validate((errors: any) => { + if (!errors) { + if (isEditMode.value && originalEditData.value) { + const submitData = JSON.parse( + JSON.stringify(originalEditData.value) + ) + const initialState = getInitialFormState() + Object.keys(initialState).forEach((key) => { + if (form.hasOwnProperty(key)) { + submitData[key] = (form as any)[key] + } + }) + emit('submit', submitData) + } else { + emit('submit', JSON.parse(JSON.stringify(form))) + } + } + }) + } + + const handleTest = () => { + ;(formRef.value as any)?.validate((errors: any) => { + if (!errors) { + if (isEditMode.value && originalEditData.value) { + const testData = JSON.parse(JSON.stringify(originalEditData.value)) + const initialState = getInitialFormState() + Object.keys(initialState).forEach((key) => { + if (form.hasOwnProperty(key)) { + testData[key] = (form as any)[key] + } + }) + emit('test', testData) + } else { + emit('test', JSON.parse(JSON.stringify(form))) + } + } + }) + } + + // Location dropdown options linked with method + const getLocationOptions = (unusedMethod: string) => { + return [ + { label: 'Header', value: 'HEADER' }, + { label: 'Param', value: 'PARAM' } + ] + } + + return () => ( + +
+ + + + + + {/* Interface timeout */} + + + {{ + suffix: () => t('thirdparty_api_source.millisecond') + }} + +
+ {t('thirdparty_api_source.interface_timeout_description')} +
+ + + + + {/* authType */} + + {{ + label: () => ( + + {t('thirdparty_api_source.auth_type')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.auth_type_detail_info') + }} + + + ), + default: () => ( + + ) + }} + + + + + + + {/* BASIC_AUTH */} + + + + + + + + {/* OAUTH2 */} + + {{ + label: () => ( + + {t('thirdparty_api_source.oauth2_token_url')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.oauth2_url_info') + }} + + + ), + default: () => ( + + ) + }} + + + + + + + + + + + + + + + + + + + {/* JWT */} + + + + + {/* additional params */} + + ({ key: '', value: '' })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { key: string; value: string } + }) => ( + + + + + ) + }} + + + + + + {/* selectInterface */} + + {{ + label: () => ( + + {t('thirdparty_api_source.input_interface')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.input_interface_detail_info') + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + + + ) + }} + + + + ({ + paramName: '', + paramValue: '', + location: 'HEADER' + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + paramName: string + paramValue: string + location: string + } + }) => ( + + + + + + ) + }} + + + + {(form.selectInterface.method === 'POST' || + form.selectInterface.method === 'PUT') && ( + + {{ + label: () => ( + + {t('thirdparty_api_source.request_body')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.input_interface_body_info') + }} + + + ), + default: () => ( + + ) + }} + + )} + + + {{ + label: () => ( + + + {t('thirdparty_api_source.extract_response_data')} + + + {{ + trigger: () => ( + + + + ), + default: () => + t( + 'thirdparty_api_source.input_interface_extract_info' + ) + }} + + + ), + default: () => ( + ({ + key: '', + jsonPath: '', + disabled: false + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + key: string + jsonPath: string + disabled: boolean + } + }) => ( + + + + + ) + }} + + ) + }} + + + + + {/* submitInterface */} + + {{ + label: () => ( + + {t('thirdparty_api_source.submit_interface')} + + {{ + trigger: () => ( + + + + ), + default: () => + t( + 'thirdparty_api_source.submit_interface_detail_info' + ) + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + + + ) + }} + + + + ({ + paramName: '', + paramValue: '', + location: 'HEADER' + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + paramName: string + paramValue: string + location: string + } + }) => ( + + + + + + ) + }} + + + + {(form.submitInterface.method === 'POST' || + form.submitInterface.method === 'PUT') && ( + + {{ + label: () => ( + + {t('thirdparty_api_source.request_body')} + + {{ + trigger: () => ( + + + + ), + default: () => + t( + 'thirdparty_api_source.submit_interface_body_info' + ) + }} + + + ), + default: () => ( + + ) + }} + + )} + + + {{ + label: () => ( + + + {t('thirdparty_api_source.extract_response_data')} + + + {{ + trigger: () => ( + + + + ), + default: () => + t( + 'thirdparty_api_source.submit_interface_extract_info' + ) + }} + + + ), + default: () => ( + ({ + key: '', + jsonPath: '', + disabled: false + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + key: string + jsonPath: string + disabled: boolean + } + }) => ( + + + + + ) + }} + + ) + }} + + + + + {/* pollStatusInterface */} + + {{ + label: () => ( + + {t('thirdparty_api_source.query_interface')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.query_interface_detail_info') + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + + + ) + }} + + + + ({ + paramName: '', + paramValue: '', + location: 'HEADER' + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + paramName: string + paramValue: string + location: string + } + }) => ( + + + + + + ) + }} + + + + {(form.pollStatusInterface.method === 'POST' || + form.pollStatusInterface.method === 'PUT') && ( + + {{ + label: () => ( + + {t('thirdparty_api_source.request_body')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.query_interface_body_info') + }} + + + ), + default: () => ( + + ) + }} + + )} + + + {{ + label: () => ( + + {t('thirdparty_api_source.success_condition')} + + {{ + trigger: () => ( + + + + ), + default: () => + t( + 'thirdparty_api_source.query_interface_success_info' + ) + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + formRef.value?.validate?.()} + /> + + ) + }} + + + + {{ + label: () => ( + + {t('thirdparty_api_source.failure_condition')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.query_interface_failed_info') + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + formRef.value?.validate?.()} + /> + + ) + }} + + + + + {/* stopInterface */} + + {{ + label: () => ( + + {t('thirdparty_api_source.stop_interface')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.stop_interface_detail_info') + }} + + + ), + default: () => ( + <> + formRef.value?.validate?.()} + /> + + + ) + }} + + + + ({ + paramName: '', + paramValue: '', + location: 'HEADER' + })} + style={{ width: '100%' }} + > + {{ + default: ({ + value + }: { + value: { + paramName: string + paramValue: string + location: string + } + }) => ( + + + + + + ) + }} + + + + {(form.stopInterface.method === 'POST' || + form.stopInterface.method === 'PUT') && ( + + {{ + label: () => ( + + {t('thirdparty_api_source.request_body')} + + {{ + trigger: () => ( + + + + ), + default: () => + t('thirdparty_api_source.stop_interface_body_info') + }} + + + ), + default: () => ( + + ) + }} + + )} + +
+ +
+ + + {t('thirdparty_api_source.cancel')} + + + {t('thirdparty_api_source.test')} + + + {t('thirdparty_api_source.submit')} + + +
+ + ) + } +}) diff --git a/dolphinscheduler-ui/src/views/thirdparty-api-source/use-table.ts b/dolphinscheduler-ui/src/views/thirdparty-api-source/use-table.ts new file mode 100644 index 000000000000..976bc55ca283 --- /dev/null +++ b/dolphinscheduler-ui/src/views/thirdparty-api-source/use-table.ts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive, ref } from 'vue' +import { + queryDataSourceListPaging, + deleteDataSource +} from '@/service/modules/data-source' + +export function useTable() { + const data = reactive({ + page: 1, + pageSize: 10, + itemCount: 0, + searchVal: ref(''), + list: [], + loading: false + }) + + const getList = async () => { + if (data.loading) return + data.loading = true + + const listRes = await queryDataSourceListPaging({ + pageNo: data.page, + pageSize: data.pageSize, + searchVal: data.searchVal + }) + data.loading = false + data.list = listRes.totalList + data.itemCount = listRes.total + } + + const updateList = () => { + if (data.list.length === 1 && data.page > 1) { + --data.page + } + getList() + } + + const deleteRecord = async (id: number) => { + const ignored = await deleteDataSource(id) + updateList() + } + + const changePage = (page: number) => { + data.page = page + getList() + } + + const changePageSize = (pageSize: number) => { + data.page = 1 + data.pageSize = pageSize + getList() + } + + return { data, changePage, changePageSize, deleteRecord, updateList } +}