From 86da5102412c6d069d8bc000497524f25d153c6a Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 15 Aug 2023 17:20:07 +0800 Subject: [PATCH 1/4] fix sql save error when contains many columns --- .../main/java/com/baidu/highflip/core/entity/runtime/Data.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index bc4239b..685618e 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -68,7 +68,7 @@ public class Data { List dimension; @Type(type = "json") - @Column(name = "columns") + @Column(name = "columns", length = 10000) List columns = new LinkedList<>(); @Column(name = "binding_id") From 7a1c1bd288fd776191c26c03d2d4edcd98eb3398 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 29 Aug 2023 15:50:29 +0800 Subject: [PATCH 2/4] fix ClassCastException with different class loader --- highflip-server/pom.xml | 4 ++++ .../server/adaptor/loader/AdaptorLoader.java | 20 +++++++++++++++---- .../src/main/resources/application.properties | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/highflip-server/pom.xml b/highflip-server/pom.xml index 104b332..8d7a243 100644 --- a/highflip-server/pom.xml +++ b/highflip-server/pom.xml @@ -49,6 +49,10 @@ org.springframework.security spring-security-config + + org.springframework.boot + spring-boot-loader + org.postgresql postgresql diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java index edd41e4..96a0ead 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java @@ -5,26 +5,38 @@ import com.baidu.highflip.core.engine.HighFlipAdaptor; import com.baidu.highflip.core.engine.InstanceRegister; import lombok.extern.slf4j.Slf4j; + +import org.springframework.boot.loader.LaunchedURLClassLoader; import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Constructor; +import java.lang.reflect.Method; import java.net.URL; -import java.net.URLClassLoader; import java.util.Properties; - @Slf4j @Component public class AdaptorLoader { - private URLClassLoader loader; + private LaunchedURLClassLoader loader; private Properties props = new Properties(); public void loadJar(URL url) throws IOException { - loader = new URLClassLoader(new URL[]{url}); + loader = (LaunchedURLClassLoader) Thread.currentThread() + .getContextClassLoader(); + try { + Method addURL = + LaunchedURLClassLoader.class.getSuperclass() + .getDeclaredMethod("addURL", + new Class[] {URL.class}); + addURL.setAccessible(true); + addURL.invoke(loader, new Object[] {url}); + } catch (Exception e) { + throw new RuntimeException(e); + } try (InputStream stream = loader .getResourceAsStream(AdaptorPropsList.HIGHFLIP_PROPERTIES_FILE)) { diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index deced10..749e53e 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,5 +1,5 @@ ######################################################### -# highflip adaptor +# highflip adaptor (file:/absolute/path/to/jar) ######################################################### highflip.server.adaptor.path= ######################################################### From b52c435ccd8c6e30f53a44fa646077befeff495d Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 31 Aug 2023 10:22:26 +0800 Subject: [PATCH 3/4] fix NPException when intersect count is 0 --- .../webank/ai/fate/adaptor/DataAdaptor.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java index 45952eb..0db06f4 100644 --- a/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java +++ b/highflip-vendors/highflip-adaptor-fate/src/main/java/com/webank/ai/fate/adaptor/DataAdaptor.java @@ -1,7 +1,5 @@ package com.webank.ai.fate.adaptor; -import static com.webank.ai.fate.common.FateConstants.DATA_ID_SEPARATOR; - import com.baidu.highflip.core.entity.runtime.Data; import com.baidu.highflip.core.entity.runtime.basic.DataCategory; import com.baidu.highflip.core.entity.runtime.basic.KeyPair; @@ -14,14 +12,17 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.util.StringUtils; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.stream.Collectors; @@ -82,9 +83,17 @@ public Iterator> readDataDense(Data data) { try (Response response = getContext().getClient() .downloadComponentResultData(jobId, componentName, role, partyId)) { - String content = DecompressUtils.decompressTarGzToStringMap( - response.body().asInputStream(), - s -> s.contains("csv")).get("data.csv"); + final Map res = + DecompressUtils.decompressTarGzToStringMap( + response.body().asInputStream(), + s -> s.contains("csv")); + if (res == null) { + return new ArrayList>().iterator(); + } + String content = res.get("data.csv"); + if (StringUtils.isEmpty(content)) { + return new ArrayList>().iterator(); + } return Arrays.stream(content.split("\n")) .map(s -> Arrays.stream(s.split(DEFAULT_DELIMITER)) .map(d -> (Object) d) From 486ed4267f4ca27132230494792c8b548f84b448 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 31 Aug 2023 11:47:00 +0800 Subject: [PATCH 4/4] make IDEA class loader compatible with spring boot class loader --- .../server/adaptor/loader/AdaptorLoader.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java index 96a0ead..9404e5a 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/loader/AdaptorLoader.java @@ -14,28 +14,34 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.net.URL; +import java.net.URLClassLoader; import java.util.Properties; @Slf4j @Component public class AdaptorLoader { - private LaunchedURLClassLoader loader; + private URLClassLoader loader; private Properties props = new Properties(); public void loadJar(URL url) throws IOException { - loader = (LaunchedURLClassLoader) Thread.currentThread() - .getContextClassLoader(); - try { - Method addURL = - LaunchedURLClassLoader.class.getSuperclass() - .getDeclaredMethod("addURL", - new Class[] {URL.class}); - addURL.setAccessible(true); - addURL.invoke(loader, new Object[] {url}); - } catch (Exception e) { - throw new RuntimeException(e); + if (Thread.currentThread() + .getContextClassLoader() instanceof LaunchedURLClassLoader) { + loader = (LaunchedURLClassLoader) Thread.currentThread() + .getContextClassLoader(); + try { + Method addURL = + LaunchedURLClassLoader.class.getSuperclass() + .getDeclaredMethod("addURL", + new Class[] {URL.class}); + addURL.setAccessible(true); + addURL.invoke(loader, new Object[] {url}); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + loader = new URLClassLoader(new URL[] {url}); } try (InputStream stream = loader