Merge pull request #10 from arenadata/feature/astra-first-release-fix

Fixes crash issue on astra linux (beanutils-core)
arenadata · Feb 13, 2024 · dba4edf · dba4edf
2 parents 3f347e6 + f3710db
commit dba4edf
Show file tree

Hide file tree

Showing 39 changed files with 338 additions and 260 deletions.
diff --git a/bin/common.sh b/bin/common.sh
@@ -38,7 +38,7 @@ if [[ -z "${SMART_CONF_DIR}" ]]; then
 fi
 
 if [[ -z "${SMART_LOG_DIR}" ]]; then
-  export SMART_LOG_DIR="${SMART_HOME}/logs"
+  export SMART_LOG_DIR="/var/log/ssm"
 fi
 export SMART_LOG_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME}
 

diff --git a/conf/hazelcast.xml b/conf/hazelcast.xml
@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config hazelcast-config-3.9.xsd"
-           xmlns="http://www.hazelcast.com/schema/config"
-           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<hazelcast xmlns="http://www.hazelcast.com/schema/config"
+           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+           xsi:schemaLocation="http://www.hazelcast.com/schema/config
+           https://www.hazelcast.com/schema/config/hazelcast-config-4.2.8.xsd">
     <network>
         <join>
             <multicast enabled="false">

diff --git a/conf/log4j.properties b/conf/log4j.properties
@@ -12,7 +12,7 @@
 # log4j configuration used during build and unit tests
 
 rootLogger.level = INFO
-property.filename = ${smart.log.dir}/${smart.log.file}
+property.filename = ${env:smart.log.dir}/${env:smart.log.file}
 appenders = R, console
 
 appender.console.type = Console

diff --git a/pom.xml b/pom.xml
@@ -58,7 +58,7 @@
     <druid.version>1.1.3</druid.version>
     <spring.version>4.3.9.RELEASE</spring.version>
     <antlr4.version>4.6</antlr4.version>
-    <hazelcast.version>3.12.13</hazelcast.version>
+    <hazelcast.version>4.2.8</hazelcast.version>
     <guava.version>15.0</guava.version>
     <gson.version>2.10.1</gson.version>
     <test.build.dir>${project.build.directory}/test-dir</test.build.dir>
@@ -68,6 +68,8 @@
     <postgres.version>42.6.0</postgres.version>
     <mysql.version>5.1.42</mysql.version>
     <sqlite.version>3.34.0</sqlite.version>
+    <commons-configuration.version>2.1.1</commons-configuration.version>
+    <commons-beanutils.version>1.9.4</commons-beanutils.version>
   </properties>
 
   <profiles>

diff --git a/smart-action/pom.xml b/smart-action/pom.xml
@@ -54,7 +54,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-admin/pom.xml b/smart-admin/pom.xml
@@ -49,7 +49,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-agent/pom.xml b/smart-agent/pom.xml
@@ -88,7 +88,7 @@
           <dependency>
             <groupId>com.puppycrawl.tools</groupId>
             <artifactId>checkstyle</artifactId>
-            <version>6.19</version>
+            <version>7.8.2</version>
           </dependency>
         </dependencies>
         <configuration>

diff --git a/smart-client/pom.xml b/smart-client/pom.xml
@@ -53,7 +53,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-client/src/main/java/org/smartdata/client/SmartClient.java b/smart-client/src/main/java/org/smartdata/client/SmartClient.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
-
 import org.smartdata.SmartConstants;
 import org.smartdata.conf.SmartConfKeys;
 import org.smartdata.metrics.FileAccessEvent;

diff --git a/smart-common/pom.xml b/smart-common/pom.xml
@@ -209,7 +209,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-common/src/main/java/org/smartdata/model/rule/RuleExecutorPlugin.java b/smart-common/src/main/java/org/smartdata/model/rule/RuleExecutorPlugin.java
@@ -31,7 +31,7 @@ public interface RuleExecutorPlugin {
    * @param ruleInfo
    * @param tResult
    */
-  void onNewRuleExecutor(final RuleInfo ruleInfo, TranslateResult tResult);
+  void onNewRuleExecutor(RuleInfo ruleInfo, TranslateResult tResult);
 
   /**
    * Called just before rule executor begin to execute rule.
@@ -40,29 +40,29 @@ public interface RuleExecutorPlugin {
    * @param tResult
    * @return continue this execution if true.
    */
-  boolean preExecution(final RuleInfo ruleInfo, TranslateResult tResult);
+  boolean preExecution(RuleInfo ruleInfo, TranslateResult tResult);
 
   /**
    * Called after rule condition checked.
    *
    * @param objects the result of checking rule condition.
    * @return object list that will be used for Cmdlet submission.
    */
-  List<String> preSubmitCmdlet(final RuleInfo ruleInfo, List<String> objects);
+  List<String> preSubmitCmdlet(RuleInfo ruleInfo, List<String> objects);
 
   /**
    * Called right before the CmdletDescriptor been submitted to CmdletManager.
    *
    * @param descriptor
    * @return the descriptor that will be used to submit to CmdletManager
    */
-  CmdletDescriptor preSubmitCmdletDescriptor(final RuleInfo ruleInfo, TranslateResult tResult,
+  CmdletDescriptor preSubmitCmdletDescriptor(RuleInfo ruleInfo, TranslateResult tResult,
       CmdletDescriptor descriptor);
 
   /**
    * Called when an RuleExecutor exits. Called only once per instance.
    *
    * @param ruleInfo
    */
-  void onRuleExecutorExit(final RuleInfo ruleInfo);
+  void onRuleExecutorExit(RuleInfo ruleInfo);
 }
diff --git a/smart-engine/pom.xml b/smart-engine/pom.xml
@@ -280,7 +280,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java
@@ -36,7 +36,6 @@
 import org.smartdata.server.engine.cmdlet.agent.AgentInfo;
 
 import java.io.IOException;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;

diff --git a/smart-engine/src/main/java/org/smartdata/server/cluster/ClusterMembershipListener.java b/smart-engine/src/main/java/org/smartdata/server/cluster/ClusterMembershipListener.java
@@ -17,9 +17,8 @@
  */
 package org.smartdata.server.cluster;
 
-import com.hazelcast.core.MemberAttributeEvent;
-import com.hazelcast.core.MembershipEvent;
-import com.hazelcast.core.MembershipListener;
+import com.hazelcast.cluster.MembershipEvent;
+import com.hazelcast.cluster.MembershipListener;
 import org.smartdata.conf.SmartConf;
 import org.smartdata.conf.SmartConfKeys;
 import org.smartdata.server.utils.HazelcastUtil;
@@ -59,8 +58,4 @@ public void memberRemoved(MembershipEvent membershipEvent) {
       this.daemon.becomeActive();
     }
   }
-
-  @Override
-  public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
-  }
 }
diff --git a/smart-engine/src/main/java/org/smartdata/server/cluster/HazelcastWorker.java b/smart-engine/src/main/java/org/smartdata/server/cluster/HazelcastWorker.java
@@ -15,9 +15,9 @@
 package org.smartdata.server.cluster;
 
 import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.ITopic;
-import com.hazelcast.core.Message;
-import com.hazelcast.core.MessageListener;
+import com.hazelcast.topic.ITopic;
+import com.hazelcast.topic.Message;
+import com.hazelcast.topic.MessageListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.SmartContext;
@@ -58,7 +58,7 @@ public HazelcastWorker(SmartContext smartContext) {
     this.executorService = Executors.newSingleThreadScheduledExecutor();
     this.instance = HazelcastInstanceProvider.getInstance();
     this.statusTopic = instance.getTopic(HazelcastExecutorService.STATUS_TOPIC);
-    String instanceId = instance.getCluster().getLocalMember().getUuid();
+    String instanceId = String.valueOf(instance.getCluster().getLocalMember().getUuid());
     this.masterMessages =
         instance.getTopic(HazelcastExecutorService.WORKER_TOPIC_PREFIX + instanceId);
     this.masterMessages.addMessageListener(new MasterMessageListener());

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletFactory.java
@@ -23,9 +23,6 @@
 import org.smartdata.action.ActionException;
 import org.smartdata.action.ActionRegistry;
 import org.smartdata.action.SmartAction;
-//import org.smartdata.alluxio.AlluxioUtil;
-//import org.smartdata.alluxio.action.AlluxioAction;
-//import alluxio.client.file.FileSystem;
 import org.smartdata.conf.SmartConfKeys;
 import org.smartdata.hdfs.HadoopUtil;
 import org.smartdata.hdfs.action.HdfsAction;

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/HazelcastExecutorService.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/HazelcastExecutorService.java
@@ -17,14 +17,13 @@
  */
 package org.smartdata.server.engine.cmdlet;
 
+import com.hazelcast.cluster.Member;
+import com.hazelcast.cluster.MembershipEvent;
+import com.hazelcast.cluster.MembershipListener;
 import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.ITopic;
-import com.hazelcast.core.Member;
-import com.hazelcast.core.MemberAttributeEvent;
-import com.hazelcast.core.MembershipEvent;
-import com.hazelcast.core.MembershipListener;
-import com.hazelcast.core.Message;
-import com.hazelcast.core.MessageListener;
+import com.hazelcast.topic.ITopic;
+import com.hazelcast.topic.Message;
+import com.hazelcast.topic.MessageListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.model.ExecutorType;
@@ -221,9 +220,6 @@ public void memberRemoved(MembershipEvent membershipEvent) {
       removeMember(member);
     }
 
-    @Override
-    public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
-    }
   }
 
   private class StatusMessageListener implements MessageListener<StatusMessage> {

diff --git a/smart-engine/src/main/java/org/smartdata/server/utils/HazelcastUtil.java b/smart-engine/src/main/java/org/smartdata/server/utils/HazelcastUtil.java
@@ -17,8 +17,8 @@
  */
 package org.smartdata.server.utils;
 
+import com.hazelcast.cluster.Member;
 import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.Member;
 
 import java.util.ArrayList;
 import java.util.List;

diff --git a/smart-hadoop-support/smart-hadoop-client-3.2/pom.xml b/smart-hadoop-support/smart-hadoop-client-3.2/pom.xml
@@ -164,15 +164,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>commons-configuration</groupId>
-            <artifactId>commons-configuration</artifactId>
-            <version>1.6</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-beanutils</groupId>
-                    <artifactId>commons-beanutils</artifactId>
-                </exclusion>
-            </exclusions>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-configuration2</artifactId>
+            <version>${commons-configuration.version}</version>
         </dependency>
     </dependencies>
 

diff --git a/smart-hadoop-support/smart-hadoop-client-3/pom.xml b/smart-hadoop-support/smart-hadoop-client-3/pom.xml
@@ -130,17 +130,6 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>commons-configuration</groupId>
-            <artifactId>commons-configuration</artifactId>
-            <version>1.6</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-beanutils</groupId>
-                    <artifactId>commons-beanutils</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
     </dependencies>
 
     <build>

diff --git a/smart-hadoop-support/smart-hadoop/pom.xml b/smart-hadoop-support/smart-hadoop/pom.xml
@@ -316,15 +316,9 @@
             <version>4.5.5</version>
         </dependency>
         <dependency>
-            <groupId>commons-configuration</groupId>
-            <artifactId>commons-configuration</artifactId>
-            <version>1.6</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-beanutils</groupId>
-                    <artifactId>commons-beanutils</artifactId>
-                </exclusion>
-            </exclusions>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-configuration2</artifactId>
+            <version>${commons-configuration.version}</version>
         </dependency>
         <dependency>
             <groupId>org.smartdata</groupId>

diff --git a/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java b/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java
@@ -182,7 +182,9 @@ private void checkAndCreateIdFiles(URI namenodeURI, Configuration conf) throws I
       throw e;
     } finally {
       try {
-        moverIdOutputStream.close();
+        if (moverIdOutputStream != null) {
+          moverIdOutputStream.close();
+        }
         moverIdOutputStream = null;
       } catch (IOException e) {
         // Ignore this exception
@@ -201,7 +203,9 @@ private void checkAndCreateIdFiles(URI namenodeURI, Configuration conf) throws I
       throw e;
     } finally {
       try {
-        ssmIdOutputStream.close();
+        if (ssmIdOutputStream != null) {
+          ssmIdOutputStream.close();
+        }
         ssmIdOutputStream = null;
       } catch (IOException ie) {
         // Ignore this exception

diff --git a/...t/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java b/...t/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java
@@ -18,9 +18,11 @@
 package org.smartdata.hdfs.metric;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.SubsetConfiguration;
+import org.apache.commons.configuration2.builder.fluent.Configurations;
+import org.apache.commons.configuration2.builder.fluent.Parameters;
+import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.apache.commons.lang.time.FastDateFormat;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -314,20 +316,21 @@ public void close() {
     }
 
     private static SubsetConfiguration loadConfiguration(String prefix, String... fileNames) {
+      Configurations configs = new Configurations();
       for (String fname : fileNames) {
         try {
-          org.apache.commons.configuration.Configuration cf = new PropertiesConfiguration(fname)
-            .interpolatedConfiguration();
+          PropertiesConfiguration cf = configs.properties(fname);
           LOG.info("loaded properties from " + fname);
           return new SubsetConfiguration(cf, prefix, ".");
         } catch (ConfigurationException e) {
-          if (e.getMessage().startsWith("Cannot locate configuration")) {
+          if (e.getMessage().contains("Cannot locate configuration source")) {
             continue;
           }
           throw new RuntimeException(e);
         }
       }
-      return new SubsetConfiguration(new PropertiesConfiguration(), prefix);
+      PropertiesConfiguration emptyConfig = new PropertiesConfiguration();
+      return new SubsetConfiguration(emptyConfig, prefix, ".");
     }
 
     /**

diff --git a/smart-integration/pom.xml b/smart-integration/pom.xml
@@ -254,7 +254,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>

diff --git a/smart-maven-plugins/pom.xml b/smart-maven-plugins/pom.xml
@@ -42,7 +42,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>6.19</version>
+                        <version>7.8.2</version>
                     </dependency>
                 </dependencies>
                 <configuration>