[ADH-4027] Parametrize hardcoded attributes

arenadata · Jan 17, 2024 · 96a2038 · 96a2038
1 parent edc27e9
commit 96a2038
Show file tree

Hide file tree

Showing 14 changed files with 228 additions and 104 deletions.
diff --git a/conf/smart-default.xml b/conf/smart-default.xml
@@ -375,4 +375,84 @@
       Warning: it requires admin privileges for ssm user to be able to configure the db.
     </description>
   </property>
+
+  <property>
+    <name>smart.access.count.day.tables.num</name>
+    <value>30</value>
+    <description>
+      The max number of access count per day tables in the Metastore.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.access.count.hour.tables.num</name>
+    <value>48</value>
+    <description>
+      The max number of access count per hour tables in the Metastore.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.access.count.minute.tables.num</name>
+    <value>120</value>
+    <description>
+      The max number of access count per minute tables in the Metastore.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.access.count.second.tables.num</name>
+    <value>30</value>
+    <description>
+      The max number of access count per second tables in the Metastore.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.access.event.fetch.interval.ms</name>
+    <value>1000</value>
+    <description>
+      The interval in milliseconds between access event fetches.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.cached.file.fetch.interval.ms</name>
+    <value>5000</value>
+    <description>
+      The interval in milliseconds between cached files fetches from HDFS.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.namespace.fetch.interval.ms</name>
+    <value>1</value>
+    <description>
+      The interval in milliseconds between namespace fetches from HDFS.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.mover.scheduler.storage.report.fetch.interval.ms</name>
+    <value>120000</value>
+    <description>
+      The interval in milliseconds between storage report fetches from HDFS DataNode in mover scheduler.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.metastore.small-file.insert.batch.size</name>
+    <value>200</value>
+    <description>
+      The max size of small file insert batch to the Metastore.
+    </description>
+  </property>
+
+  <property>
+    <name>smart.agent.master.ask.timeout.ms</name>
+    <value>5000</value>
+    <description>
+      The max time in milliseconds to wait an answer from the SmartAgent master actor during action submission.
+    </description>
+  </property>
 </configuration>
diff --git a/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java b/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java
@@ -90,6 +90,34 @@ public class SmartConfKeys {
       "smart.metastore.mysql.legacy.enabled";
   public static final boolean SMART_METASTORE_LEGACY_MYSQL_SUPPORT_DEFAULT = false;
 
+  public static final String SMART_NUM_DAY_TABLES_TO_KEEP_KEY =
+      "smart.access.count.day.tables.num";
+  public static final int SMART_NUM_DAY_TABLES_TO_KEEP_DEFAULT = 30;
+
+  public static final String SMART_NUM_HOUR_TABLES_TO_KEEP_KEY =
+      "smart.access.count.hour.tables.num";
+  public static final int SMART_NUM_HOUR_TABLES_TO_KEEP_DEFAULT = 48;
+
+  public static final String SMART_NUM_MINUTE_TABLES_TO_KEEP_KEY =
+      "smart.access.count.minute.tables.num";
+  public static final int SMART_NUM_MINUTE_TABLES_TO_KEEP_DEFAULT = 120;
+
+  public static final String SMART_NUM_SECOND_TABLES_TO_KEEP_KEY =
+      "smart.access.count.second.tables.num";
+  public static final int SMART_NUM_SECOND_TABLES_TO_KEEP_DEFAULT = 30;
+
+  public static final String SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_KEY =
+      "smart.access.event.fetch.interval.ms";
+  public static final long SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_DEFAULT = 1000L;
+
+  public static final String SMART_CACHED_FILE_FETCH_INTERVAL_MS_KEY =
+      "smart.cached.file.fetch.interval.ms";
+  public static final long SMART_CACHED_FILE_FETCH_INTERVAL_MS_DEFAULT = 5 * 1000L;
+
+  public static final String SMART_NAMESPACE_FETCH_INTERVAL_MS_KEY =
+      "smart.namespace.fetch.interval.ms";
+  public static final long SMART_NAMESPACE_FETCH_INTERVAL_MS_DEFAULT = 1L;
+
   // StatesManager
 
   // RuleManager
@@ -140,6 +168,14 @@ public class SmartConfKeys {
   public static final int SMART_FILE_DIFF_MAX_NUM_RECORDS_DEFAULT =
       10000;
 
+  public static final String SMART_MOVER_SCHEDULER_REPORT_FETCH_INTERVAL_MS_KEY =
+      "smart.mover.scheduler.storage.report.fetch.interval.ms";
+  public static final long SMART_MOVER_SCHEDULER_REPORT_FETCH_INTERVAL_MS_DEFAULT = 2 * 60 * 1000;
+
+  public static final String SMART_SMALL_FILE_METASTORE_INSERT_BATCH_SIZE_KEY =
+      "smart.metastore.small-file.insert.batch.size";
+  public static final int SMART_SMALL_FILE_METASTORE_INSERT_BATCH_SIZE_DEFAULT = 200;
+
   // Dispatcher
   public static final String SMART_CMDLET_DISPATCHER_LOG_DISP_RESULT_KEY =
       "smart.cmdlet.dispatcher.log.disp.result";
@@ -167,6 +203,10 @@ public class SmartConfKeys {
   public static final String SMART_AGENT_PORT_KEY = "smart.agent.port";
   public static final int SMART_AGENT_PORT_DEFAULT = 7048;
 
+  public static final String SMART_AGENT_MASTER_ASK_TIMEOUT_MS_KEY =
+      "smart.agent.master.ask.timeout.ms";
+  public static final long SMART_AGENT_MASTER_ASK_TIMEOUT_MS_DEFAULT = 5000L;
+
   /** Do NOT configure the following two options manually. They are set by the boot scripts. **/
   public static final String SMART_AGENT_MASTER_ADDRESS_KEY = "smart.agent.master.address";
   public static final String SMART_AGENT_ADDRESS_KEY = "smart.agent.address";

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java
@@ -76,7 +76,7 @@ public void init() throws IOException {
     LOG.info("Initializing ...");
     this.executorService = Executors.newScheduledThreadPool(4);
     this.accessCountTableManager = new AccessCountTableManager(
-        serverContext.getMetaStore(), executorService);
+        serverContext.getMetaStore(), executorService, serverContext.getConf());
     this.fileAccessEventSource = MetricsFactory.createAccessEventSource(serverContext.getConf());
     this.accessEventFetcher =
         new AccessEventFetcher(

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/agent/AgentMaster.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/agent/AgentMaster.java
@@ -53,11 +53,13 @@
 import scala.concurrent.Future;
 import scala.concurrent.duration.Duration;
 
+import static org.smartdata.conf.SmartConfKeys.SMART_AGENT_MASTER_ASK_TIMEOUT_MS_DEFAULT;
+import static org.smartdata.conf.SmartConfKeys.SMART_AGENT_MASTER_ASK_TIMEOUT_MS_KEY;
+
 public class AgentMaster {
 
   private static final Logger LOG = LoggerFactory.getLogger(AgentMaster.class);
-  public static final Timeout TIMEOUT =
-      new Timeout(Duration.create(5, TimeUnit.SECONDS));
+  public Timeout masterAskTimeout;
 
   private ActorSystem system;
   private ActorRef master;
@@ -71,6 +73,13 @@ private AgentMaster(SmartConf conf) throws IOException {
     if (addresses == null) {
       throw new IOException("AgentMaster address not configured!");
     }
+
+    long masterAskTimeoutMs = conf.getLong(
+        SMART_AGENT_MASTER_ASK_TIMEOUT_MS_KEY,
+        SMART_AGENT_MASTER_ASK_TIMEOUT_MS_DEFAULT
+    );
+    masterAskTimeout = new Timeout(Duration.create(masterAskTimeoutMs, TimeUnit.MILLISECONDS));
+
     String address = addresses[0];
     LOG.info("Agent master: " + address);
     Config config = AgentUtils.overrideRemoteAddress(
@@ -159,8 +168,8 @@ ActorRef getMasterActor() {
   }
 
   Object askMaster(Object message) throws Exception {
-    Future<Object> answer = Patterns.ask(master, message, TIMEOUT);
-    return Await.result(answer, TIMEOUT.duration());
+    Future<Object> answer = Patterns.ask(master, message, masterAskTimeout);
+    return Await.result(answer, masterAskTimeout.duration());
   }
 
   class ActorSystemLauncher extends Thread {

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java b/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java
@@ -26,15 +26,16 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
+import static org.smartdata.conf.SmartConfKeys.SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_DEFAULT;
+import static org.smartdata.conf.SmartConfKeys.SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_KEY;
+
 public class AccessEventFetcher {
   static final Logger LOG = LoggerFactory.getLogger(AccessEventFetcher.class);
 
-  private static final Long DEFAULT_INTERVAL = 1 * 1000L;
   private final ScheduledExecutorService scheduledExecutorService;
   private final Long fetchInterval;
   private ScheduledFuture scheduledFuture;
@@ -45,24 +46,10 @@ public AccessEventFetcher(
       AccessCountTableManager manager,
       ScheduledExecutorService service,
       FileAccessEventCollector collector) {
-    this(DEFAULT_INTERVAL, conf, manager, service, collector);
-  }
-
-  public AccessEventFetcher(
-      Long fetchInterval,
-      Configuration conf,
-      AccessCountTableManager manager,
-      FileAccessEventCollector collector) {
-    this(fetchInterval, conf, manager, Executors.newSingleThreadScheduledExecutor(), collector);
-  }
-
-  public AccessEventFetcher(
-      Long fetchInterval,
-      Configuration conf,
-      AccessCountTableManager manager,
-      ScheduledExecutorService service,
-      FileAccessEventCollector collector) {
-    this.fetchInterval = fetchInterval;
+    this.fetchInterval = conf.getLong(
+        SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_KEY,
+        SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_DEFAULT
+    );
     this.fetchTask = new FetchTask(conf, manager, collector);
     this.scheduledExecutorService = service;
   }

diff --git a/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java b/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HdfsStatesUpdateService.java
@@ -91,7 +91,7 @@ public void init() throws IOException {
     client = HadoopUtil.getDFSClient(nnUri, conf);
     checkAndCreateIdFiles(nnUri, context.getConf());
     this.executorService = Executors.newScheduledThreadPool(4);
-    this.cachedListFetcher = new CachedListFetcher(client, metaStore);
+    this.cachedListFetcher = new CachedListFetcher(conf, client, metaStore);
     this.inotifyEventFetcher = new InotifyEventFetcher(client,
         metaStore, executorService, new FetchFinishedCallBack(), context.getConf());
     this.dataNodeInfoFetcher = new DataNodeInfoFetcher(

diff --git a/...pport/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/CachedListFetcher.java b/...pport/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/CachedListFetcher.java
@@ -17,6 +17,7 @@
  */
 package org.smartdata.hdfs.metric.fetcher;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -42,9 +43,11 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
+import static org.smartdata.conf.SmartConfKeys.SMART_CACHED_FILE_FETCH_INTERVAL_MS_DEFAULT;
+import static org.smartdata.conf.SmartConfKeys.SMART_CACHED_FILE_FETCH_INTERVAL_MS_KEY;
+
 public class CachedListFetcher {
 
-  private static final Long DEFAULT_INTERVAL = 5 * 1000L;
   private final ScheduledExecutorService scheduledExecutorService;
   private final Long fetchInterval;
   private FetchTask fetchTask;
@@ -55,32 +58,22 @@ public class CachedListFetcher {
       LoggerFactory.getLogger(CachedListFetcher.class);
 
   public CachedListFetcher(
-      Long fetchInterval,
+      Configuration configuration,
       DFSClient dfsClient, MetaStore metaStore,
       ScheduledExecutorService service) {
-    this.fetchInterval = fetchInterval;
+    this.fetchInterval = configuration.getLong(
+        SMART_CACHED_FILE_FETCH_INTERVAL_MS_KEY,
+        SMART_CACHED_FILE_FETCH_INTERVAL_MS_DEFAULT
+    );
     this.metaStore = metaStore;
     this.fetchTask = new FetchTask(dfsClient, metaStore);
     this.scheduledExecutorService = service;
   }
 
   public CachedListFetcher(
-      Long fetchInterval,
-      DFSClient dfsClient, MetaStore metaStore) {
-    this(fetchInterval, dfsClient, metaStore,
-        Executors.newSingleThreadScheduledExecutor());
-  }
-
-  public CachedListFetcher(
+      Configuration configuration,
       DFSClient dfsClient, MetaStore metaStore) {
-    this(DEFAULT_INTERVAL, dfsClient, metaStore,
-        Executors.newSingleThreadScheduledExecutor());
-  }
-
-  public CachedListFetcher(
-      DFSClient dfsClient, MetaStore metaStore,
-      ScheduledExecutorService service) {
-    this(DEFAULT_INTERVAL, dfsClient, metaStore, service);
+    this(configuration, dfsClient, metaStore, Executors.newSingleThreadScheduledExecutor());
   }
 
   public void start() {

diff --git a/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventFetcher.java b/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventFetcher.java
@@ -87,9 +87,9 @@ public InotifyEventFetcher(DFSClient client, MetaStore metaStore,
     this.metaStore = metaStore;
     this.scheduledExecutorService = service;
     this.finishedCallback = callBack;
-    // use independent thread pool
-    this.nameSpaceFetcher = new NamespaceFetcher(client, metaStore, null);
     this.conf = new SmartConf();
+    // use independent thread pool
+    this.nameSpaceFetcher = new NamespaceFetcher(client, metaStore, null, conf);
   }
 
   public InotifyEventFetcher(DFSClient client, MetaStore metaStore,

diff --git a/...upport/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java b/...upport/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java
@@ -45,8 +45,6 @@
 import static org.smartdata.hdfs.CompatibilityHelperLoader.getHelper;
 
 public class NamespaceFetcher {
-  private static final Long DEFAULT_INTERVAL = 1L;
-
   private final ScheduledExecutorService scheduledExecutorService;
   private final long fetchInterval;
   private ScheduledFuture[] fetchTaskFutures;
@@ -57,28 +55,13 @@ public class NamespaceFetcher {
   private MetaStore metaStore;
   private SmartConf conf;
 
-  public static final Logger LOG =
-      LoggerFactory.getLogger(NamespaceFetcher.class);
-
-  public NamespaceFetcher(DFSClient client, MetaStore metaStore, ScheduledExecutorService service) {
-    this(client, metaStore, DEFAULT_INTERVAL, service, new SmartConf());
-  }
-
-  public NamespaceFetcher(DFSClient client, MetaStore metaStore, ScheduledExecutorService service,
-                          SmartConf conf) {
-    this(client, metaStore, DEFAULT_INTERVAL, service, conf);
-  }
-
-  public NamespaceFetcher(DFSClient client, MetaStore metaStore, long fetchInterval) {
-    this(client, metaStore, fetchInterval, null, new SmartConf());
-  }
+  public static final Logger LOG = LoggerFactory.getLogger(NamespaceFetcher.class);
 
-  public NamespaceFetcher(DFSClient client, MetaStore metaStore, long fetchInterval,
-                          SmartConf conf) {
-    this(client, metaStore, fetchInterval, null, conf);
+  public NamespaceFetcher(DFSClient client, MetaStore metaStore, SmartConf conf) {
+    this(client, metaStore, null, conf);
   }
 
-  public NamespaceFetcher(DFSClient client, MetaStore metaStore, long fetchInterval,
+  public NamespaceFetcher(DFSClient client, MetaStore metaStore,
                           ScheduledExecutorService service, SmartConf conf) {
     int numProducers = conf.getInt(SmartConfKeys.SMART_NAMESPACE_FETCHER_PRODUCERS_NUM_KEY,
         SmartConfKeys.SMART_NAMESPACE_FETCHER_PRODUCERS_NUM_DEFAULT);
@@ -96,7 +79,10 @@ public NamespaceFetcher(DFSClient client, MetaStore metaStore, long fetchInterva
     for (int i = 0; i < numConsumers; i++) {
       consumers[i] = new FileStatusIngester(metaStore);
     }
-    this.fetchInterval = fetchInterval;
+    this.fetchInterval = conf.getLong(
+        SmartConfKeys.SMART_NAMESPACE_FETCH_INTERVAL_MS_KEY,
+        SmartConfKeys.SMART_NAMESPACE_FETCH_INTERVAL_MS_DEFAULT
+    );
     if (service != null) {
       this.scheduledExecutorService = service;
     } else {