From f44dd85c82cf3ee3daeacbb99114884a07c907bd Mon Sep 17 00:00:00 2001 From: ortis Date: Wed, 4 Jul 2018 20:15:06 +0900 Subject: [PATCH] fix stderr exception on ssh connection fix update stat queue contention change default start param --- README.md | 4 +- pom.xml | 2 +- .../ortis/mochimo/farm_manager/Version.java | 2 +- .../farm_manager/command_lines/Start.java | 7 +- .../mochimo/farm_manager/farm/MiningFarm.java | 31 ++--- .../farm/StatisticsUpdateScheduler.java | 21 ++-- .../farm/StatisticsUpdateTask.java | 71 ++++++++++++ .../farm_manager/farm/StatisticsUpdater.java | 51 ++------- .../mochimo/farm_manager/farm/TaskBoard.java | 106 ++++++++++++++++++ .../farm_manager/farm/miner/SSHConnector.java | 3 +- start.sh | 2 +- 11 files changed, 222 insertions(+), 78 deletions(-) create mode 100644 src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateTask.java create mode 100644 src/main/java/org/ortis/mochimo/farm_manager/farm/TaskBoard.java diff --git a/README.md b/README.md index 69c2117..51189a0 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ Mochimo Farm Manager installation: 6. `stopCommand`: stop command of the miner. If not specified, a `kill` command is send 7. `logCommand`: command to retrieve miner's log. 4. *Higly Recommended*: encrypt your configuration file
`java -jar mochimo-farm-manager-version.jar encrypt plain_text_config.json encrypted_config.json` -5. Check out the running options `java -jar mochimo-farm-manager-version.jar start -h` -6. Start the manager `java -jar mochimo-farm-manager-version.jar start mining_farm_config.json html`. +5. Check out start option `java -jar mochimo-farm-manager-version.jar start -h` +6. Start the manager `java -jar mochimo-farm-manager-version.jar start mining_farm_config.json html` 7. Access the dashboard http://localhost diff --git a/pom.xml b/pom.xml index ec0cda5..f68cd7b 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ farm_manager org.ortis.mochimo mochimo-farm-manager - 0.1 + 0.2 jar diff --git a/src/main/java/org/ortis/mochimo/farm_manager/Version.java b/src/main/java/org/ortis/mochimo/farm_manager/Version.java index fab4012..cc35e7e 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/Version.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/Version.java @@ -23,5 +23,5 @@ public class Version { - public static final String VERSION = "v0.1 beta"; + public static final String VERSION = "v0.2 beta"; } diff --git a/src/main/java/org/ortis/mochimo/farm_manager/command_lines/Start.java b/src/main/java/org/ortis/mochimo/farm_manager/command_lines/Start.java index ea00e43..7e8a04c 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/command_lines/Start.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/command_lines/Start.java @@ -21,6 +21,7 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.LocalDateTime; +import java.util.Locale; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -63,10 +64,10 @@ public class Start implements Callable private String hostBind = "127.0.0.1:80"; @Option(names = { "-sp", "--statistics-parallelism" }, paramLabel = "statistics_parallelism", description = "Number of statistics computing thread") - private int statParallelism = 3; + private int statParallelism = 6; @Option(names = { "-sh", "--statistics-heartbeat" }, paramLabel = "statistics_heartbeat", description = "Delay between statistic computation in seconds") - private int statHeartbeat = 30; + private int statHeartbeat = 60; @Option(names = { "-hp", "--http-parallelism" }, paramLabel = "http_parallelism", description = "Number of http handler thread") private int httpParallelism = 5; @@ -98,7 +99,7 @@ public LocalDateTime get() } }; - switch (this.logLevel) + switch (this.logLevel.toUpperCase(Locale.ENGLISH)) { case "ERROR": LogFactory.setLevel(Level.SEVERE); diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/MiningFarm.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/MiningFarm.java index a3f8157..b780b74 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/farm/MiningFarm.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/MiningFarm.java @@ -1,24 +1,17 @@ /******************************************************************************* * Copyright (C) 2018 Ortis (cao.ortis.org@gmail.com) * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following + * conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + * SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ + package org.ortis.mochimo.farm_manager.farm; import java.io.IOException; @@ -35,10 +28,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -123,15 +114,15 @@ public MiningFarm(final MiningFarmConfig config, final Duration statisticsUpdate this.miners.add(miner); } - final BlockingQueue statisticsUpdateQueue = new LinkedBlockingQueue<>(); + final TaskBoard taskBoard = new TaskBoard(Duration.ofMillis(1000), LogFactory.getLogger("TaskBoard")); this.statisticUpdateSchedulerThread = new Thread( - new StatisticsUpdateScheduler(this, Duration.ofMillis(Math.max(1000, statisticsUpdateHeartbeat.toMillis() / 10)), statisticsUpdateHeartbeat, statisticsUpdateQueue, this.clock, log)); + new StatisticsUpdateScheduler(this, Duration.ofMillis(Math.max(1000, statisticsUpdateHeartbeat.toMillis() / 10)), statisticsUpdateHeartbeat, taskBoard, this.clock, log)); this.statisticUpdateSchedulerThread.setName("StatisticsUpdateScheduler"); this.statisticsPool = Executors.newFixedThreadPool(statisticsParallelism); for (int i = 0; i < statisticsParallelism; i++) { - final StatisticsUpdater su = new StatisticsUpdater(statisticsUpdateQueue, log); + final StatisticsUpdater su = new StatisticsUpdater(taskBoard, log); this.statisticsPool.submit(su); } diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateScheduler.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateScheduler.java index 177dcd5..8201f0c 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateScheduler.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateScheduler.java @@ -16,8 +16,6 @@ import java.time.Duration; import java.time.LocalDateTime; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; import java.util.function.Supplier; import java.util.logging.Logger; @@ -37,7 +35,7 @@ public class StatisticsUpdateScheduler implements Runnable private final MiningFarm farm; private final Duration checkHeartbeat; private final Duration updateHeartbeat; - private final Queue queue; + private final TaskBoard taskBoard; private final Supplier clock; private Logger log; @@ -53,7 +51,7 @@ public class StatisticsUpdateScheduler implements Runnable * pending update queue * @param log */ - public StatisticsUpdateScheduler(final MiningFarm farm, final Duration checkHeartbeat, final Duration updateHeartbeat, final BlockingQueue queue, final Supplier clock, + public StatisticsUpdateScheduler(final MiningFarm farm, final Duration checkHeartbeat, final Duration updateHeartbeat, final TaskBoard taskBoard, final Supplier clock, final Logger log) { this.farm = farm; @@ -65,7 +63,7 @@ public StatisticsUpdateScheduler(final MiningFarm farm, final Duration checkHear if (this.updateHeartbeat.isNegative() || this.updateHeartbeat.isZero()) throw new IllegalArgumentException("Update heartbeat duration must be positive"); - this.queue = queue; + this.taskBoard = taskBoard; this.clock = clock; this.log = log; @@ -84,21 +82,24 @@ public void run() for (final Miner miner : farm.getMiners()) { - if (this.queue.contains(miner)) + if (this.taskBoard.contains(miner)) { - this.log.finest("Statistics update of miner " + miner + " is already pending. Skipping"); + this.log.finest("Statistics update task of miner " + miner + " is already pending. Skipping"); continue; } - if (miner.getStatistics().isDefault() || Duration.between(miner.getStatistics().getTime(), this.clock.get()).compareTo(this.updateHeartbeat) > 0) + final boolean def = miner.getStatistics().isDefault(); + final Duration elpased = Duration.between(miner.getStatistics().getTime(), this.clock.get()); + if (def || elpased.compareTo(this.updateHeartbeat) > 0) { + this.log.fine("Requesting update for miner " + miner); - this.queue.add(miner); + this.taskBoard.add(miner); } } - this.log.fine("Pending statistics update -> " + this.queue.size()); + this.log.fine("Pending statistics update -> " + this.taskBoard.pendingSize()); final long elapsed = System.currentTimeMillis() - start; diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateTask.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateTask.java new file mode 100644 index 0000000..a33cb7e --- /dev/null +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdateTask.java @@ -0,0 +1,71 @@ + +package org.ortis.mochimo.farm_manager.farm; + +import java.util.concurrent.Callable; + +import org.ortis.mochimo.farm_manager.farm.miner.Miner; + +public class StatisticsUpdateTask implements Callable +{ + private final Miner miner; + + private Object owner; + private final Object lock = new Object(); + + public StatisticsUpdateTask(final Miner miner) + { + this.miner = miner; + + } + + public boolean acquire(final Object bidder) + { + synchronized (this.lock) + { + + if (this.owner == null) + { + this.owner = bidder; + return true; + } else + return false; + + } + } + + @Override + public Void call() throws Exception + { + this.miner.updateStatistics(); + + return null; + } + + public Miner getMiner() + { + return miner; + } + + @Override + public int hashCode() + { + return this.miner.hashCode(); + } + + @Override + public boolean equals(final Object o) + { + if (o == this.miner) + return true; + + if (o instanceof StatisticsUpdateTask) + { + final StatisticsUpdateTask task = (StatisticsUpdateTask) o; + + return task.getMiner().equals(this.miner); + } + + return false; + } + +} diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdater.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdater.java index 684a53e..197eb32 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdater.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/StatisticsUpdater.java @@ -1,31 +1,21 @@ /******************************************************************************* * Copyright (C) 2018 Ortis (cao.ortis.org@gmail.com) * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following + * conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + * SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ + package org.ortis.mochimo.farm_manager.farm; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import org.ortis.mochimo.farm_manager.farm.miner.Miner; import org.ortis.mochimo.farm_manager.farm.miner.MinerStatistics; import org.ortis.mochimo.farm_manager.utils.Utils; @@ -38,12 +28,12 @@ public class StatisticsUpdater implements Runnable { - private final BlockingQueue queue; + private final TaskBoard taskBoard; private final Logger log; - public StatisticsUpdater(final BlockingQueue queue, final Logger log) + public StatisticsUpdater(final TaskBoard taskBoard, final Logger log) { - this.queue = queue; + this.taskBoard = taskBoard; this.log = log; } @@ -54,24 +44,7 @@ public void run() try { while (!Thread.interrupted()) - { - - final Miner miner = this.queue.poll(1000, TimeUnit.MILLISECONDS); - - if (miner == null) - continue; - - this.log.fine("Updating statistics for miner" + miner); - - try - { - miner.updateStatistics(); - } catch (final Exception e) - { - this.log.severe("Error while updating miner " + miner + " -> " + Utils.formatException(e)); - } - - } + this.taskBoard.execute(); } catch (final InterruptedException e) { diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/TaskBoard.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/TaskBoard.java new file mode 100644 index 0000000..ca94743 --- /dev/null +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/TaskBoard.java @@ -0,0 +1,106 @@ + +package org.ortis.mochimo.farm_manager.farm; + +import java.time.Duration; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Logger; + +import org.ortis.mochimo.farm_manager.farm.miner.Miner; +import org.ortis.mochimo.farm_manager.utils.Utils; + +/** + * Keep track of pending {@link StatisticsUpdateTask} + * + * @author Ortis
+ * 2018 Jul 04 7:34:27 PM
+ */ +public class TaskBoard +{ + private final Duration sleepTime; + private final Logger log; + + public TaskBoard(final Duration sleepTime, final Logger log) + { + this.sleepTime = sleepTime; + if (this.sleepTime.isNegative() || this.sleepTime.isZero()) + throw new IllegalArgumentException("Sleep time must be greater than 0"); + + this.log = log; + } + + private final List tasks = new LinkedList<>(); + + public boolean contains(final Miner miner) + { + synchronized (this.tasks) + { + for (final StatisticsUpdateTask stu : this.tasks) + if (miner.equals(stu.getMiner())) + return true; + } + return false; + } + + public void add(final Miner miner) + { + synchronized (this.tasks) + { + this.tasks.add(new StatisticsUpdateTask(miner)); + } + + } + + public void execute() throws InterruptedException, Exception + { + StatisticsUpdateTask task = null; + synchronized (this.tasks) + { + + for (final StatisticsUpdateTask sut : this.tasks) + if (sut.acquire(Thread.currentThread())) + { + task = sut; + break; + } + } + + if (task == null) + { + this.log.finer("No task, sleeping " + this.sleepTime); + Thread.sleep(this.sleepTime.toMillis()); + } else + { + this.log.finer("Executing statistics update task " + task); + try + { + task.call(); + } catch (Exception e) + { + this.log.severe("Error while executing statistics update task " + task + " - " + Utils.formatException(e)); + + } + + remove(task); + + } + + } + + private boolean remove(final StatisticsUpdateTask task) + { + synchronized (this.tasks) + { + return this.tasks.remove(task); + } + } + + public int pendingSize() + { + synchronized (this.tasks) + { + return this.tasks.size(); + } + } + +} diff --git a/src/main/java/org/ortis/mochimo/farm_manager/farm/miner/SSHConnector.java b/src/main/java/org/ortis/mochimo/farm_manager/farm/miner/SSHConnector.java index 2180dc4..70eb995 100644 --- a/src/main/java/org/ortis/mochimo/farm_manager/farm/miner/SSHConnector.java +++ b/src/main/java/org/ortis/mochimo/farm_manager/farm/miner/SSHConnector.java @@ -120,7 +120,8 @@ public synchronized List execute(final String command) throws Exception inputReader.close(); if (errBaos.size() > 0) - throw new Exception(new String(errBaos.toByteArray())); + this.log.fine("STDERR -> " + new String(errBaos.toByteArray())); + // throw new Exception(new String(errBaos.toByteArray())); //Some terminal will send harmless error /* if (channel.getExitStatus() != 0) diff --git a/start.sh b/start.sh index 8889a80..c0e091c 100644 --- a/start.sh +++ b/start.sh @@ -5,7 +5,7 @@ MINER_BIN_PATH="/home/user/mochimo/bin" #move to bin folder -cd cd ${MINER_BIN_PATH} +cd ${MINER_BIN_PATH} #clear log file echo '' > miner.log