diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b0191dc..4f41f9b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,7 @@ name: test on: + workflow_dispatch: push: branches: [ master ] tags: [ 'v*.*' ] diff --git a/NEWS.md b/NEWS.md index 9beb8d2..7774fdb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,12 @@ # v1.16 - ??/??/2023 -## Fixes +## Improvements +* add a field to define output numeric format to display in IOPlugins +* add a field to define default columns to display in a table in IOPlugins +## Fixes +* fix old results that cannot be reloaded randomly +* fix creation of concurent RMahtExpression when there are multi thread on the same Plugin (bug with Rserve session never killed) * support missing output/input in Python wrapper * update Rsession to 3.1.8 to support R>=4.3 diff --git a/src/main/java/org/funz/Project.java b/src/main/java/org/funz/Project.java index f7381ed..9bdb450 100644 --- a/src/main/java/org/funz/Project.java +++ b/src/main/java/org/funz/Project.java @@ -42,6 +42,7 @@ */ public class Project { + @Override public String toString() { return getName(); @@ -72,6 +73,16 @@ public String toString() { public boolean deletePreviousArchive = DEFAULT_deletePreviousArchive; private AbstractShell shell; + /** + * Name of the file we wan't to check to get cache results. + * Keep it "null" or empty if we wan't to check all inputDir files m5. + */ + private String cacheFileNameToCheck = null; + + public void setCacheFileNameToCheck(String cacheFileNameToCheck) { + this.cacheFileNameToCheck = cacheFileNameToCheck; + } + /** * @return the shell */ @@ -136,6 +147,10 @@ public long getBlacklistTimeout() { return blacklistTimeout; } + public String getCacheFileNameToCheck() { + return this.cacheFileNameToCheck; + } + /* too much dangerous : InputFile are not updated nor checked, so when calling Project.cleanParameters, var will disappear... public void replaceVariable(Variable x) { Variable x_old = getVariableByName(x.getName()); diff --git a/src/main/java/org/funz/api/BatchRun_v1.java b/src/main/java/org/funz/api/BatchRun_v1.java index 75bcb5b..4faf78a 100644 --- a/src/main/java/org/funz/api/BatchRun_v1.java +++ b/src/main/java/org/funz/api/BatchRun_v1.java @@ -16,6 +16,7 @@ import org.funz.parameter.OutputFunctionExpression; import org.funz.run.Client; import org.funz.run.Computer; +import org.funz.util.Digest; import org.funz.util.Disk; import org.funz.util.Format; import org.funz.util.ZipTool; @@ -26,6 +27,8 @@ import java.io.IOException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import static java.lang.Thread.sleep; import static org.funz.Protocol.ARCHIVE_FILTER; @@ -540,7 +543,18 @@ boolean runCase(final Case c) { for (Cache _c : getCache()) { addInfoHistory(c, "Searching in cache: " + getCache()); File dir = files[0].getParentFile().getParentFile(); - File matching_output = _c.getMatchingOutputDirinCache(new File(dir + File.separator + Constants.INPUT_DIR), prj.getCode()); + File inputDir = new File(dir + File.separator + Constants.INPUT_DIR); + String cacheFileNameToCheck = prj.getCacheFileNameToCheck(); + byte[] checkSum = null; + if(cacheFileNameToCheck!=null && !cacheFileNameToCheck.isEmpty()) { + File modelVars = new File(inputDir, cacheFileNameToCheck); + if(modelVars!=null && modelVars.exists()) { + checkSum = Digest.getSum(modelVars); + } + } + + + File matching_output = _c.getMatchingOutputDirinCache(inputDir, prj.getCode(), checkSum); if (matching_output != null) { addInfoHistory(c, "Found in " + matching_output.getAbsolutePath()); @@ -1236,6 +1250,14 @@ public boolean test(Case t) { return theseCases; } + private static boolean hasDuplicate(List caseList) { + Set set = new HashSet<>(); + // Set#add returns false if the set does not change, which + // indicates that a duplicate element has been added. + for (Case aCase: caseList) if (!set.add(aCase.getName())) return true; + return false; + } + public List getFinishedCases() { List theseCases = new LinkedList<>(); for (Case t : getSelectedCases()) { @@ -1262,6 +1284,7 @@ private boolean hasPendingCases() { public List torun; + public boolean runBatch() throws Exception { //LogUtils.tic("runBatch"); @@ -1334,14 +1357,18 @@ public boolean runBatch() throws Exception { List cloneCases = new ArrayList(); //LogUtils.toc("getPendingCases"); + List finishedCaseList = getFinishedCases(); + List pendingCaseList = getPendingCases(); + boolean hasDuplicatedCases = hasDuplicate(getPendingCases()); int numToRun = torun.size(); for (int i = 0; i < numToRun; i++) { final Case c = torun.get(i); c.setObserver(observer); boolean already_launched = false; boolean cloned = false; - for (int j = 0; j < getFinishedCases().size(); j++) { //for old cases - final Case cprev = getFinishedCases().get(j); + + for (int j = 0; j < finishedCaseList.size(); j++) { //for old cases + final Case cprev = finishedCaseList.get(j); //LogUtils.tic("synchronized (cprev) "); synchronized (cprev) { if (c.getName().equals(cprev.getName())) { @@ -1354,22 +1381,27 @@ public boolean runBatch() throws Exception { } //LogUtils.toc("synchronized (cprev) "); } - if (!already_launched && !cloned) { - for (int j = 0; j < i; j++) { // when same case is just asked before. - final Case cprev = getPendingCases().get(j); - //LogUtils.tic("synchronized (cprev) "); - synchronized (cprev) { - if (c.getName().equals(cprev.getName())) { - already_launched = true; - out("Case " + c.getName() + " already launched...", 1); - cloneCases.add(new CloneCase(c, cprev)); - cloned = true; - break; + + if(hasDuplicatedCases) { + // If there is not duplicated cases, ignore the block bellow which is very inefficient + if (!already_launched && !cloned) { + for (int j = 0; j < i; j++) { // when same case is just asked before. + final Case cprev = pendingCaseList.get(j); + //LogUtils.tic("synchronized (cprev) "); + synchronized (cprev) { + if (c.getName().equals(cprev.getName())) { + already_launched = true; + out("Case " + c.getName() + " already launched...", 1); + cloneCases.add(new CloneCase(c, cprev)); + cloned = true; + break; + } } } - //LogUtils.toc("synchronized (cprev) "); } } + + if (!already_launched && !cloned) { //LogUtils.tic("new RunCase"); RunCase rc = new RunCase(c); @@ -1404,15 +1436,19 @@ public void run() { //LogUtils.toc("beforeRunCases"); setState(BATCH_RUNNING); + // TODO NC: configure nb threads + //ExecutorService executorService = Executors.newFixedThreadPool(8); // let's start only some cases (to limit concurrent RunCase threads) for (int i = 0; i < prj.getMaxCalcs(); i++) { if (i < runCases.size()) { out("Will start case " + runCases.get(i).c.getName(), 3); //LogUtils.tic("runCases.get(i).start()"); + //executorService.submit(runCases.get(i)); runCases.get(i).start(); //LogUtils.toc("runCases.get(i).start()"); } } + //executorService.shutdown(); int f = 0; String state_value = ""; @@ -1428,29 +1464,35 @@ public void run() { int[][] states = new int[getSelectedCases().size()][Case.STATE_STRINGS.length]; while (f < numToRun && !askToStop) { //out_noln(" ? ", 5); - int f_old = f; + //LogTicToc.tic("filled"); f = filled(torun); //LogTicToc.toc("filled"); // let's start only some cases (to limit concurrent RunCase threads) - for (int i = 0; i < /*f - f_old*/ Math.min(prj.getMaxCalcs(), numToRun - f); i++) { - for (int j = 0; j < runCases.size(); j++) { - //LogTicToc.tic("runCases.get(j)"); - RunCase rc = runCases.get(j); - //LogTicToc.toc("runCases.get(j)"); - if (!rc.isAlive()) { - if (rc.c != null && rc.c.getState() == Case.STATE_INTACT) { - out("Starting case " + rc.c.getName(), 3); - //LogTicToc.tic("rc.start()"); - rc.start(); - //LogTicToc.toc("rc.start()"); - //System.err.println("+"); - break; - }//else System.err.println(rc.c.getStatusInformation()); - } + int nbCaseAlive = 0; + for (int j = 0; j < runCases.size(); j++) { + //LogTicToc.tic("runCases.get(j)"); + RunCase rc = runCases.get(j); + //LogTicToc.toc("runCases.get(j)"); + if(rc.isAlive()) { + nbCaseAlive++; + } else { + if (rc.c != null && rc.c.getState() == Case.STATE_INTACT) { + out("Starting case " + rc.c.getName(), 3); + //LogTicToc.tic("rc.start()"); + rc.start(); + //LogTicToc.toc("rc.start()"); + //System.err.println("+"); + break; + }//else System.err.println(rc.c.getStatusInformation()); + } + // Only run "prj.getMaxCalcs()" number of RunCases in parallel + if(nbCaseAlive >= prj.getMaxCalcs()) { + break; } } + //out("Finished " + f + "/" + getCases().size() + " cases.", 2); if (Funz.getVerbosity() > 3) { //out("Cases status:", 3); @@ -1524,7 +1566,15 @@ public void run() { setState(BATCH_EXCEPTION); return false; } finally { + // Don't finalize formula interpreter here because Shell_v1 needs it +// try { +// // Finalize RServe or other formula interpreter +// //prj.getPlugin().getFormulaInterpreter().finalize(); +// } catch (Throwable throwable) { +// throw new Exception("Failed to close formula interpreter \n" + throwable.getMessage()); +// } try { + setState(MERGING_RESULTS); merged_results.putAll(merge(getSelectedCases()));//torun)); } catch (Exception e) { err("Failed to merge results: " + torun, 0); @@ -1561,6 +1611,7 @@ public File getArchiveDirectory() { public static final String BATCH_STARTING = "Starting..."; public static final String BATCH_ERROR = "Batch failed"; public static final String BATCH_EXCEPTION = "Batch exception"; + private static final String MERGING_RESULTS = "Merging results"; public String getState() { return state; diff --git a/src/main/java/org/funz/ioplugin/BasicIOPlugin.java b/src/main/java/org/funz/ioplugin/BasicIOPlugin.java index 8fd8223..7dbb993 100644 --- a/src/main/java/org/funz/ioplugin/BasicIOPlugin.java +++ b/src/main/java/org/funz/ioplugin/BasicIOPlugin.java @@ -1,18 +1,5 @@ package org.funz.ioplugin; -import java.io.File; -import java.io.FileFilter; -import java.io.FilenameFilter; -import java.io.IOException; -import java.net.URL; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.funz.Constants; import org.funz.log.Log; import org.funz.log.LogCollector.SeverityLevel; @@ -20,9 +7,16 @@ import org.funz.parameter.SyntaxRules; import org.funz.script.MathExpression; import org.funz.script.ParseExpression; -import org.funz.script.RMathExpression; import org.funz.util.Data; -import org.math.io.parser.ArrayString; + +import java.io.File; +import java.io.FileFilter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.URL; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Simplified IOPlugin implementation based on properties file. Following @@ -195,6 +189,9 @@ public void setInputFiles(File[] inputfiles) { } } + initializeDefaultDisplayedOutput(); + initializeOutputFormat(); + //_output.put("output_lines", new String[0]); //_output.put("out", ""); //_output.put("err", ""); @@ -398,6 +395,43 @@ public boolean accept(File f) { } // ant compile dist; cd dist; zip -r ../../plugin-R/funz-client.zip *; cd .. + + @Override + public void initializeDefaultDisplayedOutput() { + if (_properties.containsKey("defaultdisplayedoutputlist") && _properties.getProperty("defaultdisplayedoutputlist").length() > 0) { + Log.logMessage("BasicIOPlugin " + getID(), SeverityLevel.INFO, false, " defaultdisplayedoutputlist=" + _properties.getProperty("defaultdisplayedoutputlist")); + + List outputlist = new LinkedList(); + Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(_properties.getProperty("defaultdisplayedoutputlist")); + while (m.find()) outputlist.add(m.group(1)); + this._defaultDisplayedOutput=outputlist; + } + } + + @Override + public void initializeOutputFormat() { + if (_properties.containsKey("outputformat") && _properties.getProperty("outputformat").length() > 0) { + Log.logMessage("BasicIOPlugin " + getID(), SeverityLevel.INFO, false, " outputformat=" + _properties.getProperty("outputformat")); + + List outputFormatList = new LinkedList(); + Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(_properties.getProperty("outputformat")); + while (m.find()) outputFormatList.add(m.group(1)); + for(String outputFormat : outputFormatList) { + boolean readSucess = false; + if(outputFormat.contains(":")) { + String[] outputAndFormat = outputFormat.split(":"); + if(outputAndFormat.length==2) { + this._outputFormat.put(outputAndFormat[0], outputAndFormat[1]); + readSucess = true; + } + } + if(!readSucess) { + System.err.println("The following outputFormat is unreadable (please respect syntax 'output:format'): " + outputFormat); + } + } + } + } + @Override public void setCommentLine(String c) { commentLine = c; diff --git a/src/main/java/org/funz/ioplugin/ExtendedIOPlugin.java b/src/main/java/org/funz/ioplugin/ExtendedIOPlugin.java index 995ddd3..210eb76 100644 --- a/src/main/java/org/funz/ioplugin/ExtendedIOPlugin.java +++ b/src/main/java/org/funz/ioplugin/ExtendedIOPlugin.java @@ -3,19 +3,6 @@ */ package org.funz.ioplugin; -import java.io.File; -import java.io.FileFilter; -import java.io.FileReader; -import java.io.FilenameFilter; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; import org.funz.Project; import org.funz.conf.Configuration; import org.funz.log.Log; @@ -29,9 +16,11 @@ import org.funz.script.RMathExpression; import org.funz.util.ASCII; import org.funz.util.Data; -import static org.funz.util.Data.asString; import org.funz.util.Disk; -import org.math.io.parser.ArrayString; + +import java.io.*; +import java.net.URL; +import java.util.*; public class ExtendedIOPlugin implements IOPluginInterface { @@ -39,6 +28,8 @@ public class ExtendedIOPlugin implements IOPluginInterface { protected String source = null; protected File[] _inputfiles; protected HashMap _output = new HashMap(); + protected List _defaultDisplayedOutput = new ArrayList<>(); + protected Map _outputFormat = new HashMap<>(); protected String commentLine; public String[] doc_links = {}; protected int formulaLimit = SyntaxRules.LIMIT_SYMBOL_BRACKETS; @@ -46,6 +37,7 @@ public class ExtendedIOPlugin implements IOPluginInterface { public String information = "Generic default plugin"; protected int variableLimit = SyntaxRules.LIMIT_SYMBOL_SQ_BRACKETS; protected int variableStartSymbol = SyntaxRules.START_SYMBOL_DOLLAR; + private Boolean mathEngineLocker = true; MathExpression mathengine; @Override @@ -68,15 +60,20 @@ public void setFormulaInterpreter(MathExpression e) { @Override public MathExpression getFormulaInterpreter() { if (mathengine == null) { - String name = "NullProject"; - if (getProject() != null) { - name = getProject().getName(); - } - File logdir = new File(System.getProperty("java.io.tmpdir")); - if (getProject() != null) { - logdir = getProject().getLogDir(); + synchronized (mathEngineLocker) { + if(mathengine == null) { + String name = "NullProject"; + if (getProject() != null) { + name = getProject().getName(); + } + File logdir = new File(System.getProperty("java.io.tmpdir")); + if (getProject() != null) { + logdir = getProject().getLogDir(); + } + mathengine = new RMathExpression(name, Configuration.isLog("R") ? new File(logdir, name + ".Rlog") : null); + } } - mathengine = new RMathExpression(name, Configuration.isLog("R") ? new File(logdir, name + ".Rlog") : null); + } return mathengine; } @@ -184,6 +181,24 @@ public LinkedList suggestOutputFunctions() { return ofl; } + @Override + public void initializeDefaultDisplayedOutput() { + } + + @Override + public List getDefaultDisplayedOutput() { + return this._defaultDisplayedOutput; + } + + @Override + public void initializeOutputFormat() { + } + + @Override + public Map getOutputFormat() { + return this._outputFormat; + } + @Override public void setID(String fn) { _id = fn; diff --git a/src/main/java/org/funz/ioplugin/IOPluginInterface.java b/src/main/java/org/funz/ioplugin/IOPluginInterface.java index d6aa07e..04c1f2e 100644 --- a/src/main/java/org/funz/ioplugin/IOPluginInterface.java +++ b/src/main/java/org/funz/ioplugin/IOPluginInterface.java @@ -3,15 +3,16 @@ */ package org.funz.ioplugin; -import java.io.File; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import org.funz.Project; import org.funz.parameter.InputFile; import org.funz.parameter.OutputFunctionExpression; import org.funz.script.MathExpression; +import java.io.File; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + /** * Interace for code specific information and behaviour: documentation, links, * syntax, ... @@ -132,6 +133,14 @@ public interface IOPluginInterface { */ public LinkedList suggestOutputFunctions(); + public void initializeDefaultDisplayedOutput(); + + public List getDefaultDisplayedOutput(); + + public void initializeOutputFormat(); + + public Map getOutputFormat(); + /** * Get numerical rounding for this code * diff --git a/src/main/java/org/funz/parameter/Cache.java b/src/main/java/org/funz/parameter/Cache.java index 4491af0..3078a5c 100644 --- a/src/main/java/org/funz/parameter/Cache.java +++ b/src/main/java/org/funz/parameter/Cache.java @@ -24,7 +24,14 @@ static File OutputDir(File in) { } static File InfoFile(File in) { - return new File(in.getParentFile(), Case.FILE_INFO); + File infoFile = new File(in.getParentFile(), Case.FILE_INFO); + if(!infoFile.exists()) { + File oldInfoFile = new File(in.getParentFile(), Case.OLD_FILE_INFO); + if(oldInfoFile.exists()) { + infoFile = oldInfoFile; + } + } + return infoFile; } static boolean IsOutputDir(File odir) { @@ -95,6 +102,15 @@ boolean contentIncludes(InputDir idir) { } return true; } + + public boolean hasCheckSum(byte[] wantedCheckSum) { + for (int i = 0; i < this.md5sums.length; i++) { + if (Digest.matches(md5sums[i], wantedCheckSum)) { + return true; + } + } + return false; + } } class NewInputDir extends InputDir { @@ -266,26 +282,21 @@ static LinkedList findInputDirs(File root, boolean cleanup) { return inputDirs; } -// pour trouver les fichiers out potentiels: ceux dont le nom contient le nom de fichiers ayant le mm md5 que les fichiers input... - /*public File[] searchFilesinPool(File[] inputFiles, boolean withSameName) { - - InputFiles ifiles = new InputFiles(inputFiles); - File[] output_files = null; - for (int i = 0; i < resultsPool.size(); i++) { - LinkedList match; - if (withSameName) - match = ifiles.contentsIn_SameName(resultsPool.get(i)); - else - match = ifiles.contentsIn(resultsPool.get(i)); - - if (match != null) { - output_files = ifiles.getOutputFiles(resultsPool.get(i), match); - Configuration.logMessage(this,SeverityLevel.INFO,false,Messages.get("identical_case_found_in:_") + resultsPool.get(i).getAbsolutePath()); - } - } - return output_files; - }*/ // pour trouver les fichiers out potentiels: ceux contenus dans le repertoire "output" dont le repertoire 'input" contient les fichiers ayant le mm md5 que les fichiers input... public File getMatchingOutputDirinCache(File tmpinputDir, String code) { + return getMatchingOutputDirinCache(tmpinputDir, code); + } + + /** + * Return matching output directory if the input dir is the same (same md5 for all files) + * If wantedCheckSum!=null, we only check if there is a file in "input" that has this md5. Other files can be differents + * + * @param tmpinputDir - the input directory to check + * @param code - the code to match + * @param wantedCheckSum - md5sum of a unique file we want to find in "input" directory + * + * @return the "output" directory if we found a match + */ + public File getMatchingOutputDirinCache(File tmpinputDir, String code, byte[] wantedCheckSum) { assert tmpinputDir != null; while (!initialized) { @@ -298,32 +309,46 @@ public File getMatchingOutputDirinCache(File tmpinputDir, String code) { NewInputDir idir = new NewInputDir(tmpinputDir); synchronized (poolInputDirs) { + //byte[] wantedCheckSum = Digest.getSum(fileToMatch); for (int i = 0; i < poolInputDirs.size(); i++) { - if (!poolInputDirs.get(i).mayHaveFailed() && poolInputDirs.get(i).getCode() != null && poolInputDirs.get(i).getCode().equals(code) && poolInputDirs.get(i).contentEquals(idir)) { - File possible = poolInputDirs.get(i).getOutputDir(); - File[] possible_content = possible.listFiles(); - if (possible_content.length > 0) { - boolean out = false; - boolean err = false; - for (File file : possible_content) { - if (file.getName().equals("out.txt")) { - out = true; + CacheInputDir cacheInputDir = poolInputDirs.get(i); + if (!cacheInputDir.mayHaveFailed() + && cacheInputDir.getCode() != null + && cacheInputDir.getCode().equals(code)) { + + boolean foundDirMath = false; + if(wantedCheckSum != null) { + foundDirMath = cacheInputDir.hasCheckSum(wantedCheckSum); + } else { + foundDirMath = cacheInputDir.contentEquals(idir); + } + if(foundDirMath) { + File possible = poolInputDirs.get(i).getOutputDir(); + File[] possible_content = possible.listFiles(); + if (possible_content.length > 0) { + boolean out = false; + boolean err = false; + for (File file : possible_content) { + if (file.getName().equals("out.txt")) { + out = true; + } + if (file.getName().equals("err.txt")) { + err = true; + } } - if (file.getName().equals("err.txt")) { - err = true; + if (out && err) { + assert possible != null : "The output dir of input pool " + poolInputDirs.get(i).dir.getAbsolutePath() + " is empty !"; + Log.logMessage(this, SeverityLevel.INFO, false, "Identical case found: " + poolInputDirs.get(i).dir.getAbsolutePath()); + //synchronized (poolInputDirs) { + poolInputDirs.remove(i); + //} + return possible; + } else { + poolInputDirs.remove(i); } } - if (out && err) { - assert possible != null : "The output dir of input pool " + poolInputDirs.get(i).dir.getAbsolutePath() + " is empty !"; - Log.logMessage(this, SeverityLevel.INFO, false, "Identical case found: " + poolInputDirs.get(i).dir.getAbsolutePath()); - //synchronized (poolInputDirs) { - poolInputDirs.remove(i); - //} - return possible; - } else { - poolInputDirs.remove(i); - } } + } } } diff --git a/src/main/java/org/funz/parameter/Case.java b/src/main/java/org/funz/parameter/Case.java index 8fc87e7..521aaa1 100644 --- a/src/main/java/org/funz/parameter/Case.java +++ b/src/main/java/org/funz/parameter/Case.java @@ -299,6 +299,7 @@ public int getType() { } } public static final String FILE_INFO = "info.txt"; + public static final String OLD_FILE_INFO = "old.info.txt"; /** * Modification type. */