diff --git a/karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/DeleteLinkCommand.java b/karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/DeleteLinkCommand.java index bfba43168..aa56a1df2 100644 --- a/karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/DeleteLinkCommand.java +++ b/karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/DeleteLinkCommand.java @@ -101,6 +101,8 @@ private UpdateContainer deleteLink(Alignment alignment, OntologyManager ontMgr, LabeledLink delLink = alignment.getLinkById(linkId); if(delLink != null) { this.displayLabel = delLink.getLabel().getDisplayName(); + } else { + this.displayLabel = edgeUri; } alignment.removeLink(linkId); diff --git a/karma-commands/commands-bloom/pom.xml b/karma-commands/commands-bloom/pom.xml index 25accfe85..df8bafd50 100644 --- a/karma-commands/commands-bloom/pom.xml +++ b/karma-commands/commands-bloom/pom.xml @@ -12,7 +12,7 @@ commands-bloom - 2.4.0.2.1.3.0-563 + 2.9.0 @@ -102,4 +102,4 @@ - \ No newline at end of file + diff --git a/karma-commands/commands-common/src/main/java/edu/isi/karma/controller/update/WorksheetHeadersUpdate.java b/karma-commands/commands-common/src/main/java/edu/isi/karma/controller/update/WorksheetHeadersUpdate.java index 88882c03f..993315e04 100644 --- a/karma-commands/commands-common/src/main/java/edu/isi/karma/controller/update/WorksheetHeadersUpdate.java +++ b/karma-commands/commands-common/src/main/java/edu/isi/karma/controller/update/WorksheetHeadersUpdate.java @@ -46,7 +46,7 @@ public class WorksheetHeadersUpdate extends AbstractUpdate { private enum JsonKeys { worksheetId, columns, columnName, characterLength, hasNestedTable, columnClass, hNodeId, pythonTransformation, previousCommandId, - columnDerivedFrom, hNodeType, status, onError + columnDerivedFrom, hNodeType, status, onError, selectionPyCode } public WorksheetHeadersUpdate(String worksheetId, SuperSelection selection) { @@ -101,7 +101,10 @@ private JSONObject getColumnJsonObject(VHNode hNode, ColumnMetadata colMeta) thr if (t.hasNestedTable()) { Selection sel = selection.getSelection(t.getNestedTable().getId()); if (sel != null) + { hNodeObj.put(JsonKeys.status.name(), sel.getStatus().name()); + hNodeObj.put(JsonKeys.selectionPyCode.name(), colMeta.getSelectionPythonCode(t.getNestedTable().getId())); + } } hNodeObj.put(JsonKeys.hNodeType.name(), t.getHNodeType().name()); Integer colLength = colMeta.getColumnPreferredLength(hNode.getId()); diff --git a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/AddColumnConsolidator.java b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/AddColumnConsolidator.java new file mode 100644 index 000000000..8f8ae5ff7 --- /dev/null +++ b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/AddColumnConsolidator.java @@ -0,0 +1,47 @@ +package edu.isi.karma.controller.command.transformation; + +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; + +import edu.isi.karma.controller.command.Command; +import edu.isi.karma.controller.command.ICommand; +import edu.isi.karma.controller.command.worksheet.AddColumnCommand; +import edu.isi.karma.controller.history.CommandConsolidator; +import edu.isi.karma.controller.history.HistoryJsonUtil; +import edu.isi.karma.rep.Workspace; + +public class AddColumnConsolidator extends CommandConsolidator { + + @Override + public Pair consolidateCommand(List commands, + ICommand newCommand, Workspace workspace) { + if (newCommand instanceof SubmitPythonTransformationCommand) { + String model = newCommand.getModel(); + + Iterator itr = commands.iterator(); + while(itr.hasNext()) { + ICommand tmp = itr.next(); + if (((Command)tmp).getOutputColumns().equals(((Command)newCommand).getOutputColumns()) + && tmp.getModel().equals(model) + && (tmp instanceof AddColumnCommand)) { + SubmitPythonTransformationCommand py = ((SubmitPythonTransformationCommand)newCommand); + JSONArray inputJSON = new JSONArray(py.getInputParameterJson()); + + JSONArray tmpInputJSON = new JSONArray(tmp.getInputParameterJson()); + String hNodeId = HistoryJsonUtil.getStringValue("hNodeId", tmpInputJSON); + + HistoryJsonUtil.setArgumentValue("hNodeId", hNodeId, inputJSON); + py.setInputParameterJson(inputJSON.toString()); + + return new ImmutablePair<>(tmp, (Object)py); + } + } + } + return null; + } + +} diff --git a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PyTransformConsolidator.java b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PyTransformConsolidator.java index 13c95e5d5..d2eee86c6 100644 --- a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PyTransformConsolidator.java +++ b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PyTransformConsolidator.java @@ -1,20 +1,17 @@ package edu.isi.karma.controller.command.transformation; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; + import edu.isi.karma.controller.command.Command; -import edu.isi.karma.controller.command.CommandException; import edu.isi.karma.controller.command.ICommand; import edu.isi.karma.controller.history.CommandConsolidator; import edu.isi.karma.controller.history.HistoryJsonUtil; -import edu.isi.karma.controller.history.WorksheetCommandHistoryExecutor; import edu.isi.karma.rep.Workspace; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; public class PyTransformConsolidator extends CommandConsolidator { diff --git a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PythonTransformationCommand.java b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PythonTransformationCommand.java index c4038925b..ad9635d80 100755 --- a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PythonTransformationCommand.java +++ b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/PythonTransformationCommand.java @@ -130,7 +130,8 @@ protected void generateTransformedValues(Workspace workspace, repo.initializeInterpreter(interpreter); Collection nodes = new ArrayList<>(Math.max(1000, worksheet .getDataTable().getNumRows())); - worksheet.getDataTable().collectNodes(hNode.getHNodePath(f), nodes, selection); + SuperSelection tmpSelection = new SuperSelection("TEMP_SUPER_SELECTION"); + worksheet.getDataTable().collectNodes(hNode.getHNodePath(f), nodes, tmpSelection); Map rowToValueMap = new HashMap<>(); diff --git a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/SubmitPythonTransformationCommand.java b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/SubmitPythonTransformationCommand.java index 4c71f5b91..6845d8d89 100644 --- a/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/SubmitPythonTransformationCommand.java +++ b/karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/SubmitPythonTransformationCommand.java @@ -70,7 +70,7 @@ public SubmitPythonTransformationCommand(String id, String model, String newColu //logger.info("SubmitPythonTranformationCommand:" + id + " newColumnName:" + newColumnName + ", code=" + transformationCode); this.pythonNodeId = hNodeId; } - + @Override public String getCommandName() { return this.getClass().getSimpleName(); @@ -124,6 +124,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { Map mapping = gatherTransformedResults(workspace, nodeId); handleJSONOutput(workspace, mapping, newColumnNameHNode); } + newColumnAbsoluteName = newColumnNameHNode.getAbsoluteColumnName(f); WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this); return c; } else { diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/AddColumnCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/AddColumnCommand.java index 09f814184..0372dd458 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/AddColumnCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/AddColumnCommand.java @@ -156,7 +156,6 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { c.append(WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, getSuperSelection(worksheet), workspace.getContextId())); c.append(computeAlignmentAndSemanticTypesAndCreateUpdates(workspace)); - inputColumns.add(hNodeId); outputColumns.add(newHNodeId); newColumnAbsoluteName = ndid.getAbsoluteColumnName(workspace.getFactory()); return c; @@ -194,10 +193,7 @@ public UpdateContainer undoIt(Workspace workspace) { return WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, getSuperSelection(worksheet), workspace.getContextId()); } - public String getNewHNodeId() { return newHNodeId; } - - } diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java index ab099f1a7..bed3ed197 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java @@ -79,6 +79,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { } } + worksheet.getMetadataContainer().getColumnMetadata().removeSelectionPythonCode(hTable.getId()); WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this); if(!this.isExecutedInBatch()) { UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java index 55e3f136b..a187dd190 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java @@ -3,11 +3,8 @@ import edu.isi.karma.controller.command.CommandException; import edu.isi.karma.controller.command.CommandType; import edu.isi.karma.controller.command.WorksheetSelectionCommand; -import edu.isi.karma.controller.command.selection.LargeSelection; -import edu.isi.karma.controller.command.selection.LargeSelection.Operation; import edu.isi.karma.controller.command.selection.MiniSelection; import edu.isi.karma.controller.command.selection.Selection; -import edu.isi.karma.controller.command.selection.SelectionManager; import edu.isi.karma.controller.command.selection.SuperSelection; import edu.isi.karma.controller.update.ErrorUpdate; import edu.isi.karma.controller.update.UpdateContainer; @@ -65,28 +62,18 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { RepFactory factory = workspace.getFactory(); SuperSelection superSel = this.getSuperSelection(worksheet); HTable hTable = factory.getHTable(factory.getHNode(hNodeId).getHTableId()); - Selection currentSel = superSel.getSelection(hTable.getId()); Selection anotherSel = null; - if (!operation.equalsIgnoreCase(Operation.Invert.name())) { - anotherSel = new MiniSelection(workspace, worksheetId, hTable.getId(), factory.getNewId("SEL"), superSel.getName(), pythonCode, onError); - worksheet.getSelectionManager().addSelection(anotherSel); - } - if (currentSel == null && operation.equalsIgnoreCase(Operation.Invert.name()) ) { - return getErrorUpdate("No defined Selection"); - } - if (currentSel == null) { - currentSel = new MiniSelection(workspace, worksheetId, hTable.getId(), factory.getNewId("SEL"), superSel.getName(), SelectionManager.defaultCode, onError); - worksheet.getSelectionManager().addSelection(currentSel); - } + + anotherSel = new MiniSelection(workspace, worksheetId, hTable.getId(), factory.getNewId("SEL"), superSel.getName(), pythonCode, onError); + worksheet.getSelectionManager().addSelection(anotherSel); + worksheet.getMetadataContainer().getColumnMetadata().addSelectionPythonCode(hTable.getId(), this.pythonCode); + try { - Operation operation = Operation.valueOf(Operation.class, this.operation); - Selection t = new LargeSelection(workspace, worksheetId, hTable.getId(), factory.getNewId("SEL"), superSel.getName(), currentSel, anotherSel, operation); - worksheet.getSelectionManager().addSelection(t); - outputColumns.addAll(t.getInputColumns()); - previousSelection = superSel.getSelection(t.getHTableId()); + outputColumns.addAll(anotherSel.getInputColumns()); + previousSelection = superSel.getSelection(hTable.getId()); if (previousSelection != null) superSel.removeSelection(previousSelection); - superSel.addSelection(t); + superSel.addSelection(anotherSel); }catch (Exception e) { return getErrorUpdate("The operation is undefined"); @@ -104,6 +91,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { public UpdateContainer undoIt(Workspace workspace) { inputColumns.clear(); outputColumns.clear(); + RepFactory factory = workspace.getFactory(); Worksheet worksheet = workspace.getWorksheet(worksheetId); SuperSelection superSel = getSuperSelection(worksheet); HNode hNode = workspace.getFactory().getHNode(hNodeId); @@ -116,6 +104,8 @@ public UpdateContainer undoIt(Workspace workspace) { worksheet.getSelectionManager().removeSelection(currentSel); superSel.removeSelection(currentSel); } + worksheet.getMetadataContainer().getColumnMetadata().removeSelectionPythonCode(factory.getHTable(factory.getHNode(hNodeId).getHTableId()).getId()); + WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this); UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); uc.add(new WorksheetSuperSelectionListUpdate(worksheetId)); diff --git a/karma-common/src/main/java/edu/isi/karma/controller/history/CommandHistory.java b/karma-common/src/main/java/edu/isi/karma/controller/history/CommandHistory.java index b4a4efaa1..5dd347477 100644 --- a/karma-common/src/main/java/edu/isi/karma/controller/history/CommandHistory.java +++ b/karma-common/src/main/java/edu/isi/karma/controller/history/CommandHistory.java @@ -30,14 +30,25 @@ import edu.isi.karma.controller.command.ICommand.CommandTag; import edu.isi.karma.controller.history.HistoryJsonUtil.ClientJsonKeys; import edu.isi.karma.controller.history.HistoryJsonUtil.ParameterType; +import edu.isi.karma.controller.update.AbstractUpdate; import edu.isi.karma.controller.update.HistoryUpdate; +import edu.isi.karma.controller.update.TrivialErrorUpdate; import edu.isi.karma.controller.update.UpdateContainer; +import edu.isi.karma.modeling.steiner.topk.Graph; import edu.isi.karma.rep.HNode; +import edu.isi.karma.rep.Node; +import edu.isi.karma.rep.RepFactory; import edu.isi.karma.rep.Workspace; import edu.isi.karma.util.JSONUtil; import edu.isi.karma.view.VWorkspace; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.jgrapht.DirectedGraph; +import org.jgrapht.alg.CycleDetector; +import org.jgrapht.graph.DefaultDirectedGraph; +import org.jgrapht.graph.DefaultEdge; +import org.jgrapht.graph.DirectedMultigraph; import org.json.JSONArray; import org.json.JSONObject; import org.reflections.Reflections; @@ -137,6 +148,7 @@ public UpdateContainer doCommand(Command command, Workspace workspace, boolean s Pair consolidatedCommand = null; String consolidatorName = null; String worksheetId = worksheetCommandHistory.getWorksheetId(command); + RepFactory factory = workspace.getFactory(); List potentialConsolidateCommands = worksheetCommandHistory.getCommandsFromWorksheetIdAndCommandTag(worksheetId, command.getTagFromPriority()); for (CommandConsolidator consolidator : consolidators) { consolidatedCommand = consolidator.consolidateCommand(potentialConsolidateCommands, command, workspace); @@ -149,7 +161,37 @@ public UpdateContainer doCommand(Command command, Workspace workspace, boolean s if (consolidatedCommand != null) { worksheetCommandHistory.setStale(worksheetId, true); if (consolidatorName.equals("PyTransformConsolidator")) { - effects.append(consolidatedCommand.getLeft().doIt(workspace)); + Command runCommand = (Command)consolidatedCommand.getLeft(); + effects.append(runCommand.doIt(workspace)); + + if(runCommand.getInputColumns().size() > 0 && + !runCommand.getInputColumns().equals(command.getInputColumns())) { + Pair> cycleDetect = detectTransformCycle(worksheetId); + if(cycleDetect.getLeft() == false) { + + placeTransformCommandByInput(runCommand); + } else { + StringBuilder columns = new StringBuilder(); + String sep = ""; + System.out.println(cycleDetect.getRight()); + for(String nodeId : cycleDetect.getRight()) { + HNode node = factory.getHNode(nodeId); + columns.append(sep); + if(node != null) + columns.append(node.getAbsoluteColumnName(factory)); + else { + Node repNode = factory.getNode(nodeId); + if(repNode != null) + columns.append(repNode.prettyPrint(factory)); + else + columns.append(nodeId); + } + sep = ", "; + } + AbstractUpdate update = new TrivialErrorUpdate("A cycle occurs in the Column Transforms for Columns: " + columns.toString() + ". Please fix the cycle"); + effects.add(update); + } + } } if (consolidatorName.equals("UnassignSemanticTypesConsolidator")) { worksheetCommandHistory.removeCommandFromHistory(Arrays.asList(consolidatedCommand.getLeft())); @@ -179,6 +221,12 @@ public UpdateContainer doCommand(Command command, Workspace workspace, boolean s worksheetCommandHistory.replaceCommandFromHistory(consolidatedCommand.getKey(), (ICommand)consolidatedCommand.getRight()); effects.append(((ICommand) consolidatedCommand.getRight()).doIt(workspace)); } + if (consolidatorName.equals("AddColumnConsolidator")) { + worksheetCommandHistory.removeCommandFromHistory(Arrays.asList(consolidatedCommand.getLeft())); + ICommand runCommand = ((ICommand) consolidatedCommand.getRight()); + worksheetCommandHistory.insertCommandToHistory(runCommand); + effects.append(runCommand.doIt(workspace)); + } } else { effects.append(command.doIt(workspace)); @@ -219,6 +267,74 @@ public UpdateContainer doCommand(Command command, Workspace workspace, boolean s return effects; } + private void placeTransformCommandByInput(Command command) { + ICommand placeAfterCmd = null; + Set cmdInputs = command.getInputColumns(); + Set cmdOutputs = command.getOutputColumns(); + List commandsToMove = new ArrayList<>(); + for (ICommand afterCmd : worksheetCommandHistory.getCommandsAfterCommand(command, CommandTag.Transformation)) { + Set afterCmdOutputs = ((Command)afterCmd).getOutputColumns(); + Set afterCmdInputs = ((Command)afterCmd).getInputColumns(); + for(String input : cmdInputs) { + if(afterCmdOutputs.contains(input)) { + placeAfterCmd = afterCmd; + break; + } + } + for(String output : cmdOutputs) { + if(afterCmdInputs.contains(output)) { + commandsToMove.add((Command)afterCmd); + break; + } + } + } + + if (placeAfterCmd != null) { + worksheetCommandHistory.removeCommandFromHistory(Arrays.asList((ICommand)command)); + JSONArray inputJSON = new JSONArray(command.getInputParameterJson()); + JSONArray placeAfterCmdInputJSON = new JSONArray(command.getInputParameterJson()); + String placeHNodeId = HistoryJsonUtil.getStringValue("hNodeId", placeAfterCmdInputJSON); + HistoryJsonUtil.setArgumentValue("hNodeId", placeHNodeId, inputJSON); + command.setInputParameterJson(inputJSON.toString()); + worksheetCommandHistory.insertCommandToHistoryAfterCommand(command, placeAfterCmd); + } + + for(Command cmd : commandsToMove) + placeTransformCommandByInput(cmd); + } + + private Pair> detectTransformCycle(String worksheetId) { + DirectedGraph historyInputOutputGraph = new DefaultDirectedGraph<>(DefaultEdge.class); + for (ICommand cmd : worksheetCommandHistory.getCommandsFromWorksheetIdAndCommandTag(worksheetId, CommandTag.Transformation)) { + Set outputs = ((Command)cmd).getOutputColumns(); + Set inputs = ((Command)cmd).getInputColumns(); + String commandId = "Command-" + cmd.getId(); + historyInputOutputGraph.addVertex(commandId); + for(String input : inputs) { + historyInputOutputGraph.addVertex(input); + historyInputOutputGraph.addEdge(input, commandId); + } + for(String output : outputs) { + historyInputOutputGraph.addVertex(output); + historyInputOutputGraph.addEdge(commandId, output); + } + } + CycleDetector cycleDetector = new CycleDetector<>(historyInputOutputGraph); + if(cycleDetector.detectCycles()) { + Set cycleVertices = cycleDetector.findCycles(); + Set nodeVertices = new HashSet<>(); + Iterator iterator = cycleVertices.iterator(); + while(iterator.hasNext()) { + String node = iterator.next(); + if(!node.startsWith("Command-")) + nodeVertices.add(node); + } + + return new ImmutablePair<>(true, nodeVertices); + } + return new ImmutablePair>(false, null); + } + private void writeHistoryPerWorksheet(Workspace workspace, IHistorySaver historySaver) throws Exception { String workspaceId = workspace.getId(); Map comMap = new HashMap<>(); diff --git a/karma-common/src/main/java/edu/isi/karma/controller/history/HistoryJsonUtil.java b/karma-common/src/main/java/edu/isi/karma/controller/history/HistoryJsonUtil.java index 2897132f2..9ce23fd62 100644 --- a/karma-common/src/main/java/edu/isi/karma/controller/history/HistoryJsonUtil.java +++ b/karma-common/src/main/java/edu/isi/karma/controller/history/HistoryJsonUtil.java @@ -35,7 +35,11 @@ public static JSONObject getJSONObjectWithName(String arg, JSONArray json) throw } public static JSONArray getJSONArrayValue(String arg, JSONArray json) throws JSONException { - return getJSONObjectWithName(arg, json).getJSONArray(ClientJsonKeys.value.name()); + JSONObject obj = getJSONObjectWithName(arg, json); + if(obj.has(ClientJsonKeys.value.name())) + return obj.getJSONArray(ClientJsonKeys.value.name()); + else + return new JSONArray(); } public static ParameterType getParameterType(JSONObject json) throws JSONException { diff --git a/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistory.java b/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistory.java index 03dc5cc98..1426c32fe 100644 --- a/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistory.java +++ b/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistory.java @@ -1,11 +1,15 @@ package edu.isi.karma.controller.history; import edu.isi.karma.controller.command.ICommand; + import org.apache.commons.lang3.tuple.Pair; +import org.jgrapht.graph.DefaultEdge; +import org.jgrapht.graph.DirectedMultigraph; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.*; +import edu.isi.karma.controller.command.Command; /** * Created by Frank on 9/14/15. @@ -53,8 +57,7 @@ public void removeCommandFromHistory(List commands) { private final Map historyWorksheetMap = new TreeMap<>(); private static final String IMPORT_COMMANDS = "ImportCommands"; - - + public WorksheetCommandHistory() { historyWorksheetMap.put(IMPORT_COMMANDS, new CommandTagListMap()); } @@ -94,6 +97,22 @@ public void insertCommandToHistory(ICommand command) { commandTagListMap.addCommandToHistory(command); } + public void insertCommandToHistoryAfterCommand(ICommand newCommand, ICommand afterCommand) { + String worksheetId = getWorksheetId(afterCommand); + if (worksheetId == null) { + worksheetId = IMPORT_COMMANDS; + } + CommandTagListMap commandTagListMap = historyWorksheetMap.get(worksheetId); + if (commandTagListMap != null) { + for (Map.Entry> entry : commandTagListMap.commandTagListHashMap.entrySet()) { + int index = entry.getValue().indexOf(afterCommand); + if (index != -1) { + entry.getValue().add(index+1, newCommand); + } + } + } + } + public void setLastRedoCommandObject(RedoCommandObject command) { String worksheetId = getWorksheetId(command.getCommand()); @@ -200,6 +219,29 @@ public List getCommandsFromWorksheetIdAndCommandTag(String worksheetId return commands; } + public List getCommandsAfterCommand(ICommand command, ICommand.CommandTag commandTag) { + List commands = new ArrayList<>(); + String worksheetId = getWorksheetId(command); + if (worksheetId == null) { + worksheetId = IMPORT_COMMANDS; + } + CommandTagListMap map = historyWorksheetMap.get(worksheetId); + if (map != null) { + List tagCommands = map.getCommands(commandTag); + boolean start = false; + for(ICommand cmd : tagCommands) { + if(cmd.equals(command)) { + start = true; + continue; + } + if(start) + commands.add(cmd); + } + commands.addAll(map.getCommands(commandTag)); + } + return commands; + } + public RedoCommandObject getLastRedoCommandObject(String worksheetId) { if (worksheetId == null) { worksheetId = IMPORT_COMMANDS; diff --git a/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistoryExecutor.java b/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistoryExecutor.java index 811f01583..b4720ed0a 100644 --- a/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistoryExecutor.java +++ b/karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistoryExecutor.java @@ -45,7 +45,6 @@ import edu.isi.karma.rep.HNode.HNodeType; import edu.isi.karma.rep.HTable; import edu.isi.karma.rep.Workspace; -import edu.isi.karma.util.Util; import edu.isi.karma.webserver.ExecutionController; import edu.isi.karma.webserver.KarmaException; import edu.isi.karma.webserver.WorkspaceRegistry; @@ -181,13 +180,8 @@ public UpdateContainer normalizeCommandHistoryJsonInput(Workspace workspace, Str logger.debug("Column being normalized: "+ nameObjColumnName); HNode node = hTable.getHNodeFromColumnName(nameObjColumnName); if(node == null) { //Because add column can happen even if the column after which it is to be added is not present - AbstractUpdate update = new TrivialErrorUpdate(nameObjColumnName + " does not exist, using empty values"); - if (uc == null) - uc = new UpdateContainer(update); - else - uc.add(update); if (addIfNonExist) { - node = hTable.addHNode(nameObjColumnName, HNodeType.Regular, workspace.getWorksheet(worksheetId), workspace.getFactory()); + node = hTable.addHNode(nameObjColumnName, HNodeType.Transformation, workspace.getWorksheet(worksheetId), workspace.getFactory()); } else { continue; diff --git a/karma-common/src/main/java/edu/isi/karma/er/helper/PythonRepository.java b/karma-common/src/main/java/edu/isi/karma/er/helper/PythonRepository.java index 11bcedf00..4f02ff37d 100644 --- a/karma-common/src/main/java/edu/isi/karma/er/helper/PythonRepository.java +++ b/karma-common/src/main/java/edu/isi/karma/er/helper/PythonRepository.java @@ -4,7 +4,6 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -51,6 +50,9 @@ private void initialize() compileAndAddToRepository(interpreter, PythonTransformationHelper.getVDefStatement()); compileAndAddToRepository(interpreter, PythonTransformationHelper.getTransformStatement()); compileAndAddToRepository(interpreter, PythonTransformationHelper.getSelectionStatement()); + compileAndAddToRepository(interpreter, PythonTransformationHelper.getModelName()); + compileAndAddToRepository(interpreter, PythonTransformationHelper.getModelPrefix()); + compileAndAddToRepository(interpreter, PythonTransformationHelper.getModelBaseUri()); initializeInterpreter(interpreter); } @@ -96,6 +98,10 @@ public void initializeInterpreter(PythonInterpreter interpreter) interpreter.exec(scripts.get(PythonTransformationHelper.getGetValueFromNestedColumnByIndexDefStatement())); interpreter.exec(scripts.get(PythonTransformationHelper.getRowIndexDefStatement())); interpreter.exec(scripts.get(PythonTransformationHelper.getVDefStatement())); + interpreter.exec(scripts.get(PythonTransformationHelper.getModelName())); + interpreter.exec(scripts.get(PythonTransformationHelper.getModelPrefix())); + interpreter.exec(scripts.get(PythonTransformationHelper.getModelBaseUri())); + } if(localsUninitialized ||(!libraryHasBeenLoaded || reloadLibrary)) { diff --git a/karma-common/src/main/java/edu/isi/karma/er/helper/PythonTransformationHelper.java b/karma-common/src/main/java/edu/isi/karma/er/helper/PythonTransformationHelper.java index 269fcf69c..6deff01fe 100644 --- a/karma-common/src/main/java/edu/isi/karma/er/helper/PythonTransformationHelper.java +++ b/karma-common/src/main/java/edu/isi/karma/er/helper/PythonTransformationHelper.java @@ -39,6 +39,10 @@ public class PythonTransformationHelper { private static String importStatement = null; private static String valueFromNestedColumnByIndexDefStatement = null; private static String getRowIndexDefStatement = null; + private static String modelNameStatement = null; + private static String modelPrefixStatement = null; + private static String modelBaseUriStatement = null; + public static String getPyObjectValueAsString(PyObject obj) { if (obj == null) return ""; @@ -99,6 +103,7 @@ public static String getImportStatements() { importStmt.append("import json\n"); importStmt.append("import edu.isi.karma.rep.WorkspaceManager\n"); importStmt.append("import edu.isi.karma.rep.Workspace\n"); + importStmt.append("import edu.isi.karma.rep.Worksheet\n"); importStmt.append("import edu.isi.karma.rep.Node\n"); importStmt.append("import edu.isi.karma.rep.Table\n"); importStmt.append("import edu.isi.karma.rep.HTable\n"); @@ -106,6 +111,9 @@ public static String getImportStatements() { importStmt.append("import edu.isi.karma.rep.RepFactory\n"); importStmt.append("import edu.isi.karma.er.helper.PythonTransformationHelper\n"); importStmt.append("import edu.isi.karma.controller.command.transformation.PythonTransformationCommand\n"); + importStmt.append("import edu.isi.karma.rep.metadata.WorksheetProperties\n"); + importStmt.append("import edu.isi.karma.rep.metadata.WorksheetProperties.Property\n"); + importStatement = importStmt.toString(); } return importStatement; @@ -197,6 +205,50 @@ public static String getGetValueFromNestedColumnByIndexDefStatement() { } + public static String getModelName() { + if(modelNameStatement == null) { + StringBuilder methodStmt = new StringBuilder(); + methodStmt.append("def getModelName():\n"); + methodStmt.append(" worksheet = edu.isi.karma.rep.WorkspaceManager.getInstance().getWorkspace(workspaceid).getWorksheet(worksheetId)\n"); + methodStmt.append(" props = worksheet.getMetadataContainer().getWorksheetProperties()\n"); + methodStmt.append(" modelName = props.getPropertyValue(edu.isi.karma.rep.metadata.WorksheetProperties.Property.graphLabel)\n"); + methodStmt.append(" if modelName is not None:\n"); + methodStmt.append(" return modelName\n"); + methodStmt.append(" return ''\n"); + modelNameStatement = methodStmt.toString(); + } + return modelNameStatement; + } + + public static String getModelPrefix() { + if(modelPrefixStatement == null) { + StringBuilder methodStmt = new StringBuilder(); + methodStmt.append("def getModelPrefix():\n"); + methodStmt.append(" worksheet = edu.isi.karma.rep.WorkspaceManager.getInstance().getWorkspace(workspaceid).getWorksheet(worksheetId)\n"); + methodStmt.append(" props = worksheet.getMetadataContainer().getWorksheetProperties()\n"); + methodStmt.append(" prefix = props.getPropertyValue(edu.isi.karma.rep.metadata.WorksheetProperties.Property.prefix)\n"); + methodStmt.append(" if prefix is not None:\n"); + methodStmt.append(" return prefix\n"); + methodStmt.append(" return ''\n"); + modelPrefixStatement = methodStmt.toString(); + } + return modelPrefixStatement; + } + + public static String getModelBaseUri() { + if(modelBaseUriStatement == null) { + StringBuilder methodStmt = new StringBuilder(); + methodStmt.append("def getModelBaseUri():\n"); + methodStmt.append(" worksheet = edu.isi.karma.rep.WorkspaceManager.getInstance().getWorkspace(workspaceid).getWorksheet(worksheetId)\n"); + methodStmt.append(" props = worksheet.getMetadataContainer().getWorksheetProperties()\n"); + methodStmt.append(" uri = props.getPropertyValue(edu.isi.karma.rep.metadata.WorksheetProperties.Property.baseURI)\n"); + methodStmt.append(" if uri is not None:\n"); + methodStmt.append(" return uri\n"); + methodStmt.append(" return ''\n"); + modelBaseUriStatement = methodStmt.toString(); + } + return modelBaseUriStatement; + } public static String getIsEmptyDefStatement() { diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphBuilder.java b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphBuilder.java index ddd151a7a..9f900d740 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphBuilder.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphBuilder.java @@ -390,6 +390,8 @@ public boolean addNode(Node node) { } node.getLabel().setNs(label.getNs()); node.getLabel().setPrefix(label.getPrefix()); + node.getLabel().setRdfsLabel(label.getRdfsLabel()); + node.getLabel().setRdfsComment(label.getRdfsComment()); } if(node.isForced()) diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyCache.java b/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyCache.java index 598c1ac33..4406d52d9 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyCache.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyCache.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -515,6 +514,7 @@ private void loadClasses() { } + Property rdfType = this.ontHandler.getOntModel().createProperty(Uris.RDF_TYPE_URI); Resource classNode = this.ontHandler.getOntModel().createResource(Uris.RDFS_CLASS_URI); ResIterator itr = ontHandler.getOntModel().listSubjectsWithProperty(rdfType, classNode); @@ -526,9 +526,13 @@ private void loadClasses() { if (!r.isURIResource()) continue; - if (!classes.containsKey(r.getURI())) - classes.put(r.getURI(), ontHandler.getResourceLabel(r)); - + if (!classes.containsKey(r.getURI())) { + OntClass c = this.ontHandler.getOntModel().getOntClass(r.getURI()); + if(c != null) + classes.put(c.getURI(), ontHandler.getResourceLabel(c)); + else + classes.put(r.getURI(), ontHandler.getResourceLabel(r)); + } } } @@ -544,6 +548,8 @@ private void loadProperties() { this.objectProperties.put(Uris.RDF_TYPE_URI, new Label(Uris.RDF_TYPE_URI, Namespaces.RDF, Prefixes.RDF)); this.objectProperties.put(Uris.RDF_VALUE_URI, new Label(Uris.RDF_VALUE_URI, Namespaces.RDF, Prefixes.RDF)); this.objectProperties.put(Uris.RDFS_SUBCLASS_URI, new Label(Uris.RDFS_SUBCLASS_URI, Namespaces.RDFS, Prefixes.RDFS)); + + this.dataProperties.put(Uris.RDF_VALUE_URI, new Label(Uris.RDF_VALUE_URI, Namespaces.RDF, Prefixes.RDF)); this.dataProperties.put(Uris.RDFS_LABEL_URI, new Label(Uris.RDFS_LABEL_URI, Namespaces.RDFS, Prefixes.RDFS)); this.dataProperties.put(Uris.RDFS_COMMENT_URI, new Label(Uris.RDFS_COMMENT_URI, Namespaces.RDFS, Prefixes.RDFS)); diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyHandler.java b/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyHandler.java index e5fca65e3..01dd3ecde 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyHandler.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyHandler.java @@ -81,18 +81,30 @@ public Label getResourceLabel(Resource r) { OntResource ontR = null; try { ontR = (OntResource)r;} catch(Exception e) {} - if (ontR == null) + if (ontR == null) { + logger.error("No rdfs:label and rdfs:comment for resource:" + r.toString()); return new Label(r.getURI(), ns, prefix); - + } //Get the rdfs:label and comment in English //If one is not available in English, then try and get one in any other language - String rdfsLabel = ontR.getLabel("EN"); - if(rdfsLabel == null) - rdfsLabel = ontR.getLabel(null); - String rdfsComment = ontR.getComment("EN"); - if(rdfsComment == null) - rdfsComment = ontR.getComment(null); - + String rdfsLabel; + try { + rdfsLabel = ontR.getLabel("EN"); + if(rdfsLabel == null) + rdfsLabel = ontR.getLabel(null); + } catch(Exception e) { + logger.error("No rdfs:label for resource:" + r.toString()); + rdfsLabel = ""; + } + String rdfsComment; + try { + rdfsComment = ontR.getComment("EN"); + if(rdfsComment == null) + rdfsComment = ontR.getComment(null); + } catch(Exception e) { + logger.error("No Comment for resource:" + r.toString()); + rdfsComment = ""; + } return new Label(r.getURI(), ns, prefix, rdfsLabel, rdfsComment); } diff --git a/karma-common/src/main/java/edu/isi/karma/rep/ColumnMetadata.java b/karma-common/src/main/java/edu/isi/karma/rep/ColumnMetadata.java index 550643c86..86ae94949 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/ColumnMetadata.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/ColumnMetadata.java @@ -37,6 +37,8 @@ public class ColumnMetadata { private Map columnDerivedFrom; private Map columnDataStructure; private Map columnOnError; + private Map columnSelectionPythonCode; + public ColumnMetadata() { super(); this.columnPreferredLengths = new HashMap<>(); @@ -48,6 +50,7 @@ public ColumnMetadata() { this.columnDerivedFrom = new HashMap<>(); this.columnDataStructure = new HashMap<>(); this.columnOnError = new HashMap<>(); + this.columnSelectionPythonCode = new HashMap<>(); } public enum DataStructure { @@ -108,6 +111,21 @@ public void addColumnPythonTransformation(String hNodeId, String pythonTransform columnPythonTransform.put(hNodeId, pythonTransform); } + public void addSelectionPythonCode(String hNodeId, String pythonTransform) + { + columnSelectionPythonCode.put(hNodeId, pythonTransform); + } + + public void removeSelectionPythonCode(String hNodeId) + { + columnSelectionPythonCode.remove(hNodeId); + } + + public String getSelectionPythonCode(String hNodeId) + { + return columnSelectionPythonCode.get(hNodeId); + } + public void addPreviousCommandId(String hNodeId, String commandId) { columnPreviousCommandId.put(hNodeId, commandId); } diff --git a/karma-common/src/main/java/edu/isi/karma/rep/alignment/Label.java b/karma-common/src/main/java/edu/isi/karma/rep/alignment/Label.java index 8bfd32ec5..1fdc106f0 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/alignment/Label.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/alignment/Label.java @@ -88,6 +88,14 @@ public void setPrefix(String prefix) { this.prefix = prefix; } + public void setRdfsLabel(String label) { + this.rdfsLabel = label; + } + + public void setRdfsComment(String comment) { + this.rdfsComment = comment; + } + public String getUri() { return uri; } diff --git a/karma-common/src/main/java/edu/isi/karma/rep/sources/DataSource.java b/karma-common/src/main/java/edu/isi/karma/rep/sources/DataSource.java index aec937529..396be8514 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/sources/DataSource.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/sources/DataSource.java @@ -161,7 +161,8 @@ private void updateModel(DirectedWeightedMultigraph treeModel if (vertexIdToArgument.get(n.getId()) == null) continue; - Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix()); + Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix(), + n.getLabel().getRdfsLabel(), n.getLabel().getRdfsComment()); ClassAtom classAtom = new ClassAtom(classPredicate, vertexIdToArgument.get(n.getId())); m.getAtoms().add(classAtom); @@ -173,7 +174,8 @@ private void updateModel(DirectedWeightedMultigraph treeModel vertexIdToArgument.get(e.getTarget().getId()) == null) continue; - Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix()); + Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix(), + e.getLabel().getRdfsLabel(), e.getLabel().getRdfsComment()); IndividualPropertyAtom propertyAtom = null; // has_subclass is from source to target, we substitute this with a rdfs:subClassOf from target to source diff --git a/karma-common/src/main/java/edu/isi/karma/rep/sources/WebService.java b/karma-common/src/main/java/edu/isi/karma/rep/sources/WebService.java index d1f206595..cff3cb9d6 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/sources/WebService.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/sources/WebService.java @@ -423,7 +423,8 @@ private Model getInputModel(DirectedWeightedMultigraph treeMo if (vertexIdToArgument.get(n.getId()) == null) continue; - Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix()); + Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix(), + n.getLabel().getRdfsLabel(), n.getLabel().getRdfsComment()); ClassAtom classAtom = new ClassAtom(classPredicate, vertexIdToArgument.get(n.getId())); m.getAtoms().add(classAtom); @@ -437,7 +438,8 @@ private Model getInputModel(DirectedWeightedMultigraph treeMo vertexIdToArgument.get(e.getTarget().getId()) == null) continue; - Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix()); + Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix(), + e.getLabel().getRdfsLabel(), e.getLabel().getRdfsComment()); IndividualPropertyAtom propertyAtom; // // has_subclass is from source to target, we substitute this with a rdfs:subClassOf from target to source @@ -478,7 +480,8 @@ private Model getOutputModel(DirectedWeightedMultigraph treeM continue; - Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix()); + Label classPredicate = new Label(n.getLabel().getUri(), n.getLabel().getNs(), n.getLabel().getPrefix(), + n.getLabel().getRdfsLabel(), n.getLabel().getRdfsComment()); ClassAtom classAtom = new ClassAtom(classPredicate, vertexIdToArgument.get(n.getId())); m.getAtoms().add(classAtom); @@ -493,7 +496,8 @@ private Model getOutputModel(DirectedWeightedMultigraph treeM vertexIdToArgument.get(e.getTarget().getId()) == null) continue; - Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix()); + Label propertyPredicate = new Label(e.getLabel().getUri(), e.getLabel().getNs(), e.getLabel().getPrefix(), + e.getLabel().getRdfsLabel(), e.getLabel().getRdfsComment()); IndividualPropertyAtom propertyAtom; // has_subclass is from source to target, we substitute this with a rdfs:subClassOf from target to source diff --git a/karma-common/src/main/java/edu/isi/karma/util/JSONUtil.java b/karma-common/src/main/java/edu/isi/karma/util/JSONUtil.java index af104c14f..54bc90343 100644 --- a/karma-common/src/main/java/edu/isi/karma/util/JSONUtil.java +++ b/karma-common/src/main/java/edu/isi/karma/util/JSONUtil.java @@ -256,24 +256,28 @@ public static void writeJsonFile(Object o, String name) { } public static Object convertJSONLinesToJSONArray(InputStream is,String encoding) throws Exception{ - if(encoding == null){ + if (encoding == null) { encoding = "UTF-8"; } JSONArray jArray = new JSONArray(); - try{ - - BufferedReader br = new BufferedReader(new InputStreamReader(is,encoding)); - String line=null; - - while((line = br.readLine()) != null){ - jArray.put(new JSONObject(line.trim())); - } - - br.close(); - return jArray; - }catch(Exception e){ - throw new Exception("Error while reading json lines."+ e.getMessage()); + + String line = null; + try { + BufferedReader br = new BufferedReader(new InputStreamReader(is, + encoding)); + + while ((line = br.readLine()) != null) { + jArray.put(new JSONObject(line.trim())); + } + + br.close(); + return jArray; + } catch (Exception e) { + String message = "Error while reading json lines:" + e.getMessage(); + if (line != null) + message = message + "\n" + line; + throw new Exception(message); } - + } } diff --git a/karma-spark/pom.xml b/karma-spark/pom.xml index 65bd5e5d6..720dfc067 100644 --- a/karma-spark/pom.xml +++ b/karma-spark/pom.xml @@ -154,12 +154,12 @@ - cloudera + 1.5.0-cdh5.5.0 true env - cloudera + 1.5.0-cdh5.5.0 @@ -176,11 +176,11 @@ - vanilla + 1.5.0 env - vanilla + 1.5.0 @@ -197,11 +197,11 @@ - withhive + 1.5.0-hive env - hive + 1.5.0-hive @@ -218,11 +218,11 @@ - withcdhhive + 1.6.0-cdh5.10.1-hive env - cdhhive + 1.6.0-cdh5.10.1-hive @@ -251,10 +251,13 @@ mvn package -P shaded To generate jar for vanilla spark 1.5, run - mvn package -P shaded -Denv=vanilla + mvn package -P shaded -Denv=1.5.0 To generate jar for spark 1.5 with hive, run - mvn package -P shaded -Denv=hive + mvn package -P shaded -Denv=1.5.0-hive + + To generate jar for cloudera spark 1.6.0 with hive, run + mvn package -P shaded -Denv=1.6.0-cdh5.10.1-hive --> org.apache.maven.plugins @@ -263,7 +266,7 @@ true true - shaded + ${env} diff --git a/karma-util/src/main/java/edu/isi/karma/util/KarmaStats.java b/karma-util/src/main/java/edu/isi/karma/util/KarmaStats.java index d38f6cc74..ac9be2473 100644 --- a/karma-util/src/main/java/edu/isi/karma/util/KarmaStats.java +++ b/karma-util/src/main/java/edu/isi/karma/util/KarmaStats.java @@ -43,7 +43,7 @@ public static void main(String[] args) { KarmaStats stats = new KarmaStats(cl); if(!stats.parseCommandLineOptions(cl)) { - System.out.println("Parse ERROR. Please use \"java -cp JAR_NAME --inputfile INPUT_FILE --outputfile OUTPUT_FILE --pretty(optional)\" "); + System.out.println("Parse ERROR. Please use \"java -cp JAR_NAME edu.isi.karma.util.KarmaStats --inputfile INPUT_FILE --outputfile OUTPUT_FILE --pretty(optional)\" "); return; } karmaStats(stats.inputFile, stats.outputFile,stats.isPretty); @@ -67,14 +67,12 @@ public static void karmaStats(String inputFile, String outputFile,boolean isPret /* * If model name has "." in the name. * */ - if(fname.length>2) - { + for(int j=0;j - + ${project.basedir}/external_webapps/openrdf-workbench.war /openrdf-workbench - + ${project.basedir}/external_webapps/openrdf-sesame.war /openrdf-sesame - + ${project.basedir}/external_webapps/cleaningService.war /cleaningService - + ${project.basedir}/external_webapps/cluster-1.0.war /clusterService diff --git a/karma-web/src/main/webapp/css/table.css b/karma-web/src/main/webapp/css/table.css deleted file mode 100644 index 4d60e1ade..000000000 --- a/karma-web/src/main/webapp/css/table.css +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - * Copyright 2012 University of Southern California - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This code was developed by the Information Integration Group as part - * of the Karma project at the Information Sciences Institute of the - * University of Southern California. For more information, publications, - * and related projects, please see: http://www.isi.edu/integration - ******************************************************************************/ - -.solid1px { - border: solid 1px; -} -td{ - text-align:center; -} -.noLineBelow { - border: solid 1px; - border-bottom: none; -} -.noLineAboveAndBelow { - border: solid 1px; - border-top: none; - border-bottom: none; -} -#nested { - border-width: 0px; - border-spacing: 0px; - border-collapse: collapse; - border-bottom: solid 1px; -} -.table01cell { - background-color: #E0EEEE ; -} -.table02cell { - background-color: #FFE4C4; -} -.table03cell { - background-color: #AEEEEE; -} -.topLevelTableCell { - background-color: #FFFFFF; -} -#btnPrevCache { - margin-top: 25px; -} -#btnNextCache { - margin-top: 25px; -} \ No newline at end of file diff --git a/karma-web/src/main/webapp/css/tables_workspace.css b/karma-web/src/main/webapp/css/tables_workspace.css index a9032396d..0ae2ae7e6 100644 --- a/karma-web/src/main/webapp/css/tables_workspace.css +++ b/karma-web/src/main/webapp/css/tables_workspace.css @@ -551,3 +551,7 @@ span#applyHistoryButton { width: 300px; max-height: 250px; } + +.htable-selected { + background-color: #895001; +} \ No newline at end of file diff --git a/karma-web/src/main/webapp/index.jsp b/karma-web/src/main/webapp/index.jsp index 3e8bcdfdb..a5db8f741 100755 --- a/karma-web/src/main/webapp/index.jsp +++ b/karma-web/src/main/webapp/index.jsp @@ -138,7 +138,7 @@ and related projects, please see: http://www.isi.edu/integration .wk-row-selected { background-color: #EEEEEE; } - + .table-no-border td { border-top: 0 none; } diff --git a/karma-web/src/main/webapp/js/ServerResponseObjectParsing.js b/karma-web/src/main/webapp/js/ServerResponseObjectParsing.js index e45b0f09e..fef76f65e 100755 --- a/karma-web/src/main/webapp/js/ServerResponseObjectParsing.js +++ b/karma-web/src/main/webapp/js/ServerResponseObjectParsing.js @@ -45,7 +45,6 @@ function parse(data) { } } }); - if (isError) return false; @@ -214,7 +213,6 @@ function parse(data) { var sep = $("").html(" | "); var label1 = $("