Merge pull request #18 from hkuich/0.1.0

0.1.0

Author: hkuich

README.md

@@ -5,7 +5,7 @@ plugins {
 }
 
 group 'com.github.bayer-science-for-a-better-life'
-version '0.0.3'
+version '0.1.0-alpha-9'
 
 repositories {
     mavenCentral()
@@ -15,7 +15,7 @@ repositories {
 }
 
 dependencies {
-    compile group: 'io.grakn.client', name: 'grakn-client', version: '1.8.3'
+    compile group: 'io.grakn.client', name: 'grakn-client', version: '2.0.0-alpha-9'
     testCompile group: 'junit', name: 'junit', version: '4.12'
     compile 'com.google.code.gson:gson:2.8.6'
     compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
@@ -25,7 +25,7 @@ dependencies {
     compile 'info.picocli:picocli:4.5.1'
 }
 
-mainClassName = 'cli.Cli'
+mainClassName = 'cli.GramiCLI'
 
 publishing {
     publications {

build.gradle

@@ -8,7 +8,7 @@
 
 import java.io.IOException;
 
-@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.0.3", mixinStandardHelpOptions = true)
+@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0-alpha-9", mixinStandardHelpOptions = true)
 public class GramiCLI {
 
     public static void main(String[] args) {
@@ -25,22 +25,22 @@ public static void main(String[] args) {
 class MigrateCommand implements Runnable {
     @CommandLine.Spec CommandLine.Model.CommandSpec spec;
 
-    @CommandLine.Option(names = {"-d", "--dataConfigFile"}, description = "data config file in JSON format", required = true)
+    @CommandLine.Option(names = {"-dc", "--dataConfigFile"}, description = "data config file in JSON format", required = true)
     private String dataConfigFilePath;
 
-    @CommandLine.Option(names = {"-p", "--processorConfigFile"}, description = "processor config file in JSON format", required = true)
+    @CommandLine.Option(names = {"-pc", "--processorConfigFile"}, description = "processor config file in JSON format", required = true)
     private String processorConfigFilePath;
 
-    @CommandLine.Option(names = {"-m", "--migrationStatusFile"}, description = "file to track migration status in", required = true)
+    @CommandLine.Option(names = {"-ms", "--migrationStatusFile"}, description = "file to track migration status in", required = true)
     private String migrationStatusFilePath;
 
     @CommandLine.Option(names = {"-s", "--schemaFile"}, description = "your schema file as .gql", required = true)
     private String schemaFilePath;
 
-    @CommandLine.Option(names = {"-k", "--keyspace"}, description = "target keyspace in your grakn instance", required = true)
-    private String keyspaceName;
+    @CommandLine.Option(names = {"-db", "--database"}, description = "target database in your grakn instance", required = true)
+    private String databaseName;
 
-    @CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:48555)", defaultValue = "localhost:48555")
+    @CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729")
     private String graknURI;
 
     @CommandLine.Option(names = {"-cm", "--cleanMigration"}, description = "optional - delete old schema and data and restart migration from scratch - default: continue previous migration, if exists")
@@ -57,12 +57,12 @@ public void run() {
         spec.commandLine().getOut().println("\tprocessor configuration: " + processorConfigFilePath);
         spec.commandLine().getOut().println("\ttracking migration status in: " + migrationStatusFilePath);
         spec.commandLine().getOut().println("\tschema: " + schemaFilePath);
-        spec.commandLine().getOut().println("\tkeyspace: " + keyspaceName);
+        spec.commandLine().getOut().println("\tdatabase: " + databaseName);
         spec.commandLine().getOut().println("\tgrakn server: " + graknURI);
-        spec.commandLine().getOut().println("\tdelete keyspace and all data in it for a clean new migration?: " + cleanMigration);
+        spec.commandLine().getOut().println("\tdelete database and all data in it for a clean new migration?: " + cleanMigration);
         spec.commandLine().getOut().println("\tmigration scope: " + scope);
 
-        final MigrationConfig migrationConfig = new MigrationConfig(graknURI, keyspaceName, schemaFilePath, dataConfigFilePath, processorConfigFilePath);
+        final MigrationConfig migrationConfig = new MigrationConfig(graknURI, databaseName, schemaFilePath, dataConfigFilePath, processorConfigFilePath);
 
         try {
             GraknMigrator mig = new GraknMigrator(migrationConfig, migrationStatusFilePath, cleanMigration);
@@ -91,21 +91,21 @@ class SchemaUpdateCommand implements Runnable {
     @CommandLine.Option(names = {"-s", "--schemaFile"}, description = "your schema file as .gql", required = true)
     private String schemaFilePath;
 
-    @CommandLine.Option(names = {"-k", "--keyspace"}, description = "target keyspace in your grakn instance", required = true)
-    private String keyspaceName;
+    @CommandLine.Option(names = {"-db", "--database"}, description = "target database in your grakn instance", required = true)
+    private String databaseName;
 
-    @CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:48555)", defaultValue = "localhost:48555")
+    @CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729")
     private String graknURI;
 
     @Override
     public void run() {
         spec.commandLine().getOut().println("############## GraMi schema-update ###############");
         spec.commandLine().getOut().println("schema-update started with parameters:");
         spec.commandLine().getOut().println("\tschema: " + schemaFilePath);
-        spec.commandLine().getOut().println("\tkeyspace: " + keyspaceName);
+        spec.commandLine().getOut().println("\tkeyspace: " + databaseName);
         spec.commandLine().getOut().println("\tgrakn server: " + graknURI);
 
-        SchemaUpdateConfig suConfig = new SchemaUpdateConfig(graknURI, keyspaceName, schemaFilePath);
+        SchemaUpdateConfig suConfig = new SchemaUpdateConfig(graknURI, databaseName, schemaFilePath);
         SchemaUpdater su = new SchemaUpdater(suConfig);
         su.updateSchema();
     }

src/main/java/cli/GramiCLI.java

@@ -3,13 +3,16 @@
 import configuration.DataConfigEntry;
 import configuration.ProcessorConfigEntry;
 import graql.lang.Graql;
-import graql.lang.statement.Statement;
-import graql.lang.statement.StatementInstance;
+import graql.lang.pattern.Pattern;
+import graql.lang.pattern.variable.ThingVariable;
+import graql.lang.pattern.variable.ThingVariable.Thing;
+import graql.lang.pattern.variable.UnboundVariable;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Map;
 
 import static generator.GeneratorUtil.*;
@@ -21,38 +24,42 @@ public class AppendAttributeGenerator extends InsertGenerator {
     private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami");
     private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data");
 
-    public AppendAttributeGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) {
+    public AppendAttributeGenerator(DataConfigEntry dataConfigEntry,
+                                    ProcessorConfigEntry processorConfigEntry) {
         super();
         this.dce = dataConfigEntry;
         this.pce = processorConfigEntry;
         appLogger.debug("Creating AppendAttribute for processor " + processorConfigEntry.getProcessor() + " of type " + processorConfigEntry.getProcessorType());
     }
 
-    public ArrayList<ArrayList<ArrayList<Statement>>> graknAppendAttributeInsert(ArrayList<String> rows, String header) throws Exception {
-        ArrayList<ArrayList<ArrayList<Statement>>> matchInsertStatements = new ArrayList<>();
+    public HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> graknAppendAttributeInsert(ArrayList<String> rows,
+                                                                                              String header) throws Exception {
+        HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> matchInsertPatterns = new HashMap<>();
 
-        ArrayList<ArrayList<Statement>> matchStatements = new ArrayList<>();
-        ArrayList<ArrayList<Statement>> insertStatements = new ArrayList<>();
+        ArrayList<ArrayList<ThingVariable<?>>> matchPatterns = new ArrayList<>();
+        ArrayList<ArrayList<ThingVariable<?>>> insertPatterns = new ArrayList<>();
 
         int insertCounter = 0;
 
         for (String row : rows) {
-            ArrayList<ArrayList<Statement>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
+            ArrayList<ArrayList<ThingVariable<?>>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
             if (tmp != null) {
                 if (tmp.get(0) != null && tmp.get(1) != null) {
-                    matchStatements.add(tmp.get(0));
-                    insertStatements.add(tmp.get(1));
+                    matchPatterns.add(tmp.get(0));
+                    insertPatterns.add(tmp.get(1));
                     insertCounter++;
                 }
             }
 
         }
-        matchInsertStatements.add(matchStatements);
-        matchInsertStatements.add(insertStatements);
-        return matchInsertStatements;
+        matchInsertPatterns.put("match", matchPatterns);
+        matchInsertPatterns.put("insert", insertPatterns);
+        return matchInsertPatterns;
     }
 
-    public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String row, String header, int insertCounter) throws Exception {
+    public ArrayList<ArrayList<ThingVariable<?>>> graknAppendAttributeQueryFromRow(String row,
+                                                                                   String header,
+                                                                                   int insertCounter) throws Exception {
         String fileSeparator = dce.getSeparator();
         String[] rowTokens = row.split(fileSeparator);
         String[] columnNames = header.split(fileSeparator);
@@ -63,35 +70,38 @@ public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String r
             throw new IllegalArgumentException("data config entry for " + dce.getDataPath() + " is incomplete - it needs at least one attribute used for matching (\"match\": true) and at least one attribute to be appended (\"match\": false or not set at all");
         }
 
-        ArrayList<Statement> matchStatements = new ArrayList<>();
-        ArrayList<Statement> insertStatements = new ArrayList<>();
+        ArrayList<ThingVariable<?>> matchPatterns = new ArrayList<>();
+        ArrayList<ThingVariable<?>> insertPatterns = new ArrayList<>();
 
         // get all attributes that are isMatch() --> construct match clause
-        StatementInstance appendAttributeMatchStatement = addEntityToMatchStatement(insertCounter);
+        Thing appendAttributeMatchPattern = addEntityToMatchPattern(insertCounter);
         for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForMatchAttribute : dce.getAttributes()) {
-            if (generatorMappingForMatchAttribute.isMatch()){
-                appendAttributeMatchStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
+            if (generatorMappingForMatchAttribute.isMatch()) {
+                appendAttributeMatchPattern = addAttribute(rowTokens, appendAttributeMatchPattern, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
             }
         }
-        matchStatements.add(appendAttributeMatchStatement);
+        matchPatterns.add(appendAttributeMatchPattern);
 
         // get all attributes that are !isMatch() --> construct insert clause
-        Statement appendAttributeInsertStatement = addEntityToInsertStatement(insertCounter);
+        UnboundVariable thingVar = addEntityToInsertPattern(insertCounter);
+        Thing appendAttributeInsertPattern = null;
         for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAppendAttribute : dce.getAttributes()) {
-            if (!generatorMappingForAppendAttribute.isMatch()){
-                appendAttributeInsertStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
+            if (!generatorMappingForAppendAttribute.isMatch()) {
+                appendAttributeInsertPattern = addAttribute(rowTokens, thingVar, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
             }
         }
-        insertStatements.add((StatementInstance) appendAttributeInsertStatement);
+        if (appendAttributeInsertPattern != null) {
+            insertPatterns.add(appendAttributeInsertPattern);
+        }
 
-        ArrayList<ArrayList<Statement>> assembledStatements = new ArrayList<>();
-        assembledStatements.add(matchStatements);
-        assembledStatements.add(insertStatements);
+        ArrayList<ArrayList<ThingVariable<?>>> assembledPatterns = new ArrayList<>();
+        assembledPatterns.add(matchPatterns);
+        assembledPatterns.add(insertPatterns);
 
 
-        if (isValid(assembledStatements)) {
-            appLogger.debug("valid query: <" + assembleQuery(assembledStatements).toString() + ">");
-            return assembledStatements;
+        if (isValid(assembledPatterns)) {
+            appLogger.debug("valid query: <" + assembleQuery(assembledPatterns).toString() + ">");
+            return assembledPatterns;
         } else {
             dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not contain at least one match attribute and one insert attribute. Faulty tokenized row: " + Arrays.toString(rowTokens));
             return null;
@@ -102,7 +112,7 @@ private boolean validateDataConfigEntry() {
         boolean containsMatchAttribute = false;
         boolean containsAppendAttribute = false;
         for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getAttributes()) {
-            if (attributeMapping.isMatch()){
+            if (attributeMapping.isMatch()) {
                 containsMatchAttribute = true;
             }
             if (!attributeMapping.isMatch()) {
@@ -112,51 +122,58 @@ private boolean validateDataConfigEntry() {
         return containsMatchAttribute && containsAppendAttribute;
     }
 
-    private StatementInstance addEntityToMatchStatement(int insertCounter) {
+    private Thing addEntityToMatchPattern(int insertCounter) {
         if (pce.getSchemaType() != null) {
-            return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
+//            return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
+            return Graql.var("e").isa(pce.getSchemaType());
         } else {
             throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
         }
     }
 
-    private Statement addEntityToInsertStatement(int insertCounter) {
+    private UnboundVariable addEntityToInsertPattern(int insertCounter) {
         if (pce.getSchemaType() != null) {
-            return Graql.var("e-" + insertCounter);
+//            return Graql.var("e-" + insertCounter);
+            return Graql.var("e");
         } else {
             throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
         }
     }
 
-    private String assembleQuery(ArrayList<ArrayList<Statement>> queries) {
+    private String assembleQuery(ArrayList<ArrayList<ThingVariable<?>>> queries) {
         StringBuilder ret = new StringBuilder();
-        for (Statement st : queries.get(0)) {
+        for (ThingVariable st : queries.get(0)) {
             ret.append(st.toString());
         }
         ret.append(queries.get(1).get(0).toString());
         return ret.toString();
     }
 
-    private boolean isValid(ArrayList<ArrayList<Statement>> si) {
-        ArrayList<Statement> matchStatements = si.get(0);
-        ArrayList<Statement> insertStatements = si.get(1);
-        StringBuilder matchStatement = new StringBuilder();
-        for (Statement st:matchStatements) {
-            matchStatement.append(st.toString());
+    private boolean isValid(ArrayList<ArrayList<ThingVariable<?>>> si) {
+        ArrayList<ThingVariable<?>> matchPatterns = si.get(0);
+        ArrayList<ThingVariable<?>> insertPatterns = si.get(1);
+
+        if (insertPatterns.size() < 1) {
+            return false;
+        }
+
+        StringBuilder matchPattern = new StringBuilder();
+        for (Pattern st : matchPatterns) {
+            matchPattern.append(st.toString());
         }
-        String insertStatement = insertStatements.get(0).toString();
+        String insertPattern = insertPatterns.get(0).toString();
 
         // missing match attribute
-        for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping: dce.getMatchAttributes()) {
+        for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getMatchAttributes()) {
             String generatorKey = attributeMapping.getGenerator();
             ProcessorConfigEntry.ConceptGenerator generatorEntry = pce.getAttributeGenerator(generatorKey);
-            if (!matchStatement.toString().contains("has " + generatorEntry.getAttributeType())) {
+            if (!matchPattern.toString().contains("has " + generatorEntry.getAttributeType())) {
                 return false;
             }
         }
         // missing required insert attribute
-        for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry: pce.getRequiredAttributes().entrySet()) {
-            if (!insertStatement.contains("has " + generatorEntry.getValue().getAttributeType())) {
+        for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry : pce.getRequiredAttributes().entrySet()) {
+            if (!insertPattern.contains("has " + generatorEntry.getValue().getAttributeType())) {
                 return false;
             }
         }