Skip to content

Commit 49b2052

Browse files
hkuichhkuich
committed
- moved to grakn-2.0
 - minor, non-breaking code-cleanup - no new features except for small optimizations
1 parent b3d6d27 commit 49b2052

23 files changed

+982
-720
lines changed

‎build.gradle‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ plugins {
55
}
66

77
group 'com.github.bayer-science-for-a-better-life'
8-
version '0.0.3'
8+
version '0.1.0-alpha-9'
99

1010
repositories {
1111
mavenCentral()
@@ -15,7 +15,7 @@ repositories {
1515
}
1616

1717
dependencies {
18-
compile group: 'io.grakn.client', name: 'grakn-client', version: '1.8.3'
18+
compile group: 'io.grakn.client', name: 'grakn-client', version: '2.0.0-alpha-9'
1919
testCompile group: 'junit', name: 'junit', version: '4.12'
2020
compile 'com.google.code.gson:gson:2.8.6'
2121
compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
@@ -25,7 +25,7 @@ dependencies {
2525
compile 'info.picocli:picocli:4.5.1'
2626
}
2727

28-
mainClassName = 'cli.Cli'
28+
mainClassName = 'cli.GramiCLI'
2929

3030
publishing {
3131
publications {

‎src/main/java/cli/GramiCLI.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
import java.io.IOException;
1010

11-
@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.0.3", mixinStandardHelpOptions = true)
11+
@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0", mixinStandardHelpOptions = true)
1212
public class GramiCLI {
1313

1414
public static void main(String[] args) {
@@ -40,7 +40,7 @@ class MigrateCommand implements Runnable {
4040
@CommandLine.Option(names = {"-k", "--keyspace"}, description = "target keyspace in your grakn instance", required = true)
4141
private String keyspaceName;
4242

43-
@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:48555)", defaultValue = "localhost:48555")
43+
@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729")
4444
private String graknURI;
4545

4646
@CommandLine.Option(names = {"-cm", "--cleanMigration"}, description = "optional - delete old schema and data and restart migration from scratch - default: continue previous migration, if exists")

‎src/main/java/generator/AppendAttributeGenerator.java‎

Lines changed: 65 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
import configuration.DataConfigEntry;
44
import configuration.ProcessorConfigEntry;
55
import graql.lang.Graql;
6-
import graql.lang.statement.Statement;
7-
import graql.lang.statement.StatementInstance;
6+
import graql.lang.pattern.Pattern;
7+
import graql.lang.pattern.variable.ThingVariable;
8+
import graql.lang.pattern.variable.ThingVariable.Thing;
9+
import graql.lang.pattern.variable.UnboundVariable;
810
import org.apache.logging.log4j.LogManager;
911
import org.apache.logging.log4j.Logger;
1012

1113
import java.util.ArrayList;
1214
import java.util.Arrays;
15+
import java.util.HashMap;
1316
import java.util.Map;
1417

1518
import static generator.GeneratorUtil.*;
@@ -21,38 +24,42 @@ public class AppendAttributeGenerator extends InsertGenerator {
2124
private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami");
2225
private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data");
2326

24-
public AppendAttributeGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) {
27+
public AppendAttributeGenerator(DataConfigEntry dataConfigEntry,
28+
ProcessorConfigEntry processorConfigEntry) {
2529
super();
2630
this.dce = dataConfigEntry;
2731
this.pce = processorConfigEntry;
2832
appLogger.debug("Creating AppendAttribute for processor " + processorConfigEntry.getProcessor() + " of type " + processorConfigEntry.getProcessorType());
2933
}
3034

31-
public ArrayList<ArrayList<ArrayList<Statement>>> graknAppendAttributeInsert(ArrayList<String> rows, String header) throws Exception {
32-
ArrayList<ArrayList<ArrayList<Statement>>> matchInsertStatements = new ArrayList<>();
35+
public HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> graknAppendAttributeInsert(ArrayList<String> rows,
36+
String header) throws Exception {
37+
HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> matchInsertPatterns = new HashMap<>();
3338

34-
ArrayList<ArrayList<Statement>> matchStatements = new ArrayList<>();
35-
ArrayList<ArrayList<Statement>> insertStatements = new ArrayList<>();
39+
ArrayList<ArrayList<ThingVariable<?>>> matchPatterns = new ArrayList<>();
40+
ArrayList<ArrayList<ThingVariable<?>>> insertPatterns = new ArrayList<>();
3641

3742
int insertCounter = 0;
3843

3944
for (String row : rows) {
40-
ArrayList<ArrayList<Statement>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
45+
ArrayList<ArrayList<ThingVariable<?>>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
4146
if (tmp != null) {
4247
if (tmp.get(0) != null && tmp.get(1) != null) {
43-
matchStatements.add(tmp.get(0));
44-
insertStatements.add(tmp.get(1));
48+
matchPatterns.add(tmp.get(0));
49+
insertPatterns.add(tmp.get(1));
4550
insertCounter++;
4651
}
4752
}
4853

4954
}
50-
matchInsertStatements.add(matchStatements);
51-
matchInsertStatements.add(insertStatements);
52-
return matchInsertStatements;
55+
matchInsertPatterns.put("match", matchPatterns);
56+
matchInsertPatterns.put("insert", insertPatterns);
57+
return matchInsertPatterns;
5358
}
5459

55-
public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String row, String header, int insertCounter) throws Exception {
60+
public ArrayList<ArrayList<ThingVariable<?>>> graknAppendAttributeQueryFromRow(String row,
61+
String header,
62+
int insertCounter) throws Exception {
5663
String fileSeparator = dce.getSeparator();
5764
String[] rowTokens = row.split(fileSeparator);
5865
String[] columnNames = header.split(fileSeparator);
@@ -63,35 +70,38 @@ public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String r
6370
throw new IllegalArgumentException("data config entry for " + dce.getDataPath() + " is incomplete - it needs at least one attribute used for matching (\"match\": true) and at least one attribute to be appended (\"match\": false or not set at all");
6471
}
6572

66-
ArrayList<Statement> matchStatements = new ArrayList<>();
67-
ArrayList<Statement> insertStatements = new ArrayList<>();
73+
ArrayList<ThingVariable<?>> matchPatterns = new ArrayList<>();
74+
ArrayList<ThingVariable<?>> insertPatterns = new ArrayList<>();
6875

6976
// get all attributes that are isMatch() --> construct match clause
70-
StatementInstance appendAttributeMatchStatement = addEntityToMatchStatement(insertCounter);
77+
Thing appendAttributeMatchPattern = addEntityToMatchPattern(insertCounter);
7178
for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForMatchAttribute : dce.getAttributes()) {
72-
if (generatorMappingForMatchAttribute.isMatch()){
73-
appendAttributeMatchStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
79+
if (generatorMappingForMatchAttribute.isMatch()) {
80+
appendAttributeMatchPattern = addAttribute(rowTokens, appendAttributeMatchPattern, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
7481
}
7582
}
76-
matchStatements.add(appendAttributeMatchStatement);
83+
matchPatterns.add(appendAttributeMatchPattern);
7784

7885
// get all attributes that are !isMatch() --> construct insert clause
79-
Statement appendAttributeInsertStatement = addEntityToInsertStatement(insertCounter);
86+
UnboundVariable thingVar = addEntityToInsertPattern(insertCounter);
87+
Thing appendAttributeInsertPattern = null;
8088
for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAppendAttribute : dce.getAttributes()) {
81-
if (!generatorMappingForAppendAttribute.isMatch()){
82-
appendAttributeInsertStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
89+
if (!generatorMappingForAppendAttribute.isMatch()) {
90+
appendAttributeInsertPattern = addAttribute(rowTokens, thingVar, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
8391
}
8492
}
85-
insertStatements.add((StatementInstance) appendAttributeInsertStatement);
93+
if (appendAttributeInsertPattern != null) {
94+
insertPatterns.add(appendAttributeInsertPattern);
95+
}
8696

87-
ArrayList<ArrayList<Statement>> assembledStatements = new ArrayList<>();
88-
assembledStatements.add(matchStatements);
89-
assembledStatements.add(insertStatements);
97+
ArrayList<ArrayList<ThingVariable<?>>> assembledPatterns = new ArrayList<>();
98+
assembledPatterns.add(matchPatterns);
99+
assembledPatterns.add(insertPatterns);
90100

91101

92-
if (isValid(assembledStatements)) {
93-
appLogger.debug("valid query: <" + assembleQuery(assembledStatements).toString() + ">");
94-
return assembledStatements;
102+
if (isValid(assembledPatterns)) {
103+
appLogger.debug("valid query: <" + assembleQuery(assembledPatterns).toString() + ">");
104+
return assembledPatterns;
95105
} else {
96106
dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not contain at least one match attribute and one insert attribute. Faulty tokenized row: " + Arrays.toString(rowTokens));
97107
return null;
@@ -102,7 +112,7 @@ private boolean validateDataConfigEntry() {
102112
boolean containsMatchAttribute = false;
103113
boolean containsAppendAttribute = false;
104114
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getAttributes()) {
105-
if (attributeMapping.isMatch()){
115+
if (attributeMapping.isMatch()) {
106116
containsMatchAttribute = true;
107117
}
108118
if (!attributeMapping.isMatch()) {
@@ -112,51 +122,58 @@ private boolean validateDataConfigEntry() {
112122
return containsMatchAttribute && containsAppendAttribute;
113123
}
114124

115-
private StatementInstance addEntityToMatchStatement(int insertCounter) {
125+
private Thing addEntityToMatchPattern(int insertCounter) {
116126
if (pce.getSchemaType() != null) {
117-
return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
127+
// return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
128+
return Graql.var("e").isa(pce.getSchemaType());
118129
} else {
119130
throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
120131
}
121132
}
122133

123-
private Statement addEntityToInsertStatement(int insertCounter) {
134+
private UnboundVariable addEntityToInsertPattern(int insertCounter) {
124135
if (pce.getSchemaType() != null) {
125-
return Graql.var("e-" + insertCounter);
136+
// return Graql.var("e-" + insertCounter);
137+
return Graql.var("e");
126138
} else {
127139
throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
128140
}
129141
}
130142

131-
private String assembleQuery(ArrayList<ArrayList<Statement>> queries) {
143+
private String assembleQuery(ArrayList<ArrayList<ThingVariable<?>>> queries) {
132144
StringBuilder ret = new StringBuilder();
133-
for (Statement st : queries.get(0)) {
145+
for (ThingVariable st : queries.get(0)) {
134146
ret.append(st.toString());
135147
}
136148
ret.append(queries.get(1).get(0).toString());
137149
return ret.toString();
138150
}
139151

140-
private boolean isValid(ArrayList<ArrayList<Statement>> si) {
141-
ArrayList<Statement> matchStatements = si.get(0);
142-
ArrayList<Statement> insertStatements = si.get(1);
143-
StringBuilder matchStatement = new StringBuilder();
144-
for (Statement st:matchStatements) {
145-
matchStatement.append(st.toString());
152+
private boolean isValid(ArrayList<ArrayList<ThingVariable<?>>> si) {
153+
ArrayList<ThingVariable<?>> matchPatterns = si.get(0);
154+
ArrayList<ThingVariable<?>> insertPatterns = si.get(1);
155+
156+
if (insertPatterns.size() < 1) {
157+
return false;
158+
}
159+
160+
StringBuilder matchPattern = new StringBuilder();
161+
for (Pattern st : matchPatterns) {
162+
matchPattern.append(st.toString());
146163
}
147-
String insertStatement = insertStatements.get(0).toString();
164+
String insertPattern = insertPatterns.get(0).toString();
148165

149166
// missing match attribute
150-
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping: dce.getMatchAttributes()) {
167+
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getMatchAttributes()) {
151168
String generatorKey = attributeMapping.getGenerator();
152169
ProcessorConfigEntry.ConceptGenerator generatorEntry = pce.getAttributeGenerator(generatorKey);
153-
if (!matchStatement.toString().contains("has " + generatorEntry.getAttributeType())) {
170+
if (!matchPattern.toString().contains("has " + generatorEntry.getAttributeType())) {
154171
return false;
155172
}
156173
}
157174
// missing required insert attribute
158-
for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry: pce.getRequiredAttributes().entrySet()) {
159-
if (!insertStatement.contains("has " + generatorEntry.getValue().getAttributeType())) {
175+
for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry : pce.getRequiredAttributes().entrySet()) {
176+
if (!insertPattern.contains("has " + generatorEntry.getValue().getAttributeType())) {
160177
return false;
161178
}
162179
}

‎src/main/java/generator/EntityInsertGenerator.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
import configuration.DataConfigEntry;
77
import configuration.ProcessorConfigEntry;
88
import graql.lang.Graql;
9-
import graql.lang.statement.Statement;
10-
import graql.lang.statement.StatementInstance;
9+
import graql.lang.pattern.Pattern;
10+
import graql.lang.pattern.variable.ThingVariable;
11+
import graql.lang.pattern.variable.ThingVariable.Thing;
1112
import org.apache.logging.log4j.LogManager;
1213
import org.apache.logging.log4j.Logger;
1314

@@ -29,60 +30,63 @@ public EntityInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEnt
2930
appLogger.debug("Creating EntityInsertGenerator for processor " + processorConfigEntry.getProcessor() + " of type " + processorConfigEntry.getProcessorType());
3031
}
3132

32-
public ArrayList<Statement> graknEntityInsert(ArrayList<String> rows, String header) throws IllegalArgumentException {
33-
ArrayList<Statement> statements = new ArrayList<>();
33+
public ArrayList<ThingVariable<?>> graknEntityInsert(ArrayList<String> rows,
34+
String header) throws IllegalArgumentException {
35+
ArrayList<ThingVariable<?>> patterns = new ArrayList<>();
3436
int insertCounter = 0;
3537
for (String row : rows) {
3638
try {
37-
Statement temp = graknEntityQueryFromRow(row, header, insertCounter);
39+
ThingVariable temp = graknEntityQueryFromRow(row, header, insertCounter);
3840
if (temp != null) {
39-
statements.add(temp);
41+
patterns.add(temp);
4042
}
4143
insertCounter++;
4244
} catch (Exception e) {
4345
e.printStackTrace();
4446
}
4547
}
46-
return statements;
48+
return patterns;
4749
}
4850

49-
public StatementInstance graknEntityQueryFromRow(String row, String header, int insertCounter) throws Exception {
51+
public ThingVariable graknEntityQueryFromRow(String row,
52+
String header,
53+
int insertCounter) throws Exception {
5054
String fileSeparator = dce.getSeparator();
5155
String[] rowTokens = row.split(fileSeparator);
5256
String[] columnNames = header.split(fileSeparator);
5357
appLogger.debug("processing tokenized row: " + Arrays.toString(rowTokens));
5458
malformedRow(row, rowTokens, columnNames.length);
5559

56-
StatementInstance entityInsertStatement = addEntityToStatement(insertCounter);
60+
Thing entityInsertStatement = addEntityToStatement(insertCounter);
5761

5862
for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) {
5963
entityInsertStatement = addAttribute(rowTokens, entityInsertStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor());
6064
}
6165

6266
if (isValid(entityInsertStatement)) {
63-
appLogger.debug("valid query: <" + entityInsertStatement.toString() + ">");
67+
appLogger.debug("valid query: <insert " + entityInsertStatement.toString() + ";>");
6468
return entityInsertStatement;
6569
} else {
6670
dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper <isa> statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens));
6771
return null;
6872
}
6973
}
7074

71-
private StatementInstance addEntityToStatement(int insertCounter) {
75+
private Thing addEntityToStatement(int insertCounter) {
7276
if (pce.getSchemaType() != null) {
7377
return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
7478
} else {
7579
throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
7680
}
7781
}
7882

79-
private boolean isValid(StatementInstance si) {
80-
String statement = si.toString();
81-
if (!statement.contains("isa " + pce.getSchemaType())) {
83+
private boolean isValid(Pattern pa) {
84+
String patternAsString = pa.toString();
85+
if (!patternAsString.contains("isa " + pce.getSchemaType())) {
8286
return false;
8387
}
8488
for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> con : pce.getRequiredAttributes().entrySet()) {
85-
if (!statement.contains("has " + con.getValue().getAttributeType())) {
89+
if (!patternAsString.contains("has " + con.getValue().getAttributeType())) {
8690
return false;
8791
}
8892
}

0 commit comments

Comments
 (0)