Skip to content

Commit

Permalink
Merge pull request #18 from hkuich/0.1.0
Browse files Browse the repository at this point in the history 
0.1.0
  • Loading branch information
@hkuich
hkuich authored Feb 19, 2021
2 parents b3d6d27 + e5bb062 commit bdf52db
Show file tree
Hide file tree
Showing 24 changed files with 1,043 additions and 1,447 deletions.
752 changes: 43 additions & 709 deletions README.md
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ plugins {
}

group 'com.github.bayer-science-for-a-better-life'
version '0.0.3'
version '0.1.0-alpha-9'

repositories {
mavenCentral()
Expand All @@ -15,7 +15,7 @@ repositories {
}

dependencies {
compile group: 'io.grakn.client', name: 'grakn-client', version: '1.8.3'
compile group: 'io.grakn.client', name: 'grakn-client', version: '2.0.0-alpha-9'
testCompile group: 'junit', name: 'junit', version: '4.12'
compile 'com.google.code.gson:gson:2.8.6'
compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
Expand All @@ -25,7 +25,7 @@ dependencies {
compile 'info.picocli:picocli:4.5.1'
}

mainClassName = 'cli.Cli'
mainClassName = 'cli.GramiCLI'

publishing {
publications {
Expand Down
30 changes: 15 additions & 15 deletions src/main/java/cli/GramiCLI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import java.io.IOException;

@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.0.3", mixinStandardHelpOptions = true)
@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0-alpha-9", mixinStandardHelpOptions = true)
public class GramiCLI {

public static void main(String[] args) {
Expand All @@ -25,22 +25,22 @@ public static void main(String[] args) {
class MigrateCommand implements Runnable {
@CommandLine.Spec CommandLine.Model.CommandSpec spec;

@CommandLine.Option(names = {"-d", "--dataConfigFile"}, description = "data config file in JSON format", required = true)
@CommandLine.Option(names = {"-dc", "--dataConfigFile"}, description = "data config file in JSON format", required = true)
private String dataConfigFilePath;

@CommandLine.Option(names = {"-p", "--processorConfigFile"}, description = "processor config file in JSON format", required = true)
@CommandLine.Option(names = {"-pc", "--processorConfigFile"}, description = "processor config file in JSON format", required = true)
private String processorConfigFilePath;

@CommandLine.Option(names = {"-m", "--migrationStatusFile"}, description = "file to track migration status in", required = true)
@CommandLine.Option(names = {"-ms", "--migrationStatusFile"}, description = "file to track migration status in", required = true)
private String migrationStatusFilePath;

@CommandLine.Option(names = {"-s", "--schemaFile"}, description = "your schema file as .gql", required = true)
private String schemaFilePath;

@CommandLine.Option(names = {"-k", "--keyspace"}, description = "target keyspace in your grakn instance", required = true)
private String keyspaceName;
@CommandLine.Option(names = {"-db", "--database"}, description = "target database in your grakn instance", required = true)
private String databaseName;

@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:48555)", defaultValue = "localhost:48555")
@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729")
private String graknURI;

@CommandLine.Option(names = {"-cm", "--cleanMigration"}, description = "optional - delete old schema and data and restart migration from scratch - default: continue previous migration, if exists")
Expand All @@ -57,12 +57,12 @@ public void run() {
spec.commandLine().getOut().println("\tprocessor configuration: " + processorConfigFilePath);
spec.commandLine().getOut().println("\ttracking migration status in: " + migrationStatusFilePath);
spec.commandLine().getOut().println("\tschema: " + schemaFilePath);
spec.commandLine().getOut().println("\tkeyspace: " + keyspaceName);
spec.commandLine().getOut().println("\tdatabase: " + databaseName);
spec.commandLine().getOut().println("\tgrakn server: " + graknURI);
spec.commandLine().getOut().println("\tdelete keyspace and all data in it for a clean new migration?: " + cleanMigration);
spec.commandLine().getOut().println("\tdelete database and all data in it for a clean new migration?: " + cleanMigration);
spec.commandLine().getOut().println("\tmigration scope: " + scope);

final MigrationConfig migrationConfig = new MigrationConfig(graknURI, keyspaceName, schemaFilePath, dataConfigFilePath, processorConfigFilePath);
final MigrationConfig migrationConfig = new MigrationConfig(graknURI, databaseName, schemaFilePath, dataConfigFilePath, processorConfigFilePath);

try {
GraknMigrator mig = new GraknMigrator(migrationConfig, migrationStatusFilePath, cleanMigration);
Expand Down Expand Up @@ -91,21 +91,21 @@ class SchemaUpdateCommand implements Runnable {
@CommandLine.Option(names = {"-s", "--schemaFile"}, description = "your schema file as .gql", required = true)
private String schemaFilePath;

@CommandLine.Option(names = {"-k", "--keyspace"}, description = "target keyspace in your grakn instance", required = true)
private String keyspaceName;
@CommandLine.Option(names = {"-db", "--database"}, description = "target database in your grakn instance", required = true)
private String databaseName;

@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:48555)", defaultValue = "localhost:48555")
@CommandLine.Option(names = {"-g", "--grakn"}, description = "optional - grakn DB in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729")
private String graknURI;

@Override
public void run() {
spec.commandLine().getOut().println("############## GraMi schema-update ###############");
spec.commandLine().getOut().println("schema-update started with parameters:");
spec.commandLine().getOut().println("\tschema: " + schemaFilePath);
spec.commandLine().getOut().println("\tkeyspace: " + keyspaceName);
spec.commandLine().getOut().println("\tkeyspace: " + databaseName);
spec.commandLine().getOut().println("\tgrakn server: " + graknURI);

SchemaUpdateConfig suConfig = new SchemaUpdateConfig(graknURI, keyspaceName, schemaFilePath);
SchemaUpdateConfig suConfig = new SchemaUpdateConfig(graknURI, databaseName, schemaFilePath);
SchemaUpdater su = new SchemaUpdater(suConfig);
su.updateSchema();
}
Expand Down
113 changes: 65 additions & 48 deletions src/main/java/generator/AppendAttributeGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
import configuration.DataConfigEntry;
import configuration.ProcessorConfigEntry;
import graql.lang.Graql;
import graql.lang.statement.Statement;
import graql.lang.statement.StatementInstance;
import graql.lang.pattern.Pattern;
import graql.lang.pattern.variable.ThingVariable;
import graql.lang.pattern.variable.ThingVariable.Thing;
import graql.lang.pattern.variable.UnboundVariable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import static generator.GeneratorUtil.*;
Expand All @@ -21,38 +24,42 @@ public class AppendAttributeGenerator extends InsertGenerator {
private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami");
private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data");

public AppendAttributeGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) {
public AppendAttributeGenerator(DataConfigEntry dataConfigEntry,
ProcessorConfigEntry processorConfigEntry) {
super();
this.dce = dataConfigEntry;
this.pce = processorConfigEntry;
appLogger.debug("Creating AppendAttribute for processor " + processorConfigEntry.getProcessor() + " of type " + processorConfigEntry.getProcessorType());
}

public ArrayList<ArrayList<ArrayList<Statement>>> graknAppendAttributeInsert(ArrayList<String> rows, String header) throws Exception {
ArrayList<ArrayList<ArrayList<Statement>>> matchInsertStatements = new ArrayList<>();
public HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> graknAppendAttributeInsert(ArrayList<String> rows,
String header) throws Exception {
HashMap<String, ArrayList<ArrayList<ThingVariable<?>>>> matchInsertPatterns = new HashMap<>();

ArrayList<ArrayList<Statement>> matchStatements = new ArrayList<>();
ArrayList<ArrayList<Statement>> insertStatements = new ArrayList<>();
ArrayList<ArrayList<ThingVariable<?>>> matchPatterns = new ArrayList<>();
ArrayList<ArrayList<ThingVariable<?>>> insertPatterns = new ArrayList<>();

int insertCounter = 0;

for (String row : rows) {
ArrayList<ArrayList<Statement>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
ArrayList<ArrayList<ThingVariable<?>>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter);
if (tmp != null) {
if (tmp.get(0) != null && tmp.get(1) != null) {
matchStatements.add(tmp.get(0));
insertStatements.add(tmp.get(1));
matchPatterns.add(tmp.get(0));
insertPatterns.add(tmp.get(1));
insertCounter++;
}
}

}
matchInsertStatements.add(matchStatements);
matchInsertStatements.add(insertStatements);
return matchInsertStatements;
matchInsertPatterns.put("match", matchPatterns);
matchInsertPatterns.put("insert", insertPatterns);
return matchInsertPatterns;
}

public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String row, String header, int insertCounter) throws Exception {
public ArrayList<ArrayList<ThingVariable<?>>> graknAppendAttributeQueryFromRow(String row,
String header,
int insertCounter) throws Exception {
String fileSeparator = dce.getSeparator();
String[] rowTokens = row.split(fileSeparator);
String[] columnNames = header.split(fileSeparator);
Expand All @@ -63,35 +70,38 @@ public ArrayList<ArrayList<Statement>> graknAppendAttributeQueryFromRow(String r
throw new IllegalArgumentException("data config entry for " + dce.getDataPath() + " is incomplete - it needs at least one attribute used for matching (\"match\": true) and at least one attribute to be appended (\"match\": false or not set at all");
}

ArrayList<Statement> matchStatements = new ArrayList<>();
ArrayList<Statement> insertStatements = new ArrayList<>();
ArrayList<ThingVariable<?>> matchPatterns = new ArrayList<>();
ArrayList<ThingVariable<?>> insertPatterns = new ArrayList<>();

// get all attributes that are isMatch() --> construct match clause
StatementInstance appendAttributeMatchStatement = addEntityToMatchStatement(insertCounter);
Thing appendAttributeMatchPattern = addEntityToMatchPattern(insertCounter);
for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForMatchAttribute : dce.getAttributes()) {
if (generatorMappingForMatchAttribute.isMatch()){
appendAttributeMatchStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
if (generatorMappingForMatchAttribute.isMatch()) {
appendAttributeMatchPattern = addAttribute(rowTokens, appendAttributeMatchPattern, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor());
}
}
matchStatements.add(appendAttributeMatchStatement);
matchPatterns.add(appendAttributeMatchPattern);

// get all attributes that are !isMatch() --> construct insert clause
Statement appendAttributeInsertStatement = addEntityToInsertStatement(insertCounter);
UnboundVariable thingVar = addEntityToInsertPattern(insertCounter);
Thing appendAttributeInsertPattern = null;
for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAppendAttribute : dce.getAttributes()) {
if (!generatorMappingForAppendAttribute.isMatch()){
appendAttributeInsertStatement = addAttribute(rowTokens, appendAttributeMatchStatement, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
if (!generatorMappingForAppendAttribute.isMatch()) {
appendAttributeInsertPattern = addAttribute(rowTokens, thingVar, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor());
}
}
insertStatements.add((StatementInstance) appendAttributeInsertStatement);
if (appendAttributeInsertPattern != null) {
insertPatterns.add(appendAttributeInsertPattern);
}

ArrayList<ArrayList<Statement>> assembledStatements = new ArrayList<>();
assembledStatements.add(matchStatements);
assembledStatements.add(insertStatements);
ArrayList<ArrayList<ThingVariable<?>>> assembledPatterns = new ArrayList<>();
assembledPatterns.add(matchPatterns);
assembledPatterns.add(insertPatterns);


if (isValid(assembledStatements)) {
appLogger.debug("valid query: <" + assembleQuery(assembledStatements).toString() + ">");
return assembledStatements;
if (isValid(assembledPatterns)) {
appLogger.debug("valid query: <" + assembleQuery(assembledPatterns).toString() + ">");
return assembledPatterns;
} else {
dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not contain at least one match attribute and one insert attribute. Faulty tokenized row: " + Arrays.toString(rowTokens));
return null;
Expand All @@ -102,7 +112,7 @@ private boolean validateDataConfigEntry() {
boolean containsMatchAttribute = false;
boolean containsAppendAttribute = false;
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getAttributes()) {
if (attributeMapping.isMatch()){
if (attributeMapping.isMatch()) {
containsMatchAttribute = true;
}
if (!attributeMapping.isMatch()) {
Expand All @@ -112,51 +122,58 @@ private boolean validateDataConfigEntry() {
return containsMatchAttribute && containsAppendAttribute;
}

private StatementInstance addEntityToMatchStatement(int insertCounter) {
private Thing addEntityToMatchPattern(int insertCounter) {
if (pce.getSchemaType() != null) {
return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
// return Graql.var("e-" + insertCounter).isa(pce.getSchemaType());
return Graql.var("e").isa(pce.getSchemaType());
} else {
throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
}
}

private Statement addEntityToInsertStatement(int insertCounter) {
private UnboundVariable addEntityToInsertPattern(int insertCounter) {
if (pce.getSchemaType() != null) {
return Graql.var("e-" + insertCounter);
// return Graql.var("e-" + insertCounter);
return Graql.var("e");
} else {
throw new IllegalArgumentException("Required field <schemaType> not set in processor " + pce.getProcessor());
}
}

private String assembleQuery(ArrayList<ArrayList<Statement>> queries) {
private String assembleQuery(ArrayList<ArrayList<ThingVariable<?>>> queries) {
StringBuilder ret = new StringBuilder();
for (Statement st : queries.get(0)) {
for (ThingVariable st : queries.get(0)) {
ret.append(st.toString());
}
ret.append(queries.get(1).get(0).toString());
return ret.toString();
}

private boolean isValid(ArrayList<ArrayList<Statement>> si) {
ArrayList<Statement> matchStatements = si.get(0);
ArrayList<Statement> insertStatements = si.get(1);
StringBuilder matchStatement = new StringBuilder();
for (Statement st:matchStatements) {
matchStatement.append(st.toString());
private boolean isValid(ArrayList<ArrayList<ThingVariable<?>>> si) {
ArrayList<ThingVariable<?>> matchPatterns = si.get(0);
ArrayList<ThingVariable<?>> insertPatterns = si.get(1);

if (insertPatterns.size() < 1) {
return false;
}

StringBuilder matchPattern = new StringBuilder();
for (Pattern st : matchPatterns) {
matchPattern.append(st.toString());
}
String insertStatement = insertStatements.get(0).toString();
String insertPattern = insertPatterns.get(0).toString();

// missing match attribute
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping: dce.getMatchAttributes()) {
for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping : dce.getMatchAttributes()) {
String generatorKey = attributeMapping.getGenerator();
ProcessorConfigEntry.ConceptGenerator generatorEntry = pce.getAttributeGenerator(generatorKey);
if (!matchStatement.toString().contains("has " + generatorEntry.getAttributeType())) {
if (!matchPattern.toString().contains("has " + generatorEntry.getAttributeType())) {
return false;
}
}
// missing required insert attribute
for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry: pce.getRequiredAttributes().entrySet()) {
if (!insertStatement.contains("has " + generatorEntry.getValue().getAttributeType())) {
for (Map.Entry<String, ProcessorConfigEntry.ConceptGenerator> generatorEntry : pce.getRequiredAttributes().entrySet()) {
if (!insertPattern.contains("has " + generatorEntry.getValue().getAttributeType())) {
return false;
}
}
Expand Down
Loading

0 comments on commit bdf52db

Please sign in to comment.