Fixed bug at RuntimeTimePropertyGeneratorBuilder (#25)

* Fixed bug with visit method of EdgeTable and Match, which should not throw an exception

Fixed bug at InstantiatPropertyGeneratorOperator to fetch the correct
jar

Added test for a complete execution of the generator

* Removed commented code at DataSynthTest.scala

* Updated travis.yml, expecting tests to pass...

Author: ArnauPrat

.travis.yml

@@ -1,4 +1,6 @@
+dist: trusty
 language: scala
+sudo: required
 scala:
    - 2.11.2
 jdk:

src/main/scala/org/dama/datasynth/runtime/spark/operators/InstantiatePropertyGeneratorOperator.scala

@@ -24,8 +24,10 @@ class InstantiatePropertyGeneratorOperator {
     * @return The instantiated property generator
     */
   def apply[T](propertyTableName : String, info : ExecutionPlan.PropertyGenerator[T]) : PropertyGeneratorWrapper[T] = {
-    val initParameters : Seq[Object] = info.initParameters.map( x => SparkRuntime.evalValueOperator(x)).map( ref => ref.asInstanceOf[Object])
-    val urlCL = new URLClassLoader( Array[URL](new URL("file:///tmp/temp.jar")), getClass.getClassLoader());
+    val initParameters : Seq[Object] = info.initParameters.map( x => SparkRuntime.evalValueOperator(x))
+                                                          .map( ref => ref.asInstanceOf[Object])
+    val jarUrl = "file://"+SparkRuntime.getConfig().driverWorkspaceDir+"/temp.jar"
+    val urlCL = new URLClassLoader( Array[URL]( new URL(jarUrl)),getClass.getClassLoader());
     val constructor = urlCL.loadClass(info.className).getConstructors()(0)
     val generator = constructor.newInstance(initParameters : _*).asInstanceOf[PropertyGenerator[T]]
     val rndGen = SparkRuntime.fetchRndGeneratorOperator(propertyTableName)

src/main/scala/org/dama/datasynth/runtime/spark/passes/RuntimePropertyGeneratorBuilder.scala

@@ -19,16 +19,9 @@ import scala.reflect.runtime.universe.typeOf
   */
 class RuntimePropertyGeneratorBuilder(config : DataSynthConfig) extends ExecutionPlanNonVoidVisitor[RuntimeClasses] {
 
-
-  def codePropertyTableClasses(executionPlan:Seq[ExecutionPlan.Table]): RuntimeClasses={
+  def codePropertyTableClasses(executionPlan : Seq[ExecutionPlan.Table]) : RuntimeClasses = {
     /** Writing .scala files **/
-  /*  val classes:Map[String,(String,String)] = executionPlan.foldLeft(
-      Map[String,(String,String)]())( (currentDeclarations, nextNode) => currentDeclarations ++ nextNode.accept(this))
-    classes
-*/
-    val classes2:RuntimeClasses=executionPlan.foldLeft(new RuntimeClasses)((codeClassses, nextNode)=>codeClassses++nextNode.accept(this))
-    classes2
-
+    executionPlan.foldLeft(new RuntimeClasses)((codeClassses, nextNode) => codeClassses++nextNode.accept(this))
   }
 
   /**
@@ -43,35 +36,38 @@ class RuntimePropertyGeneratorBuilder(config : DataSynthConfig) extends Executio
     settings.nc.value = true
     settings.usejavacp.value = true
     settings.outputDirs.setSingleOutput(s"${config.driverWorkspaceDir}")
-    val currentJarPath : String = getClass().getProtectionDomain().getCodeSource().getLocation().getPath()
-    println(currentJarPath)
+    val currentJarPath : String = getClass()
+                                  .getProtectionDomain()
+                                  .getCodeSource()
+                                  .getLocation()
+                                  .getPath()
     settings.classpath.append(currentJarPath)
     val g = new Global(settings)
     val run = new g.Run
 
-
-
-
-    val sourceFileNames : List[String] = classes.toList.map({case (className,classCode)=>{
-    val fileName:String = s"${config.driverWorkspaceDir}/$className.scala"
-    val writer = new PrintWriter(new java.io.File(fileName))
-    writer.write(classCode)
-    writer.close()
-
-    fileName
-    }})
-
+    // Creating .scala source code files
+    val sourceFileNames : List[String] = classes.toList.map(
+      {
+        case (className,classCode) => {
+          val fileName : String = s"${config.driverWorkspaceDir}/$className.scala"
+          val writer = new PrintWriter(new java.io.File(fileName))
+          writer.write(classCode)
+          writer.close()
+          fileName
+        }
+      }
+    )
+
+    // Compiling the .scala source code files
     run.compile(sourceFileNames)
 
-    /** Building JAR **/
+    // Building JAR
     val jar : JarWriter = new JarWriter(new scala.reflect.io.File(new java.io.File(jarFileName)), new JManifest())
-    sourceFileNames.map( file => file.replace(".scala",".class")).foreach( file => jar.addFile(new scala.reflect.io.File( new java.io.File(file)),""))
+    sourceFileNames.map( file => file.replace(".scala",".class"))
+                                     .foreach( file => jar.addFile(new scala.reflect.io.File(new java.io.File(file)), ""))
     jar.close()
   }
 
-
-
-
   /**
     * Generates the name of a generated proeprty generator given an original generator name
     * @param name The name of the original property generator
@@ -113,7 +109,6 @@ class RuntimePropertyGeneratorBuilder(config : DataSynthConfig) extends Executio
     }
     val propertyGeneratorName:String = propertyGenerator.className
     val generatedClassName:String = generateClassName(className)
-   // val dependentGenerators = propertyGenerator.dependentPropertyTables.map(table => generateClassName(table.name))
     val dependentGeneratorsCallList : String = propertyGenerator.dependentPropertyTables.zipWithIndex.foldLeft("")(
       {case (current,(table,index)) => s"$current,${mkMatch(table,index)}"})
 
@@ -126,54 +121,36 @@ class RuntimePropertyGeneratorBuilder(config : DataSynthConfig) extends Executio
   }
 
   override def visit(node: ExecutionPlan.StaticValue[_]): RuntimeClasses =  {
-//    Map.empty
     throw new RuntimeException("No code should be generated for StaticValue[_]")
   }
 
   override def visit(node: ExecutionPlan.PropertyGenerator[_]): RuntimeClasses =  {
-    //    Map.empty
     throw new RuntimeException("No code should be generated for PropertyGenerator[_]")
   }
 
   override def visit(node: ExecutionPlan.StructureGenerator): RuntimeClasses =  {
-    //    Map.empty
     throw new RuntimeException("No code should be generated for StructureGenerator")
   }
 
-  /*override def visit(node: ExecutionPlan.PropertyTable[_]): Map[String, (String,String)] =  {
-    val classTypeName = generateClassName(node.name)
-    val classDeclaration = generatePGClassDefinition(classTypeName,node.generator)
-    val dependants = node.generator.dependentPropertyTables.map( table => table.accept[Map[String,(String,String)]](this) ).
-      foldLeft(Map[String,(String,String)]())( {case (accumulated, next) => accumulated ++ next} )
-    dependants + (node.name -> (classTypeName,classDeclaration))
-
-  }*/
-
   override def visit(node: ExecutionPlan.PropertyTable[_]): RuntimeClasses =  {
     val classTypeName = generateClassName(node.name)
     val classDeclaration = generatePGClassDefinition(classTypeName,node.generator)
 
-
     val dependants:RuntimeClasses = node.generator.dependentPropertyTables.map(table => table.accept[RuntimeClasses](this) ).
       foldLeft(new RuntimeClasses())( {case (accumulated, next) => accumulated ++ next} )
     val codeClass : RuntimeClass = new RuntimeClass(node.name,classTypeName,classDeclaration)
     dependants + codeClass
-dependants
   }
 
-
   override def visit(node: ExecutionPlan.EdgeTable): RuntimeClasses =  {
-    //    Map.empty
-    throw new RuntimeException("No code should be generated for EdgeTable")
+    new RuntimeClasses()
   }
 
   override def visit(node: ExecutionPlan.TableSize): RuntimeClasses =  {
-    //    Map.empty
     throw new RuntimeException("No code should be generated for TableSize")
   }
 
   override def visit(node: ExecutionPlan.Match): RuntimeClasses =  {
-    //    Map.empty
-    throw new RuntimeException("No code should be generated for Match")
+    new RuntimeClasses()
   }
 }

src/test/resources/test.json

@@ -0,0 +1,37 @@
+{
+  "nodeTypes" : [
+    {
+      "name" : "entity1",
+      "instances" : 1000000,
+      "properties" : [
+        {
+          "name": "attribute1",
+          "dataType": "Long",
+          "generator": {
+          "name":"org.dama.datasynth.common.generators.property.empirical.IntGenerator",
+          "dependencies":[],
+          "initParameters" : ["src/main/resources/distributions/intDistribution.txt:String"," :String"]}
+        },
+        {
+          "name": "attribute2",
+          "dataType": "Long",
+          "generator": {
+          "name":"org.dama.datasynth.common.generators.property.empirical.IntGenerator",
+          "dependencies":[],
+          "initParameters" : ["src/main/resources/distributions/intDistribution.txt:String"," :String"]}
+        }
+      ]
+    }
+  ],
+  "edgeTypes" : [
+    {
+      "name" : "edge1",
+      "source" : "entity1",
+      "target" : "entity1",
+      "structure" : {
+        "name" : "org.dama.datasynth.common.generators.structure.BTERGenerator",
+        "initParameters" : ["src/main/resources/degrees/dblp:String","src/main/resources/ccs/dblp:String"]
+      }
+    }
+  ]
+}

src/test/scala/org/dama/datasynth/DataSynthTest.scala

@@ -0,0 +1,34 @@
+package org.dama.datasynth
+
+import java.io.File
+import java.nio.file.{Files, Path, Paths}
+
+import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.SparkSession
+import org.junit.runner.RunWith
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import org.scalatest.junit.JUnitRunner
+
+/**
+  * Created by aprat on 17/07/17.
+  */
+@RunWith(classOf[JUnitRunner])
+class DataSynthTest extends FlatSpec with Matchers with BeforeAndAfterAll {
+
+
+  " The test schema at /src/test/resources/test.json should work " should " work " in {
+
+    SparkSession.builder().master("local[*]").getOrCreate()
+
+    val testFolder = new File("./test")
+    val dataFolder = new File("./test/data")
+    val workspaceFolder = new File("./test/workspace")
+    testFolder.mkdir()
+    dataFolder.mkdir()
+    workspaceFolder.mkdir()
+    DataSynth.main(List("--output-dir", dataFolder.getAbsolutePath,
+                        "--driver-workspace-dir", workspaceFolder.getAbsolutePath,
+                        "--schema-file", "src/test/resources/test.json").toArray)
+    FileUtils.deleteDirectory(testFolder)
+  }
+}