From 9983a42de1dc4d9cc8a810515d2c47f00f4ea2b6 Mon Sep 17 00:00:00 2001 From: nikhil7sh Date: Mon, 19 May 2014 11:34:02 +0530 Subject: [PATCH 1/3] [SPARK-1820] Make GenerateMimaIgnore @DeveloperApi annotation aware --- .../spark/tools/GenerateMIMAIgnore.scala | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index a433e8e2e89f..bbcacb099a2f 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -23,6 +23,7 @@ import java.util.jar.JarFile import scala.collection.mutable import scala.collection.JavaConversions._ import scala.reflect.runtime.universe.runtimeMirror +import scala.reflect.runtime.{universe => unv} /** * A tool for generating classes to be excluded during binary checking with MIMA. It is expected @@ -70,8 +71,22 @@ object GenerateMIMAIgnore { } } + def isDeveloperApi(className: String) = { + try { + val clazz = mirror.classSymbol(Class.forName(className, false, classLoader)) + + clazz.annotations.exists(_.tpe =:= unv.typeOf[org.apache.spark.annotation.DeveloperApi]) + } catch { + case _: Throwable => { + println("Error determining Annotations: " + className) + false + } + } + } + for (className <- classes) { val directlyPrivateSpark = isPackagePrivate(className) + val developerApi = isDeveloperApi(className) /* Inner classes defined within a private[spark] class or object are effectively invisible, so we account for them as package private. */ @@ -83,7 +98,8 @@ object GenerateMIMAIgnore { false } } - if (directlyPrivateSpark || indirectlyPrivateSpark) privateClasses += className + if (directlyPrivateSpark || indirectlyPrivateSpark || developerApi) privateClasses += + className } privateClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet } @@ -98,10 +114,10 @@ object GenerateMIMAIgnore { private def shouldExclude(name: String) = { // Heuristic to remove JVM classes that do not correspond to user-facing classes in Scala name.contains("anon") || - name.endsWith("$class") || - name.contains("$sp") || - name.contains("hive") || - name.contains("Hive") + name.endsWith("$class") || + name.contains("$sp") || + name.contains("hive") || + name.contains("Hive") } /** @@ -127,7 +143,7 @@ object GenerateMIMAIgnore { val jar = new JarFile(new File(jarPath)) val enums = jar.entries().map(_.getName).filter(_.startsWith(packageName)) val classes = for (entry <- enums if entry.endsWith(".class")) - yield Class.forName(entry.replace('/', '.').stripSuffix(".class"), false, classLoader) + yield Class.forName(entry.replace('/', '.').stripSuffix(".class"), false, classLoader) classes } -} +} \ No newline at end of file From e3c52151751fb81ef04412df45979762625e6ba8 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Wed, 28 May 2014 17:24:14 +0530 Subject: [PATCH 2/3] Incorporated patrick's suggestions and fixed the scalastyle build. --- .../spark/tools/GenerateMIMAIgnore.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index bbcacb099a2f..3f13402fb8f3 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -74,7 +74,6 @@ object GenerateMIMAIgnore { def isDeveloperApi(className: String) = { try { val clazz = mirror.classSymbol(Class.forName(className, false, classLoader)) - clazz.annotations.exists(_.tpe =:= unv.typeOf[org.apache.spark.annotation.DeveloperApi]) } catch { case _: Throwable => { @@ -98,8 +97,9 @@ object GenerateMIMAIgnore { false } } - if (directlyPrivateSpark || indirectlyPrivateSpark || developerApi) privateClasses += - className + if (directlyPrivateSpark || indirectlyPrivateSpark || developerApi) { + privateClasses += className + } } privateClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet } @@ -114,10 +114,10 @@ object GenerateMIMAIgnore { private def shouldExclude(name: String) = { // Heuristic to remove JVM classes that do not correspond to user-facing classes in Scala name.contains("anon") || - name.endsWith("$class") || - name.contains("$sp") || - name.contains("hive") || - name.contains("Hive") + name.endsWith("$class") || + name.contains("$sp") || + name.contains("hive") || + name.contains("Hive") } /** @@ -143,7 +143,7 @@ object GenerateMIMAIgnore { val jar = new JarFile(new File(jarPath)) val enums = jar.entries().map(_.getName).filter(_.startsWith(packageName)) val classes = for (entry <- enums if entry.endsWith(".class")) - yield Class.forName(entry.replace('/', '.').stripSuffix(".class"), false, classLoader) + yield Class.forName(entry.replace('/', '.').stripSuffix(".class"), false, classLoader) classes } -} \ No newline at end of file +} From de944f950ae4aad22bd41b9dced75863ed7fb805 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Thu, 29 May 2014 10:14:20 +0530 Subject: [PATCH 3/3] Code review. --- .../scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index 3f13402fb8f3..011db50b7d56 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -43,7 +43,7 @@ object GenerateMIMAIgnore { private def classesPrivateWithin(packageName: String): Set[String] = { val classes = getClasses(packageName) - val privateClasses = mutable.HashSet[String]() + val ignoredClasses = mutable.HashSet[String]() def isPackagePrivate(className: String) = { try { @@ -98,10 +98,10 @@ object GenerateMIMAIgnore { } } if (directlyPrivateSpark || indirectlyPrivateSpark || developerApi) { - privateClasses += className + ignoredClasses += className } } - privateClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet + ignoredClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet } def main(args: Array[String]) {