Revert "Move SizeEstimator out of util"

sryza · sryza · commit 2e1a9066f9d9 · 2015-05-04T17:35:33.000-07:00
This reverts commit 93f4cd0.
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -2263,6 +2263,20 @@ object SparkContext extends Logging {
    */
   def jarOfObject(obj: AnyRef): Option[String] = jarOfClass(obj.getClass)
 
+  /**
+   * :: DeveloperApi ::
+   * Estimate the number of bytes that the given object takes up on the JVM heap. The estimate
+   * includes space taken up by objects referenced by the given object, their references, and so on
+   * and so forth.
+   *
+   * This is useful for determining the amount of heap space a broadcast variable will occupy on
+   * each executor or the amount of space each object will take when caching objects in
+   * deserialized form. This is not the same as the serialized size of the object, which will
+   * typically be much smaller.
+   */
+  @DeveloperApi
+  def estimateSizeOf(obj: AnyRef): Long = SizeEstimator.estimate(obj)
+
   /**
    * Creates a modified version of a SparkConf with the parameters that can be passed separately
    * to SparkContext, to make it easier to write SparkContext's constructors. This ignores
diff --git a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala
@@ -23,8 +23,7 @@ import java.util.LinkedHashMap
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.SizeEstimator
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{SizeEstimator, Utils}
 import org.apache.spark.util.collection.SizeTrackingVector
 
 private case class MemoryEntry(value: Any, size: Long, deserialized: Boolean)
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark
+package org.apache.spark.util
 
 import java.lang.management.ManagementFactory
 import java.lang.reflect.{Field, Modifier}
@@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.collection.mutable.ArrayBuffer
 import scala.runtime.ScalaRunTime
 
-import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.Logging
 import org.apache.spark.util.collection.OpenHashSet
 
 
@@ -35,8 +35,7 @@ import org.apache.spark.util.collection.OpenHashSet
  * Based on the following JavaWorld article:
  * http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html
  */
-@DeveloperApi
-object SizeEstimator extends Logging {
+private[spark] object SizeEstimator extends Logging {
 
   // Sizes of primitive types
   private val BYTE_SIZE    = 1
@@ -156,18 +155,6 @@ object SizeEstimator extends Logging {
     val shellSize: Long,
     val pointerFields: List[Field]) {}
 
-  /**
-   * :: DeveloperApi ::
-   * Estimate the number of bytes that the given object takes up on the JVM heap. The estimate
-   * includes space taken up by objects referenced by the given object, their references, and so on
-   * and so forth.
-   *
-   * This is useful for determining the amount of heap space a broadcast variable will occupy on
-   * each executor or the amount of space each object will take when caching objects in
-   * deserialized form. This is not the same as the serialized size of the object, which will
-   * typically be much smaller.
-   */
-  @DeveloperApi
   def estimate(obj: AnyRef): Long = estimate(obj, new IdentityHashMap[AnyRef, AnyRef])
 
   private def estimate(obj: AnyRef, visited: IdentityHashMap[AnyRef, AnyRef]): Long = {
diff --git a/core/src/main/scala/org/apache/spark/util/collection/SizeTracker.scala b/core/src/main/scala/org/apache/spark/util/collection/SizeTracker.scala
@@ -19,7 +19,7 @@ package org.apache.spark.util.collection
 
 import scala.collection.mutable
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 /**
  * A general interface for collections to keep track of their estimated sizes in bytes.
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -31,7 +31,7 @@ import org.scalatest.concurrent.Eventually._
 import org.scalatest.concurrent.Timeouts._
 
 import org.apache.spark.rpc.RpcEnv
-import org.apache.spark._
+import org.apache.spark.{MapOutputTrackerMaster, SparkConf, SparkContext, SecurityManager}
 import org.apache.spark.executor.DataReadMethod
 import org.apache.spark.network.nio.NioBlockTransferService
 import org.apache.spark.scheduler.LiveListenerBus
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -15,12 +15,11 @@
  * limitations under the License.
  */
 
-package org.apache.spark
+package org.apache.spark.util
 
 import scala.collection.mutable.ArrayBuffer
 
 import org.scalatest.{BeforeAndAfterEach, BeforeAndAfterAll, FunSuite, PrivateMethodTester}
-import org.apache.spark.util.ResetSystemProperties
 
 class DummyClass1 {}
 
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable.HashSet
 import org.scalatest.FunSuite
 import org.scalatest.Matchers
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 class OpenHashMapSuite extends FunSuite with Matchers {
 
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.util.collection
 import org.scalatest.FunSuite
 import org.scalatest.Matchers
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 class OpenHashSetSuite extends FunSuite with Matchers {
 
diff --git a/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMapSuite.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable.HashSet
 import org.scalatest.FunSuite
 import org.scalatest.Matchers
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 class PrimitiveKeyOpenHashMapSuite extends FunSuite with Matchers {
 
diff --git a/core/src/test/scala/org/apache/spark/util/collection/PrimitiveVectorSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/PrimitiveVectorSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.util.collection
 
 import org.scalatest.FunSuite
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 class PrimitiveVectorSuite extends FunSuite {
 
diff --git a/core/src/test/scala/org/apache/spark/util/collection/SizeTrackerSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/SizeTrackerSuite.scala
@@ -22,7 +22,7 @@ import scala.util.Random
 
 import org.scalatest.FunSuite
 
-import org.apache.spark.SizeEstimator
+import org.apache.spark.util.SizeEstimator
 
 class SizeTrackerSuite extends FunSuite {
   val NORMAL_ERROR = 0.20
diff --git a/docs/tuning.md b/docs/tuning.md
@@ -98,8 +98,8 @@ The best way to size the amount of memory consumption a dataset will require is
 into cache, and look at the "Storage" page in the web UI. The page will tell you how much memory the RDD
 is occupying.
 
-To estimate the memory consumption of a particular object, use `SizeEstimator`'s `estimate` method
-This is useful for experimenting with different data layouts to trim memory usage, as well as
+To estimate the memory consumption of a particular object, use the `SparkContext`'s `estimateSizeOf`
+method. This is useful for experimenting with different data layouts to trim memory usage, as well as
 determining the amount of space a broadcast variable will occupy on each executor heap.
 
 ## Tuning Data Structures
