diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java index 88587c78e25a1..f3944152faefb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java @@ -18,6 +18,14 @@ package org.apache.hudi.common.util; +import org.apache.hudi.common.util.jvm.MemoryLayoutSpecification; +import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification32bit; +import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bit; +import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bitCompressed; +import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification32bit; +import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bit; +import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bitCompressed; + import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.reflect.Array; @@ -35,7 +43,7 @@ import java.util.Set; /** - * Contains utility methods for calculating the memory usage of objects. It only works on the HotSpot JVM, and infers + * Contains utility methods for calculating the memory usage of objects. It only works on the HotSpot and OpenJ9 JVMs, and infers * the actual memory layout (32 bit vs. 64 bit word size, compressed object pointers vs. uncompressed) from best * available indicators. It can reliably detect a 32 bit vs. 64 bit JVM. It can only make an educated guess at whether * compressed OOPs are used, though; specifically, it knows what the JVM's default choice of OOP compression would be @@ -46,48 +54,6 @@ * @author Attila Szegedi */ public class ObjectSizeCalculator { - - /** - * Describes constant memory overheads for various constructs in a JVM implementation. - */ - public interface MemoryLayoutSpecification { - - /** - * Returns the fixed overhead of an array of any type or length in this JVM. - * - * @return the fixed overhead of an array. - */ - int getArrayHeaderSize(); - - /** - * Returns the fixed overhead of for any {@link Object} subclass in this JVM. - * - * @return the fixed overhead of any object. - */ - int getObjectHeaderSize(); - - /** - * Returns the quantum field size for a field owned by an object in this JVM. - * - * @return the quantum field size for an object. - */ - int getObjectPadding(); - - /** - * Returns the fixed size of an object reference in this JVM. - * - * @return the size of all object references. - */ - int getReferenceSize(); - - /** - * Returns the quantum field size for a field owned by one of an object's ancestor superclasses in this JVM. - * - * @return the quantum field size for a superclass field. - */ - int getSuperclassFieldPadding(); - } - private static class CurrentLayout { private static final MemoryLayoutSpecification SPEC = getEffectiveMemoryLayoutSpecification(); @@ -328,109 +294,59 @@ private static long getPrimitiveFieldSize(Class type) { static MemoryLayoutSpecification getEffectiveMemoryLayoutSpecification() { final String vmName = System.getProperty("java.vm.name"); if (vmName == null || !(vmName.startsWith("Java HotSpot(TM) ") || vmName.startsWith("OpenJDK") - || vmName.startsWith("TwitterJDK"))) { - throw new UnsupportedOperationException("ObjectSizeCalculator only supported on HotSpot VM"); - } - - final String dataModel = System.getProperty("sun.arch.data.model"); - if ("32".equals(dataModel)) { - // Running with 32-bit data model - return new MemoryLayoutSpecification() { - @Override - public int getArrayHeaderSize() { - return 12; - } - - @Override - public int getObjectHeaderSize() { - return 8; - } - - @Override - public int getObjectPadding() { - return 8; - } - - @Override - public int getReferenceSize() { - return 4; - } - - @Override - public int getSuperclassFieldPadding() { - return 4; - } - }; - } else if (!"64".equals(dataModel)) { - throw new UnsupportedOperationException( - "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property"); + || vmName.startsWith("TwitterJDK") || vmName.startsWith("Eclipse OpenJ9"))) { + throw new UnsupportedOperationException("ObjectSizeCalculator only supported on HotSpot or Eclipse OpenJ9 VMs"); } final String strVmVersion = System.getProperty("java.vm.version"); - final int vmVersion = Integer.parseInt(strVmVersion.substring(0, strVmVersion.indexOf('.'))); - if (vmVersion >= 17) { + // Support for OpenJ9 JVM + if (strVmVersion.startsWith("openj9")) { + final String dataModel = System.getProperty("sun.arch.data.model"); + if ("32".equals(dataModel)) { + // Running with 32-bit data model + return new OpenJ9MemoryLayoutSpecification32bit(); + } else if (!"64".equals(dataModel)) { + throw new UnsupportedOperationException( + "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property"); + } + long maxMemory = 0; for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) { maxMemory += mp.getUsage().getMax(); } - if (maxMemory < 30L * 1024 * 1024 * 1024) { - // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total - // for all memory pools (yes, including code cache). - return new MemoryLayoutSpecification() { - @Override - public int getArrayHeaderSize() { - return 16; - } - - @Override - public int getObjectHeaderSize() { - return 12; - } - - @Override - public int getObjectPadding() { - return 8; - } - - @Override - public int getReferenceSize() { - return 4; - } - - @Override - public int getSuperclassFieldPadding() { - return 4; - } - }; - } - } - - // In other cases, it's a 64-bit uncompressed OOPs object model - return new MemoryLayoutSpecification() { - @Override - public int getArrayHeaderSize() { - return 24; - } - - @Override - public int getObjectHeaderSize() { - return 16; + if (maxMemory < 57L * 1024 * 1024 * 1024) { + // OpenJ9 use compressed references below 57GB of RAM total + return new OpenJ9MemoryLayoutSpecification64bitCompressed(); + } else { + // it's a 64-bit uncompressed references object model + return new OpenJ9MemoryLayoutSpecification64bit(); } - - @Override - public int getObjectPadding() { - return 8; + } else { + // Support for HotSpot JVM + final String dataModel = System.getProperty("sun.arch.data.model"); + if ("32".equals(dataModel)) { + // Running with 32-bit data model + return new HotSpotMemoryLayoutSpecification32bit(); + } else if (!"64".equals(dataModel)) { + throw new UnsupportedOperationException( + "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property"); } - @Override - public int getReferenceSize() { - return 8; + final int vmVersion = Integer.parseInt(strVmVersion.substring(0, strVmVersion.indexOf('.'))); + if (vmVersion >= 17) { + long maxMemory = 0; + for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) { + maxMemory += mp.getUsage().getMax(); + } + if (maxMemory < 30L * 1024 * 1024 * 1024) { + // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total + // for all memory pools (yes, including code cache). + return new HotSpotMemoryLayoutSpecification64bitCompressed(); + } } - @Override - public int getSuperclassFieldPadding() { - return 8; - } - }; + // In other cases, it's a 64-bit uncompressed OOPs object model + return new HotSpotMemoryLayoutSpecification64bit(); + } } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java new file mode 100644 index 0000000000000..e76c67e39276b --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class HotSpotMemoryLayoutSpecification32bit implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 12; + } + + @Override + public int getObjectHeaderSize() { + return 8; + } + + @Override + public int getObjectPadding() { + return 8; + } + + @Override + public int getReferenceSize() { + return 4; + } + + @Override + public int getSuperclassFieldPadding() { + return 4; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java new file mode 100644 index 0000000000000..4f4e2a4b6cabc --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class HotSpotMemoryLayoutSpecification64bit implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 24; + } + + @Override + public int getObjectHeaderSize() { + return 16; + } + + @Override + public int getObjectPadding() { + return 8; + } + + @Override + public int getReferenceSize() { + return 8; + } + + @Override + public int getSuperclassFieldPadding() { + return 8; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java new file mode 100644 index 0000000000000..60ad8be288991 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class HotSpotMemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 16; + } + + @Override + public int getObjectHeaderSize() { + return 12; + } + + @Override + public int getObjectPadding() { + return 8; + } + + @Override + public int getReferenceSize() { + return 4; + } + + @Override + public int getSuperclassFieldPadding() { + return 4; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java new file mode 100644 index 0000000000000..fe6d421e5c357 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +/** + * Describes constant memory overheads for various constructs in a JVM implementation. + */ +public interface MemoryLayoutSpecification { + + /** + * Returns the fixed overhead of an array of any type or length in this JVM. + * + * @return the fixed overhead of an array. + */ + int getArrayHeaderSize(); + + /** + * Returns the fixed overhead of for any {@link Object} subclass in this JVM. + * + * @return the fixed overhead of any object. + */ + int getObjectHeaderSize(); + + /** + * Returns the quantum field size for a field owned by an object in this JVM. + * + * @return the quantum field size for an object. + */ + int getObjectPadding(); + + /** + * Returns the fixed size of an object reference in this JVM. + * + * @return the size of all object references. + */ + int getReferenceSize(); + + /** + * Returns the quantum field size for a field owned by one of an object's ancestor superclasses in this JVM. + * + * @return the quantum field size for a superclass field. + */ + int getSuperclassFieldPadding(); +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java new file mode 100644 index 0000000000000..1e66e0cecf122 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class OpenJ9MemoryLayoutSpecification32bit implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 16; + } + + @Override + public int getObjectHeaderSize() { + return 4; + } + + @Override + public int getObjectPadding() { + return 4; + } + + @Override + public int getReferenceSize() { + return 4; + } + + @Override + public int getSuperclassFieldPadding() { + return 4; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java new file mode 100644 index 0000000000000..d89d05ae475b2 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class OpenJ9MemoryLayoutSpecification64bit implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 16; + } + + @Override + public int getObjectHeaderSize() { + return 16; + } + + @Override + public int getObjectPadding() { + return 8; + } + + @Override + public int getReferenceSize() { + return 8; + } + + @Override + public int getSuperclassFieldPadding() { + return 8; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java new file mode 100644 index 0000000000000..05a457525a361 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.util.jvm; + +public class OpenJ9MemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification { + @Override + public int getArrayHeaderSize() { + return 16; + } + + @Override + public int getObjectHeaderSize() { + return 4; + } + + @Override + public int getObjectPadding() { + return 4; + } + + @Override + public int getReferenceSize() { + return 4; + } + + @Override + public int getSuperclassFieldPadding() { + return 4; + } +}