From 25270be7b0f190b3aed5999cc7bd64b6db261afd Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 6 Dec 2018 19:41:12 +0100 Subject: [PATCH 01/37] Mem tools by including trait and specific implementation in the crate (it is the less impacting approach). --- Cargo.toml | 3 +- mem/Cargo.toml | 31 +++++++++++ mem/README.md | 7 +++ mem/src/alloc.rs | 124 +++++++++++++++++++++++++++++++++++++++++ mem/src/lib.rs | 98 ++++++++++++++++++++++++++++++++ mem/src/malloc_size.rs | 1 + 6 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 mem/Cargo.toml create mode 100644 mem/README.md create mode 100644 mem/src/alloc.rs create mode 100644 mem/src/lib.rs create mode 120000 mem/src/malloc_size.rs diff --git a/Cargo.toml b/Cargo.toml index a8796ce87..4d6812658 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,5 +17,6 @@ members = [ "trace-time", "trie-standardmap", "triehash", - "uint" + "uint", + "mem", ] diff --git a/mem/Cargo.toml b/mem/Cargo.toml new file mode 100644 index 000000000..de14682b2 --- /dev/null +++ b/mem/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "parity-util-mem" +version = "0.1.0" +authors = ["Parity Technologies "] +repository = "https://github.com/paritytech/parity-common" +description = "Collection of memory related utilities" +license = "GPL-3.0" + +[dependencies] +clear_on_drop = "0.2" +#malloc_size_of = { path = "../../servo/components/malloc_size_of", default-features = false } +malloc_size_of_derive = { path = "../../servo/components/malloc_size_of_derive" } +dlmalloc = { version = "0.1", optional = true } +wee_alloc = { version = "0.4", optional = true } +jemallocator = { version = "0.1", optional = true } + +elastic-array = { version = "0.10", optional = true } +ethereum-types = { version = "0.4", optional = true } + +[features] +default = ["ethereum-impls"] +# when activated mem is removed through volatile primitive instead of clear_on_drop crate +volatile-erase = [] +# use dlmalloc as global allocator +dlmalloc-global = ["dlmalloc"] +# use wee_alloc as global allocator +weealloc-global = ["wee_alloc"] +# use jemalloc as global allocator +jemalloc-global = ["jemallocator"] +# implement additional types +ethereum-impls = ["ethereum-types", "elastic-array"] diff --git a/mem/README.md b/mem/README.md new file mode 100644 index 000000000..80b1fd7ec --- /dev/null +++ b/mem/README.md @@ -0,0 +1,7 @@ +# parity-util-mem + +Collection of memory related utilities. + +## Features + +- volatile-erase : Not set by default, `Memzero` erase memory with `write_volatile`. diff --git a/mem/src/alloc.rs b/mem/src/alloc.rs new file mode 100644 index 000000000..f689fe727 --- /dev/null +++ b/mem/src/alloc.rs @@ -0,0 +1,124 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! default allocator management +//! Features are: +//! - windows: +//! - no features: default implementation from servo `heapsize` crate +//! - weealloc: enable, untested +//! - dlmalloc: enable, does not work, for compatibility only +//! - arch x86: +//! - no features: use default alloc +//! - jemalloc: use jemallocator crate +//! - weealloc: enable, untested +//! - dlmalloc: enable, does not work, for compatibility only +//! - arch wasm32: +//! - no features: return 0 for compatibility +//! - dlmalloc: return 0 for compatibility (usable_size could be implemented if needed in +//! dlmalloc crate) +//! - weealloc: enable +//! - jemalloc: compile_error + +use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf, MallocUnconditionalSizeOf}; +use std::os::raw::c_void; + +#[cfg(windows)] +mod usable_size { + #[cfg(target_os = "windows")] + extern crate winapi; + + #[cfg(target_os = "windows")] + use winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; + use std::os::raw::c_void; + + /// Get the size of a heap block. + pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { + + let heap = GetProcessHeap(); + + if HeapValidate(heap, 0, ptr) == 0 { + ptr = *(ptr as *const *const c_void).offset(-1); + } + + HeapSize(heap, 0, ptr) as usize + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } +} + +#[cfg(all(not(windows), not(target_arch = "wasm32")))] +// default +mod usable_size { + use super::*; + + extern "C" { + #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] + pub fn malloc_usable_size(ptr: *const c_void) -> usize; + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } + +} + +/// Get a new instance of a MallocSizeOfOps +pub fn new_malloc_size_ops() -> MallocSizeOfOps { + MallocSizeOfOps::new( + usable_size::malloc_usable_size, + usable_size::new_enclosing_size_fn(), + None, + ) +} + +/*/// Glue trait for quick switch in parity-ethereum +/// `malloc_size_of` traits should be used next +pub trait HeapSizeOf { + fn heap_size_of_children(&self) -> usize; +} + +impl HeapSizeOf for T { + fn heap_size_of_children(&self) -> usize { + let mut ops = new_malloc_size_ops(); + + ::size_of(self, &mut ops) + } +}*/ + +/// Extension methods for `MallocSizeOf` +pub trait MallocSizeOfExt: MallocSizeOf { + fn m_size_of(&self) -> usize { + let mut ops = new_malloc_size_ops(); + ::size_of(self, &mut ops) + } +} + +impl MallocSizeOfExt for T {} + + +/// we currently do not have use case where a conditional fn is use so +/// we default to unconditional mettering +/// It would be interesting to run some test with global mutex other weak handle in ops to check +/// how much we measure multiple times +impl MallocSizeOf for std::sync::Arc { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.unconditional_size_of(ops) + } +} diff --git a/mem/src/lib.rs b/mem/src/lib.rs new file mode 100644 index 000000000..b53999f2a --- /dev/null +++ b/mem/src/lib.rs @@ -0,0 +1,98 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Memory related utilities. + +extern crate clear_on_drop as cod; + +//extern crate malloc_size_of as malloc_size; +#[macro_use] extern crate malloc_size_of_derive as malloc_size_derive; + +use std::ops::{Deref, DerefMut}; + +#[cfg(feature = "volatile-erase")] +use std::ptr; + +#[cfg(not(feature = "volatile-erase"))] +pub use cod::clear::Clear; + +pub mod alloc; + +/// This is a copy of patched crate `malloc_size_of` as a module. +/// We need to have it as an inner module to be able to define our own traits implementation, +/// if at some point the trait become standard enough we could use the right way of doing it +/// by implementing it in our type traits crates. At this time a move on this trait if implemented +/// at primitive types level would impact to much of the dependency to be easilly manageable. +#[macro_use] mod malloc_size; + +#[cfg(feature = "ethereum-impls")] +pub mod impls; + +/// reexport clear_on_drop crate +pub mod clear_on_drop { + pub use cod::*; +} + +pub use malloc_size_derive::*; +pub use malloc_size::{ + MallocSizeOfOps, + MallocSizeOf, +}; +pub use alloc::MallocSizeOfExt; + +/// Wrapper to zero out memory when dropped. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Memzero> { + mem: T, +} + +impl> From for Memzero { + fn from(mem: T) -> Memzero { + Memzero { mem } + } +} + +#[cfg(feature = "volatile-erase")] +impl> Drop for Memzero { + fn drop(&mut self) { + unsafe { + for byte_ref in self.mem.as_mut() { + ptr::write_volatile(byte_ref, 0) + } + } + } +} + +#[cfg(not(feature = "volatile-erase"))] +impl> Drop for Memzero { + fn drop(&mut self) { + self.as_mut().clear(); + } +} + +impl> Deref for Memzero { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.mem + } +} + +impl> DerefMut for Memzero { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.mem + } +} diff --git a/mem/src/malloc_size.rs b/mem/src/malloc_size.rs new file mode 120000 index 000000000..8d759630c --- /dev/null +++ b/mem/src/malloc_size.rs @@ -0,0 +1 @@ +../../../servo/components/malloc_size_of/lib.rs \ No newline at end of file From 94bb29355ba489aeed7cf41573d263fea673c951 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 7 Dec 2018 10:58:27 +0100 Subject: [PATCH 02/37] Plug malloc_size_of as a patched copy. Documentation is done by `get_malloc_size_src.sh`. Add missing impls.rs with ethereum types related implementation. --- mem/Cargo.toml | 5 +- mem/README.md | 7 + mem/get_malloc_size_src.sh | 13 + mem/malloc_size_of_derive/Cargo.toml | 15 + mem/malloc_size_of_derive/LICENSE-APACHE | 201 ++++ mem/malloc_size_of_derive/LICENSE-MIT | 23 + mem/malloc_size_of_derive/lib.rs | 125 +++ mem/slim_malloc_size_of.patch | 429 +++++++++ mem/src/alloc.rs | 69 +- mem/src/impls.rs | 56 ++ mem/src/malloc_size.rs | 1118 +++++++++++++++++++++- 11 files changed, 2043 insertions(+), 18 deletions(-) create mode 100755 mem/get_malloc_size_src.sh create mode 100644 mem/malloc_size_of_derive/Cargo.toml create mode 100644 mem/malloc_size_of_derive/LICENSE-APACHE create mode 100644 mem/malloc_size_of_derive/LICENSE-MIT create mode 100644 mem/malloc_size_of_derive/lib.rs create mode 100644 mem/slim_malloc_size_of.patch create mode 100644 mem/src/impls.rs mode change 120000 => 100644 mem/src/malloc_size.rs diff --git a/mem/Cargo.toml b/mem/Cargo.toml index de14682b2..28c1a0260 100644 --- a/mem/Cargo.toml +++ b/mem/Cargo.toml @@ -8,8 +8,7 @@ license = "GPL-3.0" [dependencies] clear_on_drop = "0.2" -#malloc_size_of = { path = "../../servo/components/malloc_size_of", default-features = false } -malloc_size_of_derive = { path = "../../servo/components/malloc_size_of_derive" } +malloc_size_of_derive = { path = "./malloc_size_of_derive" } dlmalloc = { version = "0.1", optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } @@ -29,3 +28,5 @@ weealloc-global = ["wee_alloc"] jemalloc-global = ["jemallocator"] # implement additional types ethereum-impls = ["ethereum-types", "elastic-array"] +# Default malloc to conditional mettering +conditional-mettering = [] diff --git a/mem/README.md b/mem/README.md index 80b1fd7ec..ee1e5fa2c 100644 --- a/mem/README.md +++ b/mem/README.md @@ -5,3 +5,10 @@ Collection of memory related utilities. ## Features - volatile-erase : Not set by default, `Memzero` erase memory with `write_volatile`. +- conditional-metering : Try to avoid counting `Arc` twice. For test only. + +Others feature are here to define global allocator, see `src/alloc.rs`. + +## Dependency + +This crate groups common dependency, `clear_on_drop` is reexported, and a patched copy of unpublished `malloc_size_of` from servo project is used and partially reexported. diff --git a/mem/get_malloc_size_src.sh b/mem/get_malloc_size_src.sh new file mode 100755 index 000000000..75df50571 --- /dev/null +++ b/mem/get_malloc_size_src.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# script/process to update code from servo project (malloc_size_of) +# untested, note that we do not use submodule due to size of git repo +git clone https://github.com/servo/servo.git +cd servo +git checkout 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc +git apply ../slim_malloc_size_of.patch +#git merge master +cp components/malloc_size_of/lib.rs ../src/malloc_size.rs +cp -r components/malloc_size_of_derive .. +cd .. +rm -rf ./servo diff --git a/mem/malloc_size_of_derive/Cargo.toml b/mem/malloc_size_of_derive/Cargo.toml new file mode 100644 index 000000000..4cfc2ee12 --- /dev/null +++ b/mem/malloc_size_of_derive/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "malloc_size_of_derive" +version = "0.0.1" +authors = ["The Servo Project Developers"] +license = "MIT/Apache-2.0" +publish = false + +[lib] +path = "lib.rs" +proc-macro = true + +[dependencies] +proc-macro2 = "0.4" +syn = { version = "0.15", features = ["full"] } +synstructure = "0.10" diff --git a/mem/malloc_size_of_derive/LICENSE-APACHE b/mem/malloc_size_of_derive/LICENSE-APACHE new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/mem/malloc_size_of_derive/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/mem/malloc_size_of_derive/LICENSE-MIT b/mem/malloc_size_of_derive/LICENSE-MIT new file mode 100644 index 000000000..31aa79387 --- /dev/null +++ b/mem/malloc_size_of_derive/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/mem/malloc_size_of_derive/lib.rs b/mem/malloc_size_of_derive/lib.rs new file mode 100644 index 000000000..0838f6cfb --- /dev/null +++ b/mem/malloc_size_of_derive/lib.rs @@ -0,0 +1,125 @@ +// Copyright 2016-2017 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! A crate for deriving the MallocSizeOf trait. + +extern crate proc_macro2; +#[macro_use] +extern crate syn; +#[macro_use] +extern crate synstructure; + +#[cfg(not(test))] +decl_derive!([MallocSizeOf, attributes(ignore_malloc_size_of)] => malloc_size_of_derive); + +fn malloc_size_of_derive(s: synstructure::Structure) -> proc_macro2::TokenStream { + let match_body = s.each(|binding| { + let ignore = binding + .ast() + .attrs + .iter() + .any(|attr| match attr.interpret_meta().unwrap() { + syn::Meta::Word(ref ident) | syn::Meta::List(syn::MetaList { ref ident, .. }) + if ident == "ignore_malloc_size_of" => + { + panic!( + "#[ignore_malloc_size_of] should have an explanation, \ + e.g. #[ignore_malloc_size_of = \"because reasons\"]" + ); + } + syn::Meta::NameValue(syn::MetaNameValue { ref ident, .. }) + if ident == "ignore_malloc_size_of" => + { + true + }, + _ => false, + }); + if ignore { + None + } else if let syn::Type::Array(..) = binding.ast().ty { + Some(quote! { + for item in #binding.iter() { + sum += ::malloc_size_of::MallocSizeOf::size_of(item, ops); + } + }) + } else { + Some(quote! { + sum += ::malloc_size_of::MallocSizeOf::size_of(#binding, ops); + }) + } + }); + + let ast = s.ast(); + let name = &ast.ident; + let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl(); + let mut where_clause = where_clause.unwrap_or(&parse_quote!(where)).clone(); + for param in ast.generics.type_params() { + let ident = ¶m.ident; + where_clause + .predicates + .push(parse_quote!(#ident: ::malloc_size_of::MallocSizeOf)); + } + + let tokens = quote! { + impl #impl_generics ::malloc_size_of::MallocSizeOf for #name #ty_generics #where_clause { + #[inline] + #[allow(unused_variables, unused_mut, unreachable_code)] + fn size_of(&self, ops: &mut ::malloc_size_of::MallocSizeOfOps) -> usize { + let mut sum = 0; + match *self { + #match_body + } + sum + } + } + }; + + tokens +} + +#[test] +fn test_struct() { + let source = syn::parse_str( + "struct Foo { bar: Bar, baz: T, #[ignore_malloc_size_of = \"\"] z: Arc }", + ) + .unwrap(); + let source = synstructure::Structure::new(&source); + + let expanded = malloc_size_of_derive(source).to_string(); + let mut no_space = expanded.replace(" ", ""); + macro_rules! match_count { + ($e: expr, $count: expr) => { + assert_eq!( + no_space.matches(&$e.replace(" ", "")).count(), + $count, + "counting occurences of {:?} in {:?} (whitespace-insensitive)", + $e, + expanded + ) + }; + } + match_count!("struct", 0); + match_count!("ignore_malloc_size_of", 0); + match_count!("impl ::malloc_size_of::MallocSizeOf for Foo where T: ::malloc_size_of::MallocSizeOf {", 1); + match_count!("sum += ::malloc_size_of::MallocSizeOf::size_of(", 2); + + let source = syn::parse_str("struct Bar([Baz; 3]);").unwrap(); + let source = synstructure::Structure::new(&source); + let expanded = malloc_size_of_derive(source).to_string(); + no_space = expanded.replace(" ", ""); + match_count!("for item in", 1); +} + +#[should_panic(expected = "should have an explanation")] +#[test] +fn test_no_reason() { + let input = syn::parse_str("struct A { #[ignore_malloc_size_of] b: C }").unwrap(); + malloc_size_of_derive(synstructure::Structure::new(&input)); +} diff --git a/mem/slim_malloc_size_of.patch b/mem/slim_malloc_size_of.patch new file mode 100644 index 000000000..859cc13c8 --- /dev/null +++ b/mem/slim_malloc_size_of.patch @@ -0,0 +1,429 @@ +diff --git a/components/malloc_size_of/Cargo.toml b/components/malloc_size_of/Cargo.toml +index 22840c89d8..4a2b27f827 100644 +--- a/components/malloc_size_of/Cargo.toml ++++ b/components/malloc_size_of/Cargo.toml +@@ -9,6 +9,10 @@ publish = false + path = "lib.rs" + + [features] ++default = [ ++ "extra", ++ "serde_only", ++] + servo = [ + "crossbeam-channel", + "hyper", +@@ -23,29 +27,45 @@ servo = [ + "webrender_api", + "xml5ever", + ] ++serde_only = [ ++ "serde", ++ "serde_bytes", ++] ++extra = [ ++ "app_units", ++ "cssparser", ++ "euclid", ++ "hashglobe", ++ "selectors", ++ "servo_arc", ++ "smallbitvec", ++ "smallvec", ++ "thin-slice", ++ "void", ++] + + [dependencies] +-app_units = "0.7" ++app_units = { version = "0.7", optional = true } + crossbeam-channel = { version = "0.3", optional = true } +-cssparser = "0.25" +-euclid = "0.19" +-hashglobe = { path = "../hashglobe" } ++cssparser = { version = "0.25", optional = true } ++euclid = { version = "0.19", optional = true } ++hashglobe = { path = "../hashglobe", optional = true } + hyper = { version = "0.12", optional = true } + hyper_serde = { version = "0.9", optional = true } + keyboard-types = {version = "0.4.3", optional = true} +-selectors = { path = "../selectors" } ++selectors = { path = "../selectors", optional = true } + serde = { version = "1.0.27", optional = true } + serde_bytes = { version = "0.10", optional = true } +-servo_arc = { path = "../servo_arc" } +-smallbitvec = "2.1.0" +-smallvec = "0.6" ++servo_arc = { path = "../servo_arc", optional = true } ++smallbitvec = { version = "2.1.0", optional = true } ++smallvec = { version = "0.6", optional = true } + string_cache = { version = "0.7", optional = true } +-thin-slice = "0.1.0" ++thin-slice = { version = "0.1.0", optional = true } + time = { version = "0.1.17", optional = true } + url = { version = "1.2", optional = true } + webrender_api = { git = "https://github.com/servo/webrender", features = ["ipc"], optional = true } + xml5ever = { version = "0.12", optional = true } +-void = "1.0.2" ++void = { version = "1.0.2", optional = true } + + [target.'cfg(target_os = "android")'.dependencies] + mozjs = { version = "0.9.5", optional = true, features=["init_once"]} +diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs +index 778082b5f0..9c52bcfcd4 100644 +--- a/components/malloc_size_of/lib.rs ++++ b/components/malloc_size_of/lib.rs +@@ -43,11 +43,18 @@ + //! measured as well as the thing it points to. E.g. + //! ` as MallocSizeOf>::size_of(field, ops)`. + ++ ++// This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. ++ ++#[cfg(feature = "extra")] + extern crate app_units; + #[cfg(feature = "servo")] + extern crate crossbeam_channel; ++#[cfg(feature = "extra")] + extern crate cssparser; ++#[cfg(feature = "extra")] + extern crate euclid; ++#[cfg(feature = "extra")] + extern crate hashglobe; + #[cfg(feature = "servo")] + extern crate hyper; +@@ -57,41 +64,51 @@ extern crate hyper_serde; + extern crate keyboard_types; + #[cfg(feature = "servo")] + extern crate mozjs as js; ++#[cfg(feature = "extra")] + extern crate selectors; +-#[cfg(feature = "servo")] ++#[cfg(any(feature = "servo", feature = "serde_only"))] + extern crate serde; +-#[cfg(feature = "servo")] ++#[cfg(any(feature = "servo", feature = "serde_only"))] + extern crate serde_bytes; ++#[cfg(feature = "extra")] + extern crate servo_arc; ++#[cfg(feature = "extra")] + extern crate smallbitvec; ++#[cfg(feature = "extra")] + extern crate smallvec; + #[cfg(feature = "servo")] + extern crate string_cache; ++#[cfg(feature = "extra")] + extern crate thin_slice; + #[cfg(feature = "servo")] + extern crate time; + #[cfg(feature = "url")] + extern crate url; ++#[cfg(feature = "extra")] + extern crate void; + #[cfg(feature = "webrender_api")] + extern crate webrender_api; + #[cfg(feature = "servo")] + extern crate xml5ever; + +-#[cfg(feature = "servo")] ++#[cfg(not(feature = "extra"))] ++use std::sync as servo_arc; ++ ++#[cfg(any(feature = "servo", feature = "serde_only"))] + use serde_bytes::ByteBuf; + use std::hash::{BuildHasher, Hash}; + use std::mem::size_of; + use std::ops::Range; + use std::ops::{Deref, DerefMut}; + use std::os::raw::c_void; ++#[cfg(feature = "extra")] + use void::Void; + + /// A C function that takes a pointer to a heap allocation and returns its size. +-type VoidPtrToSizeFn = unsafe extern "C" fn(ptr: *const c_void) -> usize; ++pub type VoidPtrToSizeFn = unsafe extern "C" fn(ptr: *const c_void) -> usize; + + /// A closure implementing a stateful predicate on pointers. +-type VoidPtrToBoolFnMut = FnMut(*const c_void) -> bool; ++pub type VoidPtrToBoolFnMut = FnMut(*const c_void) -> bool; + + /// Operations used when measuring heap usage of data structures. + pub struct MallocSizeOfOps { +@@ -241,6 +258,7 @@ impl MallocSizeOf for Box { + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = 0; +@@ -253,6 +271,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for thin_slice::ThinBoxedSlice { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.shallow_size_of(ops) + (**self).size_of(ops) +@@ -351,14 +370,14 @@ impl MallocSizeOf for [T] { + } + } + +-#[cfg(feature = "servo")] ++#[cfg(any(feature = "servo", feature = "serde_only"))] + impl MallocShallowSizeOf for ByteBuf { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } + } + +-#[cfg(feature = "servo")] ++#[cfg(any(feature = "servo", feature = "serde_only"))] + impl MallocSizeOf for ByteBuf { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); +@@ -412,6 +431,7 @@ impl MallocSizeOf for std::collections::VecDeque { + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for smallvec::SmallVec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if self.spilled() { +@@ -422,6 +442,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for smallvec::SmallVec + where + A: smallvec::Array, +@@ -471,6 +492,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for hashglobe::hash_set::HashSet + where + T: Eq + Hash, +@@ -488,6 +510,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for hashglobe::hash_set::HashSet + where + T: Eq + Hash + MallocSizeOf, +@@ -502,6 +525,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for hashglobe::fake::HashSet + where + T: Eq + Hash, +@@ -513,6 +537,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for hashglobe::fake::HashSet + where + T: Eq + Hash + MallocSizeOf, +@@ -587,6 +612,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for hashglobe::hash_map::HashMap + where + K: Eq + Hash, +@@ -604,6 +630,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for hashglobe::hash_map::HashMap + where + K: Eq + Hash + MallocSizeOf, +@@ -620,6 +647,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocShallowSizeOf for hashglobe::fake::HashMap + where + K: Eq + Hash, +@@ -631,6 +659,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for hashglobe::fake::HashMap + where + K: Eq + Hash + MallocSizeOf, +@@ -657,9 +686,20 @@ impl MallocSizeOf for std::marker::PhantomData { + //impl !MallocSizeOf for Arc { } + //impl !MallocShallowSizeOf for Arc { } + ++#[cfg(feature = "extra")] ++fn arc_ptr(s: &servo_arc::Arc) -> * const T { ++ s.heap_ptr() ++} ++ ++#[cfg(not(feature = "extra"))] ++fn arc_ptr(s: &servo_arc::Arc) -> * const T { ++ let sc = s.clone(); ++ servo_arc::Arc::into_raw(sc) ++} ++ + impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.heap_ptr()) } ++ unsafe { ops.malloc_size_of(arc_ptr(self)) } + } + } + +@@ -671,7 +711,7 @@ impl MallocUnconditionalSizeOf for servo_arc::Arc { + + impl MallocConditionalShallowSizeOf for servo_arc::Arc { + fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- if ops.have_seen_ptr(self.heap_ptr()) { ++ if ops.have_seen_ptr(arc_ptr(self)) { + 0 + } else { + self.unconditional_shallow_size_of(ops) +@@ -681,7 +721,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { + + impl MallocConditionalSizeOf for servo_arc::Arc { + fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- if ops.have_seen_ptr(self.heap_ptr()) { ++ if ops.have_seen_ptr(arc_ptr(self)) { + 0 + } else { + self.unconditional_size_of(ops) +@@ -701,6 +741,7 @@ impl MallocSizeOf for std::sync::Mutex { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for smallbitvec::SmallBitVec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if let Some(ptr) = self.heap_ptr() { +@@ -711,30 +752,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::Length { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedScale { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedPoint2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.x.size_of(ops) + self.y.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedRect { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.origin.size_of(ops) + self.size.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedSideOffsets2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.top.size_of(ops) + +@@ -744,12 +790,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedSize2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.width.size_of(ops) + self.height.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for euclid::TypedTransform2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.m11.size_of(ops) + +@@ -761,6 +809,7 @@ impl MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform3D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.m11.size_of(ops) + +@@ -782,12 +831,14 @@ impl MallocSizeOf for euclid::TypedTransform3D MallocSizeOf for euclid::TypedVector2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.x.size_of(ops) + self.y.size_of(ops) + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for selectors::parser::AncestorHashes { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let selectors::parser::AncestorHashes { ref packed_hashes } = *self; +@@ -795,6 +846,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for selectors::parser::Selector + where + Impl::NonTSPseudoClass: MallocSizeOf, +@@ -815,6 +867,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for selectors::parser::Component + where + Impl::NonTSPseudoClass: MallocSizeOf, +@@ -860,6 +913,7 @@ where + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf + for selectors::attr::AttrSelectorWithOptionalNamespace + { +@@ -868,6 +922,7 @@ impl MallocSizeOf + } + } + ++#[cfg(feature = "extra")] + impl MallocSizeOf for Void { + #[inline] + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +@@ -929,8 +984,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range + malloc_size_of_is_0!(Range, Range, Range, Range, Range); + malloc_size_of_is_0!(Range, Range); + ++#[cfg(feature = "extra")] + malloc_size_of_is_0!(app_units::Au); + ++#[cfg(feature = "extra")] + malloc_size_of_is_0!(cssparser::RGBA, cssparser::TokenSerializationType); + + #[cfg(feature = "url")] diff --git a/mem/src/alloc.rs b/mem/src/alloc.rs index f689fe727..d6224c60e 100644 --- a/mem/src/alloc.rs +++ b/mem/src/alloc.rs @@ -32,7 +32,11 @@ //! - weealloc: enable //! - jemalloc: compile_error -use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf, MallocUnconditionalSizeOf}; +use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; +#[cfg(feature = "conditional-mettering")] +use malloc_size::MallocConditionalSizeOf; +#[cfg(not(feature = "conditional-mettering"))] +use malloc_size::MallocUnconditionalSizeOf; use std::os::raw::c_void; #[cfg(windows)] @@ -88,37 +92,72 @@ pub fn new_malloc_size_ops() -> MallocSizeOfOps { ) } -/*/// Glue trait for quick switch in parity-ethereum -/// `malloc_size_of` traits should be used next -pub trait HeapSizeOf { - fn heap_size_of_children(&self) -> usize; +#[cfg(feature = "conditional-mettering")] +/// Get a new instance of a MallocSizeOfOps with a haveseen ptr function +pub fn new_count_malloc_size_ops(count_fn: Box) -> MallocSizeOfOps { + MallocSizeOfOps::new( + usable_size::malloc_usable_size, + usable_size::new_enclosing_size_fn(), + count_fn, + ) } -impl HeapSizeOf for T { - fn heap_size_of_children(&self) -> usize { - let mut ops = new_malloc_size_ops(); +#[cfg(feature = "conditional-mettering")] +/// count function for testing purpose only (slow) +pub fn test_count() -> Box bool> { + let mut set = std::collections::HashSet::new(); + Box::new(move |ptr| { + let r = if set.contains(&ptr) { + true + } else { + set.insert(ptr); + false + }; + r + }) +} - ::size_of(self, &mut ops) +#[cfg(not(feature = "conditional-mettering"))] +/// Extension methods for `MallocSizeOf` +pub trait MallocSizeOfExt: MallocSizeOf { + fn m_size_of(&self) -> usize { + let mut ops = new_malloc_size_ops(); + ::size_of(self, &mut ops) } -}*/ +} +// TODO remove this implementation (conditional metering on purpose only) +#[cfg(feature = "conditional-mettering")] /// Extension methods for `MallocSizeOf` pub trait MallocSizeOfExt: MallocSizeOf { fn m_size_of(&self) -> usize { let mut ops = new_malloc_size_ops(); - ::size_of(self, &mut ops) + let mut opscond = new_count_malloc_size_ops(test_count()); + let cond = ::size_of(self, &mut opscond); + let notcond = Self as MallocSizeOf>::size_of(self, &mut ops); + if cond != notcond { + println!("conditional mettering did absorb: {}", notcond - cond); + } + cond } } impl MallocSizeOfExt for T {} - /// we currently do not have use case where a conditional fn is use so /// we default to unconditional mettering /// It would be interesting to run some test with global mutex other weak handle in ops to check /// how much we measure multiple times +#[cfg(not(feature = "conditional-mettering"))] impl MallocSizeOf for std::sync::Arc { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.unconditional_size_of(ops) - } + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.unconditional_size_of(ops) + } +} + +#[cfg(feature = "conditional-mettering")] +impl MallocSizeOf for std::sync::Arc { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.conditional_size_of(ops) + } } diff --git a/mem/src/impls.rs b/mem/src/impls.rs new file mode 100644 index 000000000..72080bb97 --- /dev/null +++ b/mem/src/impls.rs @@ -0,0 +1,56 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Implementation of common types. +//! Note that the implementation (there is some macro call redundancy here) could be done in the types crate, +//! but it creates some hard to maintain/update trait dependencies. + +extern crate elastic_array; +extern crate ethereum_types; + +use self::ethereum_types::*; +use self::elastic_array::*; +use super::{MallocSizeOf, MallocSizeOfOps}; + +malloc_size_of_is_0!( + U64, U128, U256, U512, H32, H64, + H128, H160, H256, H264, H512, H520, + Bloom +); + +macro_rules! impl_elastic_array { + ($name: ident, $dummy: ident, $size: expr) => ( + impl MallocSizeOf for $name + where T: MallocSizeOf { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self[..].size_of(ops) + } + } + ) +} + +impl_elastic_array!(ElasticArray2, ElasticArray2Dummy, 2); +impl_elastic_array!(ElasticArray4, ElasticArray4Dummy, 4); +impl_elastic_array!(ElasticArray8, ElasticArray8Dummy, 8); +impl_elastic_array!(ElasticArray16, ElasticArray16Dummy, 16); +impl_elastic_array!(ElasticArray32, ElasticArray32Dummy, 32); +impl_elastic_array!(ElasticArray36, ElasticArray36Dummy, 36); +impl_elastic_array!(ElasticArray64, ElasticArray64Dummy, 64); +impl_elastic_array!(ElasticArray128, ElasticArray128Dummy, 128); +impl_elastic_array!(ElasticArray256, ElasticArray256Dummy, 256); +impl_elastic_array!(ElasticArray512, ElasticArray512Dummy, 512); +impl_elastic_array!(ElasticArray1024, ElasticArray1024Dummy, 1024); +impl_elastic_array!(ElasticArray2048, ElasticArray2048Dummy, 2048); diff --git a/mem/src/malloc_size.rs b/mem/src/malloc_size.rs deleted file mode 120000 index 8d759630c..000000000 --- a/mem/src/malloc_size.rs +++ /dev/null @@ -1 +0,0 @@ -../../../servo/components/malloc_size_of/lib.rs \ No newline at end of file diff --git a/mem/src/malloc_size.rs b/mem/src/malloc_size.rs new file mode 100644 index 000000000..206299128 --- /dev/null +++ b/mem/src/malloc_size.rs @@ -0,0 +1,1117 @@ +// Copyright 2016-2017 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! A crate for measuring the heap usage of data structures in a way that +//! integrates with Firefox's memory reporting, particularly the use of +//! mozjemalloc and DMD. In particular, it has the following features. +//! - It isn't bound to a particular heap allocator. +//! - It provides traits for both "shallow" and "deep" measurement, which gives +//! flexibility in the cases where the traits can't be used. +//! - It allows for measuring blocks even when only an interior pointer can be +//! obtained for heap allocations, e.g. `HashSet` and `HashMap`. (This relies +//! on the heap allocator having suitable support, which mozjemalloc has.) +//! - It allows handling of types like `Rc` and `Arc` by providing traits that +//! are different to the ones for non-graph structures. +//! +//! Suggested uses are as follows. +//! - When possible, use the `MallocSizeOf` trait. (Deriving support is +//! provided by the `malloc_size_of_derive` crate.) +//! - If you need an additional synchronization argument, provide a function +//! that is like the standard trait method, but with the extra argument. +//! - If you need multiple measurements for a type, provide a function named +//! `add_size_of` that takes a mutable reference to a struct that contains +//! the multiple measurement fields. +//! - When deep measurement (via `MallocSizeOf`) cannot be implemented for a +//! type, shallow measurement (via `MallocShallowSizeOf`) in combination with +//! iteration can be a useful substitute. +//! - `Rc` and `Arc` are always tricky, which is why `MallocSizeOf` is not (and +//! should not be) implemented for them. +//! - If an `Rc` or `Arc` is known to be a "primary" reference and can always +//! be measured, it should be measured via the `MallocUnconditionalSizeOf` +//! trait. +//! - If an `Rc` or `Arc` should be measured only if it hasn't been seen +//! before, it should be measured via the `MallocConditionalSizeOf` trait. +//! - Using universal function call syntax is a good idea when measuring boxed +//! fields in structs, because it makes it clear that the Box is being +//! measured as well as the thing it points to. E.g. +//! ` as MallocSizeOf>::size_of(field, ops)`. + + +// This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. + +#[cfg(feature = "extra")] +extern crate app_units; +#[cfg(feature = "servo")] +extern crate crossbeam_channel; +#[cfg(feature = "extra")] +extern crate cssparser; +#[cfg(feature = "extra")] +extern crate euclid; +#[cfg(feature = "extra")] +extern crate hashglobe; +#[cfg(feature = "servo")] +extern crate hyper; +#[cfg(feature = "servo")] +extern crate hyper_serde; +#[cfg(feature = "servo")] +extern crate keyboard_types; +#[cfg(feature = "servo")] +extern crate mozjs as js; +#[cfg(feature = "extra")] +extern crate selectors; +#[cfg(any(feature = "servo", feature = "serde_only"))] +extern crate serde; +#[cfg(any(feature = "servo", feature = "serde_only"))] +extern crate serde_bytes; +#[cfg(feature = "extra")] +extern crate servo_arc; +#[cfg(feature = "extra")] +extern crate smallbitvec; +#[cfg(feature = "extra")] +extern crate smallvec; +#[cfg(feature = "servo")] +extern crate string_cache; +#[cfg(feature = "extra")] +extern crate thin_slice; +#[cfg(feature = "servo")] +extern crate time; +#[cfg(feature = "url")] +extern crate url; +#[cfg(feature = "extra")] +extern crate void; +#[cfg(feature = "webrender_api")] +extern crate webrender_api; +#[cfg(feature = "servo")] +extern crate xml5ever; + +#[cfg(not(feature = "extra"))] +use std::sync as servo_arc; + +#[cfg(any(feature = "servo", feature = "serde_only"))] +use serde_bytes::ByteBuf; +use std::hash::{BuildHasher, Hash}; +use std::mem::size_of; +use std::ops::Range; +use std::ops::{Deref, DerefMut}; +use std::os::raw::c_void; +#[cfg(feature = "extra")] +use void::Void; + +/// A C function that takes a pointer to a heap allocation and returns its size. +pub type VoidPtrToSizeFn = unsafe extern "C" fn(ptr: *const c_void) -> usize; + +/// A closure implementing a stateful predicate on pointers. +pub type VoidPtrToBoolFnMut = FnMut(*const c_void) -> bool; + +/// Operations used when measuring heap usage of data structures. +pub struct MallocSizeOfOps { + /// A function that returns the size of a heap allocation. + size_of_op: VoidPtrToSizeFn, + + /// Like `size_of_op`, but can take an interior pointer. Optional because + /// not all allocators support this operation. If it's not provided, some + /// memory measurements will actually be computed estimates rather than + /// real and accurate measurements. + enclosing_size_of_op: Option, + + /// Check if a pointer has been seen before, and remember it for next time. + /// Useful when measuring `Rc`s and `Arc`s. Optional, because many places + /// don't need it. + have_seen_ptr_op: Option>, +} + +impl MallocSizeOfOps { + pub fn new( + size_of: VoidPtrToSizeFn, + malloc_enclosing_size_of: Option, + have_seen_ptr: Option>, + ) -> Self { + MallocSizeOfOps { + size_of_op: size_of, + enclosing_size_of_op: malloc_enclosing_size_of, + have_seen_ptr_op: have_seen_ptr, + } + } + + /// Check if an allocation is empty. This relies on knowledge of how Rust + /// handles empty allocations, which may change in the future. + fn is_empty(ptr: *const T) -> bool { + // The correct condition is this: + // `ptr as usize <= ::std::mem::align_of::()` + // But we can't call align_of() on a ?Sized T. So we approximate it + // with the following. 256 is large enough that it should always be + // larger than the required alignment, but small enough that it is + // always in the first page of memory and therefore not a legitimate + // address. + return ptr as *const usize as usize <= 256; + } + + /// Call `size_of_op` on `ptr`, first checking that the allocation isn't + /// empty, because some types (such as `Vec`) utilize empty allocations. + pub unsafe fn malloc_size_of(&self, ptr: *const T) -> usize { + if MallocSizeOfOps::is_empty(ptr) { + 0 + } else { + (self.size_of_op)(ptr as *const c_void) + } + } + + /// Is an `enclosing_size_of_op` available? + pub fn has_malloc_enclosing_size_of(&self) -> bool { + self.enclosing_size_of_op.is_some() + } + + /// Call `enclosing_size_of_op`, which must be available, on `ptr`, which + /// must not be empty. + pub unsafe fn malloc_enclosing_size_of(&self, ptr: *const T) -> usize { + assert!(!MallocSizeOfOps::is_empty(ptr)); + (self.enclosing_size_of_op.unwrap())(ptr as *const c_void) + } + + /// Call `have_seen_ptr_op` on `ptr`. + pub fn have_seen_ptr(&mut self, ptr: *const T) -> bool { + let have_seen_ptr_op = self + .have_seen_ptr_op + .as_mut() + .expect("missing have_seen_ptr_op"); + have_seen_ptr_op(ptr as *const c_void) + } +} + +/// Trait for measuring the "deep" heap usage of a data structure. This is the +/// most commonly-used of the traits. +pub trait MallocSizeOf { + /// Measure the heap usage of all descendant heap-allocated structures, but + /// not the space taken up by the value itself. + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +/// Trait for measuring the "shallow" heap usage of a container. +pub trait MallocShallowSizeOf { + /// Measure the heap usage of immediate heap-allocated descendant + /// structures, but not the space taken up by the value itself. Anything + /// beyond the immediate descendants must be measured separately, using + /// iteration. + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +/// Like `MallocSizeOf`, but with a different name so it cannot be used +/// accidentally with derive(MallocSizeOf). For use with types like `Rc` and +/// `Arc` when appropriate (e.g. when measuring a "primary" reference). +pub trait MallocUnconditionalSizeOf { + /// Measure the heap usage of all heap-allocated descendant structures, but + /// not the space taken up by the value itself. + fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +/// `MallocUnconditionalSizeOf` combined with `MallocShallowSizeOf`. +pub trait MallocUnconditionalShallowSizeOf { + /// `unconditional_size_of` combined with `shallow_size_of`. + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +/// Like `MallocSizeOf`, but only measures if the value hasn't already been +/// measured. For use with types like `Rc` and `Arc` when appropriate (e.g. +/// when there is no "primary" reference). +pub trait MallocConditionalSizeOf { + /// Measure the heap usage of all heap-allocated descendant structures, but + /// not the space taken up by the value itself, and only if that heap usage + /// hasn't already been measured. + fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +/// `MallocConditionalSizeOf` combined with `MallocShallowSizeOf`. +pub trait MallocConditionalShallowSizeOf { + /// `conditional_size_of` combined with `shallow_size_of`. + fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; +} + +impl MallocSizeOf for String { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } +} + +impl<'a, T: ?Sized> MallocSizeOf for &'a T { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + // Zero makes sense for a non-owning reference. + 0 + } +} + +impl MallocShallowSizeOf for Box { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(&**self) } + } +} + +impl MallocSizeOf for Box { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.shallow_size_of(ops) + (**self).size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = 0; + unsafe { + n += thin_slice::ThinBoxedSlice::spilled_storage(self) + .map_or(0, |ptr| ops.malloc_size_of(ptr)); + n += ops.malloc_size_of(&**self); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for thin_slice::ThinBoxedSlice { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.shallow_size_of(ops) + (**self).size_of(ops) + } +} + +impl MallocSizeOf for () { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +impl MallocSizeOf for (T1, T2) +where + T1: MallocSizeOf, + T2: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + self.1.size_of(ops) + } +} + +impl MallocSizeOf for (T1, T2, T3) +where + T1: MallocSizeOf, + T2: MallocSizeOf, + T3: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + self.1.size_of(ops) + self.2.size_of(ops) + } +} + +impl MallocSizeOf for (T1, T2, T3, T4) +where + T1: MallocSizeOf, + T2: MallocSizeOf, + T3: MallocSizeOf, + T4: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + self.1.size_of(ops) + self.2.size_of(ops) + self.3.size_of(ops) + } +} + +impl MallocSizeOf for Option { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if let Some(val) = self.as_ref() { + val.size_of(ops) + } else { + 0 + } + } +} + +impl MallocSizeOf for Result { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + match *self { + Ok(ref x) => x.size_of(ops), + Err(ref e) => e.size_of(ops), + } + } +} + +impl MallocSizeOf for std::cell::Cell { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.get().size_of(ops) + } +} + +impl MallocSizeOf for std::cell::RefCell { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.borrow().size_of(ops) + } +} + +impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> +where + B::Owned: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + match *self { + std::borrow::Cow::Borrowed(_) => 0, + std::borrow::Cow::Owned(ref b) => b.size_of(ops), + } + } +} + +impl MallocSizeOf for [T] { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = 0; + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} + +#[cfg(any(feature = "servo", feature = "serde_only"))] +impl MallocShallowSizeOf for ByteBuf { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } +} + +#[cfg(any(feature = "servo", feature = "serde_only"))] +impl MallocSizeOf for ByteBuf { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} + +impl MallocShallowSizeOf for Vec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } +} + +impl MallocSizeOf for Vec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} + +impl MallocShallowSizeOf for std::collections::VecDeque { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.has_malloc_enclosing_size_of() { + if let Some(front) = self.front() { + // The front element is an interior pointer. + unsafe { ops.malloc_enclosing_size_of(&*front) } + } else { + // This assumes that no memory is allocated when the VecDeque is empty. + 0 + } + } else { + // An estimate. + self.capacity() * size_of::() + } + } +} + +impl MallocSizeOf for std::collections::VecDeque { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for smallvec::SmallVec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if self.spilled() { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } else { + 0 + } + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for smallvec::SmallVec +where + A: smallvec::Array, + A::Item: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} + +impl MallocShallowSizeOf for std::collections::HashSet +where + T: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.has_malloc_enclosing_size_of() { + // The first value from the iterator gives us an interior pointer. + // `ops.malloc_enclosing_size_of()` then gives us the storage size. + // This assumes that the `HashSet`'s contents (values and hashes) + // are all stored in a single contiguous heap allocation. + self.iter() + .next() + .map_or(0, |t| unsafe { ops.malloc_enclosing_size_of(t) }) + } else { + // An estimate. + self.capacity() * (size_of::() + size_of::()) + } + } +} + +impl MallocSizeOf for std::collections::HashSet +where + T: Eq + Hash + MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for t in self.iter() { + n += t.size_of(ops); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for hashglobe::hash_set::HashSet +where + T: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + // See the implementation for std::collections::HashSet for details. + if ops.has_malloc_enclosing_size_of() { + self.iter() + .next() + .map_or(0, |t| unsafe { ops.malloc_enclosing_size_of(t) }) + } else { + self.capacity() * (size_of::() + size_of::()) + } + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for hashglobe::hash_set::HashSet +where + T: Eq + Hash + MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for t in self.iter() { + n += t.size_of(ops); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for hashglobe::fake::HashSet +where + T: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + use std::ops::Deref; + self.deref().shallow_size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for hashglobe::fake::HashSet +where + T: Eq + Hash + MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + use std::ops::Deref; + self.deref().size_of(ops) + } +} + +impl MallocShallowSizeOf for std::collections::HashMap +where + K: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + // See the implementation for std::collections::HashSet for details. + if ops.has_malloc_enclosing_size_of() { + self.values() + .next() + .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) + } else { + self.capacity() * (size_of::() + size_of::() + size_of::()) + } + } +} + +impl MallocSizeOf for std::collections::HashMap +where + K: Eq + Hash + MallocSizeOf, + V: MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for (k, v) in self.iter() { + n += k.size_of(ops); + n += v.size_of(ops); + } + n + } +} + +impl MallocShallowSizeOf for std::collections::BTreeMap +where + K: Eq + Hash, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.has_malloc_enclosing_size_of() { + self.values() + .next() + .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) + } else { + self.len() * (size_of::() + size_of::() + size_of::()) + } + } +} + +impl MallocSizeOf for std::collections::BTreeMap +where + K: Eq + Hash + MallocSizeOf, + V: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for (k, v) in self.iter() { + n += k.size_of(ops); + n += v.size_of(ops); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for hashglobe::hash_map::HashMap +where + K: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + // See the implementation for std::collections::HashSet for details. + if ops.has_malloc_enclosing_size_of() { + self.values() + .next() + .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) + } else { + self.capacity() * (size_of::() + size_of::() + size_of::()) + } + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for hashglobe::hash_map::HashMap +where + K: Eq + Hash + MallocSizeOf, + V: MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for (k, v) in self.iter() { + n += k.size_of(ops); + n += v.size_of(ops); + } + n + } +} + +#[cfg(feature = "extra")] +impl MallocShallowSizeOf for hashglobe::fake::HashMap +where + K: Eq + Hash, + S: BuildHasher, +{ + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + use std::ops::Deref; + self.deref().shallow_size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for hashglobe::fake::HashMap +where + K: Eq + Hash + MallocSizeOf, + V: MallocSizeOf, + S: BuildHasher, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + use std::ops::Deref; + self.deref().size_of(ops) + } +} + +// PhantomData is always 0. +impl MallocSizeOf for std::marker::PhantomData { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +// XXX: we don't want MallocSizeOf to be defined for Rc and Arc. If negative +// trait bounds are ever allowed, this code should be uncommented. +// (We do have a compile-fail test for this: +// rc_arc_must_not_derive_malloc_size_of.rs) +//impl !MallocSizeOf for Arc { } +//impl !MallocShallowSizeOf for Arc { } + +#[cfg(feature = "extra")] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { + s.heap_ptr() +} + +#[cfg(not(feature = "extra"))] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { + let sc = s.clone(); + servo_arc::Arc::into_raw(sc) +} + +impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(arc_ptr(self)) } + } +} + +impl MallocUnconditionalSizeOf for servo_arc::Arc { + fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) + } +} + +impl MallocConditionalShallowSizeOf for servo_arc::Arc { + fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.have_seen_ptr(arc_ptr(self)) { + 0 + } else { + self.unconditional_shallow_size_of(ops) + } + } +} + +impl MallocConditionalSizeOf for servo_arc::Arc { + fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.have_seen_ptr(arc_ptr(self)) { + 0 + } else { + self.unconditional_size_of(ops) + } + } +} + +/// If a mutex is stored directly as a member of a data type that is being measured, +/// it is the unique owner of its contents and deserves to be measured. +/// +/// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, +/// the Arc will not be automatically measured so there is no risk of overcounting the mutex's +/// contents. +impl MallocSizeOf for std::sync::Mutex { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + (*self.lock().unwrap()).size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for smallbitvec::SmallBitVec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if let Some(ptr) = self.heap_ptr() { + unsafe { ops.malloc_size_of(ptr) } + } else { + 0 + } + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::Length { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedScale { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedPoint2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.x.size_of(ops) + self.y.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedRect { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.origin.size_of(ops) + self.size.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedSideOffsets2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.top.size_of(ops) + + self.right.size_of(ops) + + self.bottom.size_of(ops) + + self.left.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedSize2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.width.size_of(ops) + self.height.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedTransform2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.m11.size_of(ops) + + self.m12.size_of(ops) + + self.m21.size_of(ops) + + self.m22.size_of(ops) + + self.m31.size_of(ops) + + self.m32.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedTransform3D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.m11.size_of(ops) + + self.m12.size_of(ops) + + self.m13.size_of(ops) + + self.m14.size_of(ops) + + self.m21.size_of(ops) + + self.m22.size_of(ops) + + self.m23.size_of(ops) + + self.m24.size_of(ops) + + self.m31.size_of(ops) + + self.m32.size_of(ops) + + self.m33.size_of(ops) + + self.m34.size_of(ops) + + self.m41.size_of(ops) + + self.m42.size_of(ops) + + self.m43.size_of(ops) + + self.m44.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for euclid::TypedVector2D { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.x.size_of(ops) + self.y.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for selectors::parser::AncestorHashes { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let selectors::parser::AncestorHashes { ref packed_hashes } = *self; + packed_hashes.size_of(ops) + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for selectors::parser::Selector +where + Impl::NonTSPseudoClass: MallocSizeOf, + Impl::PseudoElement: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = 0; + + // It's OK to measure this ThinArc directly because it's the + // "primary" reference. (The secondary references are on the + // Stylist.) + n += unsafe { ops.malloc_size_of(self.thin_arc_heap_ptr()) }; + for component in self.iter_raw_match_order() { + n += component.size_of(ops); + } + + n + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for selectors::parser::Component +where + Impl::NonTSPseudoClass: MallocSizeOf, + Impl::PseudoElement: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + use selectors::parser::Component; + + match self { + Component::AttributeOther(ref attr_selector) => attr_selector.size_of(ops), + Component::Negation(ref components) => components.size_of(ops), + Component::NonTSPseudoClass(ref pseudo) => (*pseudo).size_of(ops), + Component::Slotted(ref selector) | Component::Host(Some(ref selector)) => { + selector.size_of(ops) + }, + Component::PseudoElement(ref pseudo) => (*pseudo).size_of(ops), + Component::Combinator(..) | + Component::ExplicitAnyNamespace | + Component::ExplicitNoNamespace | + Component::DefaultNamespace(..) | + Component::Namespace(..) | + Component::ExplicitUniversalType | + Component::LocalName(..) | + Component::ID(..) | + Component::Class(..) | + Component::AttributeInNoNamespaceExists { .. } | + Component::AttributeInNoNamespace { .. } | + Component::FirstChild | + Component::LastChild | + Component::OnlyChild | + Component::Root | + Component::Empty | + Component::Scope | + Component::NthChild(..) | + Component::NthLastChild(..) | + Component::NthOfType(..) | + Component::NthLastOfType(..) | + Component::FirstOfType | + Component::LastOfType | + Component::OnlyOfType | + Component::Host(None) => 0, + } + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf + for selectors::attr::AttrSelectorWithOptionalNamespace +{ + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +#[cfg(feature = "extra")] +impl MallocSizeOf for Void { + #[inline] + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + void::unreachable(*self) + } +} + +#[cfg(feature = "servo")] +impl MallocSizeOf for string_cache::Atom { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +// This is measured properly by the heap measurement implemented in +// SpiderMonkey. +#[cfg(feature = "servo")] +impl MallocSizeOf for js::jsapi::Heap { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +/// For use on types where size_of() returns 0. +#[macro_export] +macro_rules! malloc_size_of_is_0( + ($($ty:ty),+) => ( + $( + impl $crate::MallocSizeOf for $ty { + #[inline(always)] + fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { + 0 + } + } + )+ + ); + ($($ty:ident<$($gen:ident),+>),+) => ( + $( + impl<$($gen: $crate::MallocSizeOf),+> $crate::MallocSizeOf for $ty<$($gen),+> { + #[inline(always)] + fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { + 0 + } + } + )+ + ); +); + +malloc_size_of_is_0!(bool, char, str); +malloc_size_of_is_0!(u8, u16, u32, u64, u128, usize); +malloc_size_of_is_0!(i8, i16, i32, i64, i128, isize); +malloc_size_of_is_0!(f32, f64); + +malloc_size_of_is_0!(std::sync::atomic::AtomicBool); +malloc_size_of_is_0!(std::sync::atomic::AtomicIsize); +malloc_size_of_is_0!(std::sync::atomic::AtomicUsize); + +malloc_size_of_is_0!(Range, Range, Range, Range, Range); +malloc_size_of_is_0!(Range, Range, Range, Range, Range); +malloc_size_of_is_0!(Range, Range); + +#[cfg(feature = "extra")] +malloc_size_of_is_0!(app_units::Au); + +#[cfg(feature = "extra")] +malloc_size_of_is_0!(cssparser::RGBA, cssparser::TokenSerializationType); + +#[cfg(feature = "url")] +impl MallocSizeOf for url::Host { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + match *self { + url::Host::Domain(ref s) => s.size_of(ops), + _ => 0, + } + } +} +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::BorderRadius); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::BorderStyle); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::BoxShadowClipMode); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ClipAndScrollInfo); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ColorF); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ComplexClipRegion); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ExtendMode); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::FilterOp); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ExternalScrollId); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::FontInstanceKey); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::GradientStop); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::GlyphInstance); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::NinePatchBorder); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ImageKey); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ImageRendering); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::LineStyle); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::MixBlendMode); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::NormalBorder); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::RepeatMode); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::ScrollSensitivity); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::StickyOffsetBounds); +#[cfg(feature = "webrender_api")] +malloc_size_of_is_0!(webrender_api::TransformStyle); + +#[cfg(feature = "servo")] +impl MallocSizeOf for keyboard_types::Key { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + match self { + keyboard_types::Key::Character(ref s) => s.size_of(ops), + _ => 0, + } + } +} + +#[cfg(feature = "servo")] +malloc_size_of_is_0!(keyboard_types::Modifiers); + +#[cfg(feature = "servo")] +impl MallocSizeOf for xml5ever::QualName { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.prefix.size_of(ops) + self.ns.size_of(ops) + self.local.size_of(ops) + } +} + +#[cfg(feature = "servo")] +malloc_size_of_is_0!(time::Duration); +#[cfg(feature = "servo")] +malloc_size_of_is_0!(time::Tm); + +#[cfg(feature = "servo")] +impl MallocSizeOf for hyper_serde::Serde +where + for<'de> hyper_serde::De: serde::Deserialize<'de>, + for<'a> hyper_serde::Ser<'a, T>: serde::Serialize, + T: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.0.size_of(ops) + } +} + +// Placeholder for unique case where internals of Sender cannot be measured. +// malloc size of is 0 macro complains about type supplied! +#[cfg(feature = "servo")] +impl MallocSizeOf for crossbeam_channel::Sender { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +#[cfg(feature = "servo")] +impl MallocSizeOf for hyper::StatusCode { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 + } +} + +/// Measurable that defers to inner value and used to verify MallocSizeOf implementation in a +/// struct. +#[derive(Clone)] +pub struct Measurable(pub T); + +impl Deref for Measurable { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl DerefMut for Measurable { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} From 197a27db5824a3cc076ae54e5aa4304a0d63027a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 7 Dec 2018 16:53:44 +0100 Subject: [PATCH 03/37] A few additional impl for parity-eth use --- mem/Cargo.toml | 7 ++++--- mem/src/impls.rs | 37 +++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/mem/Cargo.toml b/mem/Cargo.toml index 28c1a0260..70c34cfb5 100644 --- a/mem/Cargo.toml +++ b/mem/Cargo.toml @@ -13,8 +13,9 @@ dlmalloc = { version = "0.1", optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } -elastic-array = { version = "0.10", optional = true } -ethereum-types = { version = "0.4", optional = true } +elastic-array = { version = "*", optional = true } +ethereum-types = { version = "*", optional = true } +parking_lot = { version = "*", optional = true } [features] default = ["ethereum-impls"] @@ -27,6 +28,6 @@ weealloc-global = ["wee_alloc"] # use jemalloc as global allocator jemalloc-global = ["jemallocator"] # implement additional types -ethereum-impls = ["ethereum-types", "elastic-array"] +ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] # Default malloc to conditional mettering conditional-mettering = [] diff --git a/mem/src/impls.rs b/mem/src/impls.rs index 72080bb97..e0e4497b9 100644 --- a/mem/src/impls.rs +++ b/mem/src/impls.rs @@ -20,26 +20,30 @@ extern crate elastic_array; extern crate ethereum_types; +extern crate parking_lot; use self::ethereum_types::*; use self::elastic_array::*; +use self::parking_lot::{Mutex, RwLock}; use super::{MallocSizeOf, MallocSizeOfOps}; +malloc_size_of_is_0!(std::time::Instant); + malloc_size_of_is_0!( - U64, U128, U256, U512, H32, H64, - H128, H160, H256, H264, H512, H520, - Bloom + U64, U128, U256, U512, H32, H64, + H128, H160, H256, H264, H512, H520, + Bloom ); macro_rules! impl_elastic_array { ($name: ident, $dummy: ident, $size: expr) => ( - impl MallocSizeOf for $name - where T: MallocSizeOf { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self[..].size_of(ops) - } - } - ) + impl MallocSizeOf for $name + where T: MallocSizeOf { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self[..].size_of(ops) + } + } + ) } impl_elastic_array!(ElasticArray2, ElasticArray2Dummy, 2); @@ -54,3 +58,16 @@ impl_elastic_array!(ElasticArray256, ElasticArray256Dummy, 256); impl_elastic_array!(ElasticArray512, ElasticArray512Dummy, 512); impl_elastic_array!(ElasticArray1024, ElasticArray1024Dummy, 1024); impl_elastic_array!(ElasticArray2048, ElasticArray2048Dummy, 2048); + + +impl MallocSizeOf for Mutex { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + (*self.lock()).size_of(ops) + } +} + +impl MallocSizeOf for RwLock { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.read().size_of(ops) + } +} From 2a0cc0834d21194e6c53bfc4d9702879f19849f4 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 7 Dec 2018 19:41:15 +0100 Subject: [PATCH 04/37] Allocators. --- mem/Cargo.toml | 2 +- mem/src/alloc.rs | 161 +++++++++++++++++++++++++++++++++++++++++------ mem/src/lib.rs | 24 +++++++ 3 files changed, 168 insertions(+), 19 deletions(-) diff --git a/mem/Cargo.toml b/mem/Cargo.toml index 70c34cfb5..f278a5c90 100644 --- a/mem/Cargo.toml +++ b/mem/Cargo.toml @@ -9,7 +9,7 @@ license = "GPL-3.0" [dependencies] clear_on_drop = "0.2" malloc_size_of_derive = { path = "./malloc_size_of_derive" } -dlmalloc = { version = "0.1", optional = true } +dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } diff --git a/mem/src/alloc.rs b/mem/src/alloc.rs index d6224c60e..982ecd559 100644 --- a/mem/src/alloc.rs +++ b/mem/src/alloc.rs @@ -17,20 +17,22 @@ //! default allocator management //! Features are: //! - windows: -//! - no features: default implementation from servo `heapsize` crate -//! - weealloc: enable, untested -//! - dlmalloc: enable, does not work, for compatibility only +//! - no features: default implementation from servo `heapsize` crate +//! - weealloc: return 0 for compatibility +//! - dlmalloc: enable, does not work, for compatibility only +//! - jemalloc: compile error //! - arch x86: -//! - no features: use default alloc -//! - jemalloc: use jemallocator crate -//! - weealloc: enable, untested -//! - dlmalloc: enable, does not work, for compatibility only +//! - no features: use default alloc +//! - jemalloc: use jemallocator crate +//! - weealloc: return 0 for compatibility +//! - dlmalloc: enable, does not work, for compatibility only //! - arch wasm32: -//! - no features: return 0 for compatibility -//! - dlmalloc: return 0 for compatibility (usable_size could be implemented if needed in -//! dlmalloc crate) -//! - weealloc: enable -//! - jemalloc: compile_error +//! - no features: return 0 for compatibility +//! - dlmalloc: return 0 for compatibility (usable_size could be implemented if needed in +//! dlmalloc crate) +//! - weealloc: return 0 for compatibility +//! - jemalloc: compile error + use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; #[cfg(feature = "conditional-mettering")] @@ -39,6 +41,9 @@ use malloc_size::MallocConditionalSizeOf; use malloc_size::MallocUnconditionalSizeOf; use std::os::raw::c_void; +#[cfg(not(feature = "weealloc-global"))] +#[cfg(not(feature = "dlmalloc-global"))] +#[cfg(not(feature = "jemalloc-global"))] #[cfg(windows)] mod usable_size { #[cfg(target_os = "windows")] @@ -49,7 +54,7 @@ mod usable_size { use std::os::raw::c_void; /// Get the size of a heap block. - pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { + pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { let heap = GetProcessHeap(); @@ -66,6 +71,9 @@ mod usable_size { } } +#[cfg(not(feature = "weealloc-global"))] +#[cfg(not(feature = "dlmalloc-global"))] +#[cfg(not(feature = "jemalloc-global"))] #[cfg(all(not(windows), not(target_arch = "wasm32")))] // default mod usable_size { @@ -83,6 +91,68 @@ mod usable_size { } +#[cfg(not(feature = "weealloc-global"))] +#[cfg(not(feature = "dlmalloc-global"))] +#[cfg(not(feature = "jemalloc-global"))] +#[cfg(target_arch = "wasm32")] +mod usable_size { + use super::*; + + /// Warning this is for compatibility only TODO dlmalloc impl + pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { + 0 + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } +} + +#[cfg(feature = "jemalloc-global")] +mod usable_size { + use super::*; + + pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { + jemallocator::usable_size(ptr) + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } +} + +#[cfg(feature = "dlmalloc-global")] +mod usable_size { + use super::*; + + /// Warning this is for compatibility only TODO dlmalloc impl + pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { + 0 + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } +} + +#[cfg(feature = "weealloc-global")] +mod usable_size { + use super::*; + + /// Warning this is for compatibility only + pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { + 0 + } + + #[inline] + pub fn new_enclosing_size_fn() -> Option { + None + } +} + /// Get a new instance of a MallocSizeOfOps pub fn new_malloc_size_ops() -> MallocSizeOfOps { MallocSizeOfOps::new( @@ -134,11 +204,11 @@ pub trait MallocSizeOfExt: MallocSizeOf { let mut ops = new_malloc_size_ops(); let mut opscond = new_count_malloc_size_ops(test_count()); let cond = ::size_of(self, &mut opscond); - let notcond = Self as MallocSizeOf>::size_of(self, &mut ops); - if cond != notcond { - println!("conditional mettering did absorb: {}", notcond - cond); - } - cond + let notcond = Self as MallocSizeOf>::size_of(self, &mut ops); + if cond != notcond { + println!("conditional mettering did absorb: {}", notcond - cond); + } + cond } } @@ -161,3 +231,58 @@ impl MallocSizeOf for std::sync::Arc { self.conditional_size_of(ops) } } + +#[test] +fn inner_vec_test() { + let vec1 = [0u8;20].to_vec(); + let vec2 = [0u8;2000].to_vec(); + let vec3:Vec = Vec::with_capacity(1000); + let vec4:Vec> = vec![vec1.clone(), vec3.clone(), vec3.clone()]; + //let vec5:Vec> = vec![vec![0,2], vec![1,2,3,54]]; + + + let mut s :usize = 0; + s = std::mem::size_of_val(&*vec1); + println!("{}",s); + s = std::mem::size_of_val(&*vec2); + println!("{}",s); + s = std::mem::size_of_val(&*vec3); + println!("{}",s); + s = std::mem::size_of_val(&*vec4); + println!("{}",s); + let mut man_count = 0; + let mut man_count2 = 0; + let mut man_count3 = 0; + for i in vec4.iter() { + man_count += std::mem::size_of_val(&*i); + man_count2 += std::mem::size_of_val(&i); + man_count3 += std::mem::size_of_val(&**i); + } + println!("man {}",man_count); + println!("man2 {}",man_count2); + println!("man3 {}",man_count3); + let s1 = vec1.m_size_of(); + println!("{}",s1); + let s2 = vec2.m_size_of(); + println!("{}",s2); + let s3 = vec3.m_size_of(); + println!("{}",s3); + let s4 = vec4.m_size_of(); + println!("{}",s); + let mut man_count = 0; + let mut man_count2 = 0; + let mut man_count3 = 0; + for i in vec4.iter() { +// man_count += unsafe { usable_size::malloc_usable_size(i.as_ptr() as *const c_void) }; + man_count += (*i).m_size_of(); + man_count2 += i.m_size_of(); + man_count3 += (&*i).m_size_of(); + } + println!("man {}",man_count); + println!("man2 {}",man_count2); + println!("man3 {}",man_count3); + + //let s = vec5.m_size_of(); + //println!("{}",s); + assert!(s4 >= s1 + s2 + s3) +} diff --git a/mem/src/lib.rs b/mem/src/lib.rs index b53999f2a..07bff6c52 100644 --- a/mem/src/lib.rs +++ b/mem/src/lib.rs @@ -29,6 +29,30 @@ use std::ptr; #[cfg(not(feature = "volatile-erase"))] pub use cod::clear::Clear; +#[cfg(feature = "jemalloc-global")] +extern crate jemallocator; + +#[cfg(feature = "dlmalloc-global")] +extern crate dlmalloc; + +#[cfg(feature = "weealloc-global")] +extern crate wee_alloc; + +#[cfg(feature = "jemalloc-global")] +#[global_allocator] +/// global allocator +pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + +#[cfg(feature = "dlmalloc-global")] +#[global_allocator] +/// global allocator +pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; + +#[cfg(feature = "weealloc-global")] +#[global_allocator] +/// global allocator +pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; + pub mod alloc; /// This is a copy of patched crate `malloc_size_of` as a module. From 418e03bd52776faa06e6d17d87990e35f1b9febe Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 10 Dec 2018 11:30:04 +0100 Subject: [PATCH 05/37] heapsize estimation for allocator without use_size fn. --- mem/Cargo.toml | 13 ++++-- mem/README.md | 2 +- mem/get_malloc_size_src.sh | 2 +- mem/slim_malloc_size_of.patch | 88 +++++++++++++++++++++++------------ mem/src/alloc.rs | 63 ++----------------------- mem/src/lib.rs | 5 +- mem/src/malloc_size.rs | 7 ++- mem/src/sizeof.rs | 62 ++++++++++++++++++++++++ 8 files changed, 146 insertions(+), 96 deletions(-) create mode 100644 mem/src/sizeof.rs diff --git a/mem/Cargo.toml b/mem/Cargo.toml index f278a5c90..63595d27c 100644 --- a/mem/Cargo.toml +++ b/mem/Cargo.toml @@ -12,22 +12,29 @@ malloc_size_of_derive = { path = "./malloc_size_of_derive" } dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } +serde = { version = "1.0", optional = true } +serde_bytes = { version = "0.10", optional = true } elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } parking_lot = { version = "*", optional = true } [features] -default = ["ethereum-impls"] +default = ["ethereum-impls", "estimate-heapsize"] # when activated mem is removed through volatile primitive instead of clear_on_drop crate volatile-erase = [] # use dlmalloc as global allocator -dlmalloc-global = ["dlmalloc"] +dlmalloc-global = ["dlmalloc", "estimate-heapsize"] # use wee_alloc as global allocator -weealloc-global = ["wee_alloc"] +weealloc-global = ["wee_alloc", "estimate-heapsize"] # use jemalloc as global allocator jemalloc-global = ["jemallocator"] # implement additional types ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] # Default malloc to conditional mettering conditional-mettering = [] +# Use serde impl from mallocsizeof +serde_only = ["serde", "serde_bytes"] +# Full estimate: no call to allocator +estimate-heapsize = ["no_ops_shallow"] +no_ops_shallow = [] diff --git a/mem/README.md b/mem/README.md index ee1e5fa2c..f24b96f74 100644 --- a/mem/README.md +++ b/mem/README.md @@ -5,8 +5,8 @@ Collection of memory related utilities. ## Features - volatile-erase : Not set by default, `Memzero` erase memory with `write_volatile`. +- estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. - conditional-metering : Try to avoid counting `Arc` twice. For test only. - Others feature are here to define global allocator, see `src/alloc.rs`. ## Dependency diff --git a/mem/get_malloc_size_src.sh b/mem/get_malloc_size_src.sh index 75df50571..4559dbeb8 100755 --- a/mem/get_malloc_size_src.sh +++ b/mem/get_malloc_size_src.sh @@ -10,4 +10,4 @@ git apply ../slim_malloc_size_of.patch cp components/malloc_size_of/lib.rs ../src/malloc_size.rs cp -r components/malloc_size_of_derive .. cd .. -rm -rf ./servo +#rm -rf ./servo diff --git a/mem/slim_malloc_size_of.patch b/mem/slim_malloc_size_of.patch index 859cc13c8..d5bf55c2e 100644 --- a/mem/slim_malloc_size_of.patch +++ b/mem/slim_malloc_size_of.patch @@ -1,5 +1,5 @@ diff --git a/components/malloc_size_of/Cargo.toml b/components/malloc_size_of/Cargo.toml -index 22840c89d8..4a2b27f827 100644 +index 22840c89d8..dbf1208ed9 100644 --- a/components/malloc_size_of/Cargo.toml +++ b/components/malloc_size_of/Cargo.toml @@ -9,6 +9,10 @@ publish = false @@ -17,6 +17,7 @@ index 22840c89d8..4a2b27f827 100644 "webrender_api", "xml5ever", ] +- +serde_only = [ + "serde", + "serde_bytes", @@ -33,7 +34,7 @@ index 22840c89d8..4a2b27f827 100644 + "thin-slice", + "void", +] - ++no_ops_shallow = [] [dependencies] -app_units = "0.7" +app_units = { version = "0.7", optional = true } @@ -70,7 +71,7 @@ index 22840c89d8..4a2b27f827 100644 [target.'cfg(target_os = "android")'.dependencies] mozjs = { version = "0.9.5", optional = true, features=["init_once"]} diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..9c52bcfcd4 100644 +index 778082b5f0..1a89015a32 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,11 +43,18 @@ @@ -126,11 +127,12 @@ index 778082b5f0..9c52bcfcd4 100644 extern crate xml5ever; -#[cfg(feature = "servo")] +-use serde_bytes::ByteBuf; +#[cfg(not(feature = "extra"))] +use std::sync as servo_arc; + +#[cfg(any(feature = "servo", feature = "serde_only"))] - use serde_bytes::ByteBuf; ++use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; use std::ops::Range; @@ -149,7 +151,23 @@ index 778082b5f0..9c52bcfcd4 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -241,6 +258,7 @@ impl MallocSizeOf for Box { +@@ -216,6 +233,7 @@ pub trait MallocConditionalShallowSizeOf { + fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; + } + ++#[cfg(not(feature = "no_ops_shallow"))] + impl MallocSizeOf for String { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } +@@ -229,6 +247,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { + } + } + ++#[cfg(not(feature = "no_ops_shallow"))] + impl MallocShallowSizeOf for Box { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(&**self) } +@@ -241,6 +260,7 @@ impl MallocSizeOf for Box { } } @@ -157,7 +175,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = 0; -@@ -253,6 +271,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { +@@ -253,6 +273,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { } } @@ -165,11 +183,12 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for thin_slice::ThinBoxedSlice { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.shallow_size_of(ops) + (**self).size_of(ops) -@@ -351,14 +370,14 @@ impl MallocSizeOf for [T] { +@@ -351,14 +372,15 @@ impl MallocSizeOf for [T] { } } -#[cfg(feature = "servo")] ++#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(any(feature = "servo", feature = "serde_only"))] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -182,7 +201,15 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for ByteBuf { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); -@@ -412,6 +431,7 @@ impl MallocSizeOf for std::collections::VecDeque { +@@ -369,6 +391,7 @@ impl MallocSizeOf for ByteBuf { + } + } + ++#[cfg(not(feature = "no_ops_shallow"))] + impl MallocShallowSizeOf for Vec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } +@@ -412,6 +435,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } @@ -190,7 +217,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for smallvec::SmallVec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if self.spilled() { -@@ -422,6 +442,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { +@@ -422,6 +446,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { } } @@ -198,7 +225,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for smallvec::SmallVec where A: smallvec::Array, -@@ -471,6 +492,7 @@ where +@@ -471,6 +496,7 @@ where } } @@ -206,7 +233,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for hashglobe::hash_set::HashSet where T: Eq + Hash, -@@ -488,6 +510,7 @@ where +@@ -488,6 +514,7 @@ where } } @@ -214,7 +241,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for hashglobe::hash_set::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -502,6 +525,7 @@ where +@@ -502,6 +529,7 @@ where } } @@ -222,7 +249,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for hashglobe::fake::HashSet where T: Eq + Hash, -@@ -513,6 +537,7 @@ where +@@ -513,6 +541,7 @@ where } } @@ -230,7 +257,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for hashglobe::fake::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -587,6 +612,7 @@ where +@@ -587,6 +616,7 @@ where } } @@ -238,7 +265,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for hashglobe::hash_map::HashMap where K: Eq + Hash, -@@ -604,6 +630,7 @@ where +@@ -604,6 +634,7 @@ where } } @@ -246,7 +273,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for hashglobe::hash_map::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -620,6 +647,7 @@ where +@@ -620,6 +651,7 @@ where } } @@ -254,7 +281,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocShallowSizeOf for hashglobe::fake::HashMap where K: Eq + Hash, -@@ -631,6 +659,7 @@ where +@@ -631,6 +663,7 @@ where } } @@ -262,7 +289,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for hashglobe::fake::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -657,9 +686,20 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,9 +690,21 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } @@ -277,6 +304,7 @@ index 778082b5f0..9c52bcfcd4 100644 + servo_arc::Arc::into_raw(sc) +} + ++#[cfg(not(feature = "no_ops_shallow"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.heap_ptr()) } @@ -284,7 +312,7 @@ index 778082b5f0..9c52bcfcd4 100644 } } -@@ -671,7 +711,7 @@ impl MallocUnconditionalSizeOf for servo_arc::Arc { +@@ -671,7 +716,7 @@ impl MallocUnconditionalSizeOf for servo_arc::Arc { impl MallocConditionalShallowSizeOf for servo_arc::Arc { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -293,7 +321,7 @@ index 778082b5f0..9c52bcfcd4 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -681,7 +721,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -681,7 +726,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { impl MallocConditionalSizeOf for servo_arc::Arc { fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -302,7 +330,7 @@ index 778082b5f0..9c52bcfcd4 100644 0 } else { self.unconditional_size_of(ops) -@@ -701,6 +741,7 @@ impl MallocSizeOf for std::sync::Mutex { +@@ -701,6 +746,7 @@ impl MallocSizeOf for std::sync::Mutex { } } @@ -310,7 +338,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for smallbitvec::SmallBitVec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if let Some(ptr) = self.heap_ptr() { -@@ -711,30 +752,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { +@@ -711,30 +757,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { } } @@ -346,7 +374,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for euclid::TypedSideOffsets2D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.top.size_of(ops) + -@@ -744,12 +790,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { +@@ -744,12 +795,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { } } @@ -361,7 +389,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for euclid::TypedTransform2D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.m11.size_of(ops) + -@@ -761,6 +809,7 @@ impl MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform3D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.m11.size_of(ops) + -@@ -782,12 +831,14 @@ impl MallocSizeOf for euclid::TypedTransform3D MallocSizeOf for euclid::TypedTransform3D usize { let selectors::parser::AncestorHashes { ref packed_hashes } = *self; -@@ -795,6 +846,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { +@@ -795,6 +851,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { } } @@ -392,7 +420,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for selectors::parser::Selector where Impl::NonTSPseudoClass: MallocSizeOf, -@@ -815,6 +867,7 @@ where +@@ -815,6 +872,7 @@ where } } @@ -400,7 +428,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for selectors::parser::Component where Impl::NonTSPseudoClass: MallocSizeOf, -@@ -860,6 +913,7 @@ where +@@ -860,6 +918,7 @@ where } } @@ -408,7 +436,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for selectors::attr::AttrSelectorWithOptionalNamespace { -@@ -868,6 +922,7 @@ impl MallocSizeOf +@@ -868,6 +927,7 @@ impl MallocSizeOf } } @@ -416,7 +444,7 @@ index 778082b5f0..9c52bcfcd4 100644 impl MallocSizeOf for Void { #[inline] fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -929,8 +984,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,8 +989,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/mem/src/alloc.rs b/mem/src/alloc.rs index 982ecd559..160819329 100644 --- a/mem/src/alloc.rs +++ b/mem/src/alloc.rs @@ -54,7 +54,7 @@ mod usable_size { use std::os::raw::c_void; /// Get the size of a heap block. - pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { + pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { let heap = GetProcessHeap(); @@ -100,7 +100,7 @@ mod usable_size { /// Warning this is for compatibility only TODO dlmalloc impl pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { - 0 + panic!("Please run with `estimate-heapsize` feature") } #[inline] @@ -129,7 +129,7 @@ mod usable_size { /// Warning this is for compatibility only TODO dlmalloc impl pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { - 0 + panic!("Running estimation this code should never be reached") } #[inline] @@ -144,7 +144,7 @@ mod usable_size { /// Warning this is for compatibility only pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { - 0 + panic!("Running estimation this code should never be reached") } #[inline] @@ -231,58 +231,3 @@ impl MallocSizeOf for std::sync::Arc { self.conditional_size_of(ops) } } - -#[test] -fn inner_vec_test() { - let vec1 = [0u8;20].to_vec(); - let vec2 = [0u8;2000].to_vec(); - let vec3:Vec = Vec::with_capacity(1000); - let vec4:Vec> = vec![vec1.clone(), vec3.clone(), vec3.clone()]; - //let vec5:Vec> = vec![vec![0,2], vec![1,2,3,54]]; - - - let mut s :usize = 0; - s = std::mem::size_of_val(&*vec1); - println!("{}",s); - s = std::mem::size_of_val(&*vec2); - println!("{}",s); - s = std::mem::size_of_val(&*vec3); - println!("{}",s); - s = std::mem::size_of_val(&*vec4); - println!("{}",s); - let mut man_count = 0; - let mut man_count2 = 0; - let mut man_count3 = 0; - for i in vec4.iter() { - man_count += std::mem::size_of_val(&*i); - man_count2 += std::mem::size_of_val(&i); - man_count3 += std::mem::size_of_val(&**i); - } - println!("man {}",man_count); - println!("man2 {}",man_count2); - println!("man3 {}",man_count3); - let s1 = vec1.m_size_of(); - println!("{}",s1); - let s2 = vec2.m_size_of(); - println!("{}",s2); - let s3 = vec3.m_size_of(); - println!("{}",s3); - let s4 = vec4.m_size_of(); - println!("{}",s); - let mut man_count = 0; - let mut man_count2 = 0; - let mut man_count3 = 0; - for i in vec4.iter() { -// man_count += unsafe { usable_size::malloc_usable_size(i.as_ptr() as *const c_void) }; - man_count += (*i).m_size_of(); - man_count2 += i.m_size_of(); - man_count3 += (&*i).m_size_of(); - } - println!("man {}",man_count); - println!("man2 {}",man_count2); - println!("man3 {}",man_count3); - - //let s = vec5.m_size_of(); - //println!("{}",s); - assert!(s4 >= s1 + s2 + s3) -} diff --git a/mem/src/lib.rs b/mem/src/lib.rs index 07bff6c52..0d3d3b4b8 100644 --- a/mem/src/lib.rs +++ b/mem/src/lib.rs @@ -55,6 +55,9 @@ pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; pub mod alloc; +#[cfg(feature = "estimate-heapsize")] +pub mod sizeof; + /// This is a copy of patched crate `malloc_size_of` as a module. /// We need to have it as an inner module to be able to define our own traits implementation, /// if at some point the trait become standard enough we could use the right way of doing it @@ -73,7 +76,7 @@ pub mod clear_on_drop { pub use malloc_size_derive::*; pub use malloc_size::{ MallocSizeOfOps, - MallocSizeOf, + MallocSizeOf, }; pub use alloc::MallocSizeOfExt; diff --git a/mem/src/malloc_size.rs b/mem/src/malloc_size.rs index 206299128..1a89015a3 100644 --- a/mem/src/malloc_size.rs +++ b/mem/src/malloc_size.rs @@ -95,7 +95,7 @@ extern crate xml5ever; use std::sync as servo_arc; #[cfg(any(feature = "servo", feature = "serde_only"))] -use serde_bytes::ByteBuf; +use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; use std::ops::Range; @@ -233,6 +233,7 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } +#[cfg(not(feature = "no_ops_shallow"))] impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -246,6 +247,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } +#[cfg(not(feature = "no_ops_shallow"))] impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } @@ -370,6 +372,7 @@ impl MallocSizeOf for [T] { } } +#[cfg(not(feature = "no_ops_shallow"))] #[cfg(any(feature = "servo", feature = "serde_only"))] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -388,6 +391,7 @@ impl MallocSizeOf for ByteBuf { } } +#[cfg(not(feature = "no_ops_shallow"))] impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -697,6 +701,7 @@ fn arc_ptr(s: &servo_arc::Arc) -> * const T { servo_arc::Arc::into_raw(sc) } +#[cfg(not(feature = "no_ops_shallow"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(arc_ptr(self)) } diff --git a/mem/src/sizeof.rs b/mem/src/sizeof.rs new file mode 100644 index 000000000..01cacb339 --- /dev/null +++ b/mem/src/sizeof.rs @@ -0,0 +1,62 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Estimation for heapsize calculation. Usable to replace call to allocator method (for some +//! allocators or simply because we just need a determinist indicator of cunsumption). + +#[cfg(feature = "serde_only")] +extern crate serde_bytes; + +use crate::malloc_size::{ + MallocSizeOf, + MallocShallowSizeOf, + MallocUnconditionalShallowSizeOf, + MallocSizeOfOps +}; +use std::mem::{size_of, size_of_val}; +#[cfg(feature = "serde_only")] +use self::serde_bytes::ByteBuf; + +impl MallocShallowSizeOf for Box { + fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of_val(&**self) + } +} + +impl MallocSizeOf for String { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + self.capacity() * size_of::() + } +} + +#[cfg(feature = "serde_only")] +impl MallocShallowSizeOf for ByteBuf { + fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + AsRef::>::as_ref(self).capacity() * size_of::() + } +} + +impl MallocShallowSizeOf for Vec { + fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + self.capacity() * size_of::() + } +} + +impl MallocUnconditionalShallowSizeOf for std::sync::Arc { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } +} From 6105606d59c6d593d742eab29374a67556523cc2 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 10 Dec 2018 12:51:01 +0100 Subject: [PATCH 06/37] Init no_std compat. --- mem/Cargo.toml | 3 +- mem/slim_malloc_size_of.patch | 138 +++++++++++++++++++++------- mem/src/{alloc.rs => allocators.rs} | 5 + mem/src/impls.rs | 5 + mem/src/lib.rs | 14 ++- mem/src/malloc_size.rs | 29 ++++++ 6 files changed, 160 insertions(+), 34 deletions(-) rename mem/src/{alloc.rs => allocators.rs} (98%) diff --git a/mem/Cargo.toml b/mem/Cargo.toml index 63595d27c..78276f0e8 100644 --- a/mem/Cargo.toml +++ b/mem/Cargo.toml @@ -20,7 +20,8 @@ ethereum-types = { version = "*", optional = true } parking_lot = { version = "*", optional = true } [features] -default = ["ethereum-impls", "estimate-heapsize"] +default = ["std", "ethereum-impls"] +std = [] # when activated mem is removed through volatile primitive instead of clear_on_drop crate volatile-erase = [] # use dlmalloc as global allocator diff --git a/mem/slim_malloc_size_of.patch b/mem/slim_malloc_size_of.patch index d5bf55c2e..f676e3deb 100644 --- a/mem/slim_malloc_size_of.patch +++ b/mem/slim_malloc_size_of.patch @@ -71,7 +71,7 @@ index 22840c89d8..dbf1208ed9 100644 [target.'cfg(target_os = "android")'.dependencies] mozjs = { version = "0.9.5", optional = true, features=["init_once"]} diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..1a89015a32 100644 +index 778082b5f0..75ba0909e8 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,11 +43,18 @@ @@ -93,7 +93,7 @@ index 778082b5f0..1a89015a32 100644 extern crate hashglobe; #[cfg(feature = "servo")] extern crate hyper; -@@ -57,41 +64,51 @@ extern crate hyper_serde; +@@ -57,41 +64,68 @@ extern crate hyper_serde; extern crate keyboard_types; #[cfg(feature = "servo")] extern crate mozjs as js; @@ -105,6 +105,7 @@ index 778082b5f0..1a89015a32 100644 -#[cfg(feature = "servo")] +#[cfg(any(feature = "servo", feature = "serde_only"))] extern crate serde_bytes; ++#[cfg(feature = "std")] +#[cfg(feature = "extra")] extern crate servo_arc; +#[cfg(feature = "extra")] @@ -128,6 +129,17 @@ index 778082b5f0..1a89015a32 100644 -#[cfg(feature = "servo")] -use serde_bytes::ByteBuf; ++#[cfg(not(feature = "std"))] ++use alloc::vec::Vec; ++#[cfg(not(feature = "std"))] ++use alloc::string::String; ++#[cfg(not(feature = "std"))] ++mod std { ++ pub use core::*; ++ pub use alloc::collections; ++} ++ ++#[cfg(feature = "std")] +#[cfg(not(feature = "extra"))] +use std::sync as servo_arc; + @@ -137,7 +149,12 @@ index 778082b5f0..1a89015a32 100644 use std::mem::size_of; use std::ops::Range; use std::ops::{Deref, DerefMut}; ++#[cfg(feature = "std")] use std::os::raw::c_void; ++#[cfg(not(feature = "std"))] ++use core::ffi::c_void; ++#[cfg(not(feature = "std"))] ++pub use alloc::boxed::Box; +#[cfg(feature = "extra")] use void::Void; @@ -151,7 +168,7 @@ index 778082b5f0..1a89015a32 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -216,6 +233,7 @@ pub trait MallocConditionalShallowSizeOf { +@@ -216,6 +250,7 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } @@ -159,7 +176,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -229,6 +247,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { +@@ -229,6 +264,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } @@ -167,7 +184,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } -@@ -241,6 +260,7 @@ impl MallocSizeOf for Box { +@@ -241,6 +277,7 @@ impl MallocSizeOf for Box { } } @@ -175,7 +192,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = 0; -@@ -253,6 +273,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { +@@ -253,6 +290,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { } } @@ -183,7 +200,15 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for thin_slice::ThinBoxedSlice { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.shallow_size_of(ops) + (**self).size_of(ops) -@@ -351,14 +372,15 @@ impl MallocSizeOf for [T] { +@@ -329,6 +367,7 @@ impl MallocSizeOf for std::cell::RefCell { + } + } + ++#[cfg(feature = "std")] + impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> + where + B::Owned: MallocSizeOf, +@@ -351,14 +390,15 @@ impl MallocSizeOf for [T] { } } @@ -201,7 +226,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for ByteBuf { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); -@@ -369,6 +391,7 @@ impl MallocSizeOf for ByteBuf { +@@ -369,6 +409,7 @@ impl MallocSizeOf for ByteBuf { } } @@ -209,7 +234,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -412,6 +435,7 @@ impl MallocSizeOf for std::collections::VecDeque { +@@ -412,6 +453,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } @@ -217,7 +242,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for smallvec::SmallVec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if self.spilled() { -@@ -422,6 +446,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { +@@ -422,6 +464,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { } } @@ -225,7 +250,23 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for smallvec::SmallVec where A: smallvec::Array, -@@ -471,6 +496,7 @@ where +@@ -436,6 +479,7 @@ where + } + } + ++#[cfg(feature = "std")] + impl MallocShallowSizeOf for std::collections::HashSet + where + T: Eq + Hash, +@@ -457,6 +501,7 @@ where + } + } + ++#[cfg(feature = "std")] + impl MallocSizeOf for std::collections::HashSet + where + T: Eq + Hash + MallocSizeOf, +@@ -471,6 +516,7 @@ where } } @@ -233,7 +274,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for hashglobe::hash_set::HashSet where T: Eq + Hash, -@@ -488,6 +514,7 @@ where +@@ -488,6 +534,7 @@ where } } @@ -241,7 +282,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for hashglobe::hash_set::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -502,6 +529,7 @@ where +@@ -502,6 +549,7 @@ where } } @@ -249,7 +290,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for hashglobe::fake::HashSet where T: Eq + Hash, -@@ -513,6 +541,7 @@ where +@@ -513,6 +561,7 @@ where } } @@ -257,7 +298,23 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for hashglobe::fake::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -587,6 +616,7 @@ where +@@ -524,6 +573,7 @@ where + } + } + ++#[cfg(feature = "std")] + impl MallocShallowSizeOf for std::collections::HashMap + where + K: Eq + Hash, +@@ -541,6 +591,7 @@ where + } + } + ++#[cfg(feature = "std")] + impl MallocSizeOf for std::collections::HashMap + where + K: Eq + Hash + MallocSizeOf, +@@ -587,6 +638,7 @@ where } } @@ -265,7 +322,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for hashglobe::hash_map::HashMap where K: Eq + Hash, -@@ -604,6 +634,7 @@ where +@@ -604,6 +656,7 @@ where } } @@ -273,7 +330,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for hashglobe::hash_map::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -620,6 +651,7 @@ where +@@ -620,6 +673,7 @@ where } } @@ -281,7 +338,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocShallowSizeOf for hashglobe::fake::HashMap where K: Eq + Hash, -@@ -631,6 +663,7 @@ where +@@ -631,6 +685,7 @@ where } } @@ -289,21 +346,24 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for hashglobe::fake::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -657,9 +690,21 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,21 +712,38 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } ++#[cfg(feature = "std")] +#[cfg(feature = "extra")] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { + s.heap_ptr() +} + ++#[cfg(feature = "std")] +#[cfg(not(feature = "extra"))] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { + let sc = s.clone(); + servo_arc::Arc::into_raw(sc) +} + ++#[cfg(feature = "std")] +#[cfg(not(feature = "no_ops_shallow"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -312,8 +372,14 @@ index 778082b5f0..1a89015a32 100644 } } -@@ -671,7 +716,7 @@ impl MallocUnconditionalSizeOf for servo_arc::Arc { ++#[cfg(feature = "std")] + impl MallocUnconditionalSizeOf for servo_arc::Arc { + fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) + } + } ++#[cfg(feature = "std")] impl MallocConditionalShallowSizeOf for servo_arc::Arc { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if ops.have_seen_ptr(self.heap_ptr()) { @@ -321,8 +387,11 @@ index 778082b5f0..1a89015a32 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -681,7 +726,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +751,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { + } + } ++#[cfg(feature = "std")] impl MallocConditionalSizeOf for servo_arc::Arc { fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if ops.have_seen_ptr(self.heap_ptr()) { @@ -330,7 +399,14 @@ index 778082b5f0..1a89015a32 100644 0 } else { self.unconditional_size_of(ops) -@@ -701,6 +746,7 @@ impl MallocSizeOf for std::sync::Mutex { +@@ -695,12 +768,14 @@ impl MallocConditionalSizeOf for servo_arc::Arc { + /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, + /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's + /// contents. ++#[cfg(feature = "std")] + impl MallocSizeOf for std::sync::Mutex { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + (*self.lock().unwrap()).size_of(ops) } } @@ -338,7 +414,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for smallbitvec::SmallBitVec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if let Some(ptr) = self.heap_ptr() { -@@ -711,30 +757,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { +@@ -711,30 +786,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { } } @@ -374,7 +450,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for euclid::TypedSideOffsets2D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.top.size_of(ops) + -@@ -744,12 +795,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { +@@ -744,12 +824,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { } } @@ -389,7 +465,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for euclid::TypedTransform2D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.m11.size_of(ops) + -@@ -761,6 +814,7 @@ impl MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform3D { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.m11.size_of(ops) + -@@ -782,12 +836,14 @@ impl MallocSizeOf for euclid::TypedTransform3D MallocSizeOf for euclid::TypedTransform3D usize { let selectors::parser::AncestorHashes { ref packed_hashes } = *self; -@@ -795,6 +851,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { +@@ -795,6 +880,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { } } @@ -420,7 +496,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for selectors::parser::Selector where Impl::NonTSPseudoClass: MallocSizeOf, -@@ -815,6 +872,7 @@ where +@@ -815,6 +901,7 @@ where } } @@ -428,7 +504,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for selectors::parser::Component where Impl::NonTSPseudoClass: MallocSizeOf, -@@ -860,6 +918,7 @@ where +@@ -860,6 +947,7 @@ where } } @@ -436,7 +512,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for selectors::attr::AttrSelectorWithOptionalNamespace { -@@ -868,6 +927,7 @@ impl MallocSizeOf +@@ -868,6 +956,7 @@ impl MallocSizeOf } } @@ -444,7 +520,7 @@ index 778082b5f0..1a89015a32 100644 impl MallocSizeOf for Void { #[inline] fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -929,8 +989,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,8 +1018,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/mem/src/alloc.rs b/mem/src/allocators.rs similarity index 98% rename from mem/src/alloc.rs rename to mem/src/allocators.rs index 160819329..d15340e46 100644 --- a/mem/src/alloc.rs +++ b/mem/src/allocators.rs @@ -39,7 +39,10 @@ use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; use malloc_size::MallocConditionalSizeOf; #[cfg(not(feature = "conditional-mettering"))] use malloc_size::MallocUnconditionalSizeOf; +#[cfg(feature = "std")] use std::os::raw::c_void; +#[cfg(not(feature = "std"))] +use core::ffi::c_void; #[cfg(not(feature = "weealloc-global"))] #[cfg(not(feature = "dlmalloc-global"))] @@ -218,6 +221,7 @@ impl MallocSizeOfExt for T {} /// we default to unconditional mettering /// It would be interesting to run some test with global mutex other weak handle in ops to check /// how much we measure multiple times +#[cfg(feature = "std")] #[cfg(not(feature = "conditional-mettering"))] impl MallocSizeOf for std::sync::Arc { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -225,6 +229,7 @@ impl MallocSizeOf for std::sync::Arc { } } +#[cfg(feature = "std")] #[cfg(feature = "conditional-mettering")] impl MallocSizeOf for std::sync::Arc { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { diff --git a/mem/src/impls.rs b/mem/src/impls.rs index e0e4497b9..13ad066df 100644 --- a/mem/src/impls.rs +++ b/mem/src/impls.rs @@ -27,7 +27,12 @@ use self::elastic_array::*; use self::parking_lot::{Mutex, RwLock}; use super::{MallocSizeOf, MallocSizeOfOps}; +#[cfg(not(feature = "std"))] +use core as std; + +#[cfg(feature = "std")] malloc_size_of_is_0!(std::time::Instant); +malloc_size_of_is_0!(std::time::Duration); malloc_size_of_is_0!( U64, U128, U256, U512, H32, H64, diff --git a/mem/src/lib.rs b/mem/src/lib.rs index 0d3d3b4b8..2129da471 100644 --- a/mem/src/lib.rs +++ b/mem/src/lib.rs @@ -16,6 +16,13 @@ //! Memory related utilities. +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "std"), feature(core_intrinsics))] +#![cfg_attr(not(feature = "std"), feature(alloc))] + +#[cfg(not(feature = "std"))] +extern crate alloc; + extern crate clear_on_drop as cod; //extern crate malloc_size_of as malloc_size; @@ -53,11 +60,14 @@ pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; /// global allocator pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; -pub mod alloc; +pub mod allocators; #[cfg(feature = "estimate-heapsize")] pub mod sizeof; +#[cfg(not(feature = "std"))] +use core as std; + /// This is a copy of patched crate `malloc_size_of` as a module. /// We need to have it as an inner module to be able to define our own traits implementation, /// if at some point the trait become standard enough we could use the right way of doing it @@ -78,7 +88,7 @@ pub use malloc_size::{ MallocSizeOfOps, MallocSizeOf, }; -pub use alloc::MallocSizeOfExt; +pub use allocators::MallocSizeOfExt; /// Wrapper to zero out memory when dropped. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/mem/src/malloc_size.rs b/mem/src/malloc_size.rs index 1a89015a3..75ba0909e 100644 --- a/mem/src/malloc_size.rs +++ b/mem/src/malloc_size.rs @@ -70,6 +70,7 @@ extern crate selectors; extern crate serde; #[cfg(any(feature = "servo", feature = "serde_only"))] extern crate serde_bytes; +#[cfg(feature = "std")] #[cfg(feature = "extra")] extern crate servo_arc; #[cfg(feature = "extra")] @@ -91,6 +92,17 @@ extern crate webrender_api; #[cfg(feature = "servo")] extern crate xml5ever; +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; +#[cfg(not(feature = "std"))] +use alloc::string::String; +#[cfg(not(feature = "std"))] +mod std { + pub use core::*; + pub use alloc::collections; +} + +#[cfg(feature = "std")] #[cfg(not(feature = "extra"))] use std::sync as servo_arc; @@ -100,7 +112,12 @@ use std::hash::{BuildHasher, Hash}; use std::mem::size_of; use std::ops::Range; use std::ops::{Deref, DerefMut}; +#[cfg(feature = "std")] use std::os::raw::c_void; +#[cfg(not(feature = "std"))] +use core::ffi::c_void; +#[cfg(not(feature = "std"))] +pub use alloc::boxed::Box; #[cfg(feature = "extra")] use void::Void; @@ -350,6 +367,7 @@ impl MallocSizeOf for std::cell::RefCell { } } +#[cfg(feature = "std")] impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> where B::Owned: MallocSizeOf, @@ -461,6 +479,7 @@ where } } +#[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashSet where T: Eq + Hash, @@ -482,6 +501,7 @@ where } } +#[cfg(feature = "std")] impl MallocSizeOf for std::collections::HashSet where T: Eq + Hash + MallocSizeOf, @@ -553,6 +573,7 @@ where } } +#[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashMap where K: Eq + Hash, @@ -570,6 +591,7 @@ where } } +#[cfg(feature = "std")] impl MallocSizeOf for std::collections::HashMap where K: Eq + Hash + MallocSizeOf, @@ -690,17 +712,20 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } +#[cfg(feature = "std")] #[cfg(feature = "extra")] fn arc_ptr(s: &servo_arc::Arc) -> * const T { s.heap_ptr() } +#[cfg(feature = "std")] #[cfg(not(feature = "extra"))] fn arc_ptr(s: &servo_arc::Arc) -> * const T { let sc = s.clone(); servo_arc::Arc::into_raw(sc) } +#[cfg(feature = "std")] #[cfg(not(feature = "no_ops_shallow"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -708,12 +733,14 @@ impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { } } +#[cfg(feature = "std")] impl MallocUnconditionalSizeOf for servo_arc::Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) } } +#[cfg(feature = "std")] impl MallocConditionalShallowSizeOf for servo_arc::Arc { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if ops.have_seen_ptr(arc_ptr(self)) { @@ -724,6 +751,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } +#[cfg(feature = "std")] impl MallocConditionalSizeOf for servo_arc::Arc { fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if ops.have_seen_ptr(arc_ptr(self)) { @@ -740,6 +768,7 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. +#[cfg(feature = "std")] impl MallocSizeOf for std::sync::Mutex { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { (*self.lock().unwrap()).size_of(ops) From 59b4d551aa757960a56fa9c44d1336c428558feb Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 10 Dec 2018 18:07:28 +0100 Subject: [PATCH 07/37] Remove conditianal test code. Fix a no_std deps issue. --- mem/src/allocators.rs | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/mem/src/allocators.rs b/mem/src/allocators.rs index d15340e46..8bc9ae76e 100644 --- a/mem/src/allocators.rs +++ b/mem/src/allocators.rs @@ -36,14 +36,19 @@ use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; #[cfg(feature = "conditional-mettering")] -use malloc_size::MallocConditionalSizeOf; +use malloc_size::{MallocConditionalSizeOf, VoidPtrToBoolFnMut}; +#[cfg(feature = "std")] #[cfg(not(feature = "conditional-mettering"))] use malloc_size::MallocUnconditionalSizeOf; #[cfg(feature = "std")] use std::os::raw::c_void; #[cfg(not(feature = "std"))] use core::ffi::c_void; - +#[cfg(not(feature = "std"))] +use alloc::collections::btree_set::BTreeSet; +#[cfg(not(feature = "std"))] +#[cfg(feature = "conditional-mettering")] +pub use alloc::boxed::Box; #[cfg(not(feature = "weealloc-global"))] #[cfg(not(feature = "dlmalloc-global"))] #[cfg(not(feature = "jemalloc-global"))] @@ -171,14 +176,27 @@ pub fn new_count_malloc_size_ops(count_fn: Box) -> MallocSiz MallocSizeOfOps::new( usable_size::malloc_usable_size, usable_size::new_enclosing_size_fn(), - count_fn, + Some(count_fn), ) } +#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "std")] +fn new_set() -> std::collections::HashSet { + std::collections::HashSet::new() +} + +#[cfg(feature = "conditional-mettering")] +#[cfg(not(feature = "std"))] +fn new_set() -> BTreeSet { + BTreeSet::new() +} + + #[cfg(feature = "conditional-mettering")] /// count function for testing purpose only (slow) pub fn test_count() -> Box bool> { - let mut set = std::collections::HashSet::new(); + let mut set = new_set(); Box::new(move |ptr| { let r = if set.contains(&ptr) { true @@ -190,7 +208,6 @@ pub fn test_count() -> Box bool> { }) } -#[cfg(not(feature = "conditional-mettering"))] /// Extension methods for `MallocSizeOf` pub trait MallocSizeOfExt: MallocSizeOf { fn m_size_of(&self) -> usize { @@ -199,22 +216,6 @@ pub trait MallocSizeOfExt: MallocSizeOf { } } -// TODO remove this implementation (conditional metering on purpose only) -#[cfg(feature = "conditional-mettering")] -/// Extension methods for `MallocSizeOf` -pub trait MallocSizeOfExt: MallocSizeOf { - fn m_size_of(&self) -> usize { - let mut ops = new_malloc_size_ops(); - let mut opscond = new_count_malloc_size_ops(test_count()); - let cond = ::size_of(self, &mut opscond); - let notcond = Self as MallocSizeOf>::size_of(self, &mut ops); - if cond != notcond { - println!("conditional mettering did absorb: {}", notcond - cond); - } - cond - } -} - impl MallocSizeOfExt for T {} /// we currently do not have use case where a conditional fn is use so From efadc31e0aafe66c41cfdc7461b0a05c89a5e66f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 11 Dec 2018 17:04:20 +0100 Subject: [PATCH 08/37] Renaming of mem repo to crate name. Documentation enhancements. Explicits imports. --- Cargo.toml | 2 +- mem/README.md | 14 ----- {mem => parity-util-mem}/Cargo.toml | 7 +-- parity-util-mem/README.md | 17 ++++++ .../get_malloc_size_src.sh | 0 .../malloc_size_of_derive/Cargo.toml | 0 .../malloc_size_of_derive/LICENSE-APACHE | 0 .../malloc_size_of_derive/LICENSE-MIT | 0 .../malloc_size_of_derive/lib.rs | 0 .../slim_malloc_size_of.patch | 0 {mem => parity-util-mem}/src/allocators.rs | 52 +++++++++++++------ {mem => parity-util-mem}/src/impls.rs | 28 ++++++++-- {mem => parity-util-mem}/src/lib.rs | 17 +++--- {mem => parity-util-mem}/src/malloc_size.rs | 0 {mem => parity-util-mem}/src/sizeof.rs | 0 15 files changed, 91 insertions(+), 46 deletions(-) delete mode 100644 mem/README.md rename {mem => parity-util-mem}/Cargo.toml (89%) create mode 100644 parity-util-mem/README.md rename {mem => parity-util-mem}/get_malloc_size_src.sh (100%) rename {mem => parity-util-mem}/malloc_size_of_derive/Cargo.toml (100%) rename {mem => parity-util-mem}/malloc_size_of_derive/LICENSE-APACHE (100%) rename {mem => parity-util-mem}/malloc_size_of_derive/LICENSE-MIT (100%) rename {mem => parity-util-mem}/malloc_size_of_derive/lib.rs (100%) rename {mem => parity-util-mem}/slim_malloc_size_of.patch (100%) rename {mem => parity-util-mem}/src/allocators.rs (77%) rename {mem => parity-util-mem}/src/impls.rs (82%) rename {mem => parity-util-mem}/src/lib.rs (88%) rename {mem => parity-util-mem}/src/malloc_size.rs (100%) rename {mem => parity-util-mem}/src/sizeof.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 4d6812658..9d7e8dbe0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,5 @@ members = [ "trie-standardmap", "triehash", "uint", - "mem", + "parity-util-mem", ] diff --git a/mem/README.md b/mem/README.md deleted file mode 100644 index f24b96f74..000000000 --- a/mem/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# parity-util-mem - -Collection of memory related utilities. - -## Features - -- volatile-erase : Not set by default, `Memzero` erase memory with `write_volatile`. -- estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. -- conditional-metering : Try to avoid counting `Arc` twice. For test only. -Others feature are here to define global allocator, see `src/alloc.rs`. - -## Dependency - -This crate groups common dependency, `clear_on_drop` is reexported, and a patched copy of unpublished `malloc_size_of` from servo project is used and partially reexported. diff --git a/mem/Cargo.toml b/parity-util-mem/Cargo.toml similarity index 89% rename from mem/Cargo.toml rename to parity-util-mem/Cargo.toml index 78276f0e8..eb888f400 100644 --- a/mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -32,9 +32,10 @@ weealloc-global = ["wee_alloc", "estimate-heapsize"] jemalloc-global = ["jemallocator"] # implement additional types ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] -# Default malloc to conditional mettering -conditional-mettering = [] -# Use serde impl from mallocsizeof +# Include conditional metering +# (keep state to avoid multiple measurement of counted references). +conditional-metering = [] +# Include serde related struct metering. serde_only = ["serde", "serde_bytes"] # Full estimate: no call to allocator estimate-heapsize = ["no_ops_shallow"] diff --git a/parity-util-mem/README.md b/parity-util-mem/README.md new file mode 100644 index 000000000..be6b961b4 --- /dev/null +++ b/parity-util-mem/README.md @@ -0,0 +1,17 @@ +# parity-util-mem + +Collection of memory related utilities. + +## Features + +- volatile-erase : Not set by default, `Memzero` struct will be erasing memory throug a simple [`write_volatile`](https://doc.rust-lang.org/std/ptr/fn.write_volatile.html) call. +- estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. +- conditional-metering : Try to avoid counting `Arc` twice. For test only. + +Others features define global allocator, see `src/alloc.rs`. + +## Dependency + +This crate groups common dependency, [`clear_on_drop`](https://crates.io/crates/clear_on_drop) is reexported, and a patched copy of unpublished [`malloc_size_of`](https://github.com/servo/servo/tree/master/components/malloc_size_of) from servo project is copied and partially reexported. + +`Malloc_size_of` code is used internally as a module with a few modification to be able to implement type locally. diff --git a/mem/get_malloc_size_src.sh b/parity-util-mem/get_malloc_size_src.sh similarity index 100% rename from mem/get_malloc_size_src.sh rename to parity-util-mem/get_malloc_size_src.sh diff --git a/mem/malloc_size_of_derive/Cargo.toml b/parity-util-mem/malloc_size_of_derive/Cargo.toml similarity index 100% rename from mem/malloc_size_of_derive/Cargo.toml rename to parity-util-mem/malloc_size_of_derive/Cargo.toml diff --git a/mem/malloc_size_of_derive/LICENSE-APACHE b/parity-util-mem/malloc_size_of_derive/LICENSE-APACHE similarity index 100% rename from mem/malloc_size_of_derive/LICENSE-APACHE rename to parity-util-mem/malloc_size_of_derive/LICENSE-APACHE diff --git a/mem/malloc_size_of_derive/LICENSE-MIT b/parity-util-mem/malloc_size_of_derive/LICENSE-MIT similarity index 100% rename from mem/malloc_size_of_derive/LICENSE-MIT rename to parity-util-mem/malloc_size_of_derive/LICENSE-MIT diff --git a/mem/malloc_size_of_derive/lib.rs b/parity-util-mem/malloc_size_of_derive/lib.rs similarity index 100% rename from mem/malloc_size_of_derive/lib.rs rename to parity-util-mem/malloc_size_of_derive/lib.rs diff --git a/mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch similarity index 100% rename from mem/slim_malloc_size_of.patch rename to parity-util-mem/slim_malloc_size_of.patch diff --git a/mem/src/allocators.rs b/parity-util-mem/src/allocators.rs similarity index 77% rename from mem/src/allocators.rs rename to parity-util-mem/src/allocators.rs index 8bc9ae76e..043691e8a 100644 --- a/mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -35,10 +35,10 @@ use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] use malloc_size::{MallocConditionalSizeOf, VoidPtrToBoolFnMut}; #[cfg(feature = "std")] -#[cfg(not(feature = "conditional-mettering"))] +#[cfg(not(feature = "conditional-metering"))] use malloc_size::MallocUnconditionalSizeOf; #[cfg(feature = "std")] use std::os::raw::c_void; @@ -47,7 +47,7 @@ use core::ffi::c_void; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet; #[cfg(not(feature = "std"))] -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] pub use alloc::boxed::Box; #[cfg(not(feature = "weealloc-global"))] #[cfg(not(feature = "dlmalloc-global"))] @@ -62,6 +62,7 @@ mod usable_size { use std::os::raw::c_void; /// Get the size of a heap block. + /// Call windows allocator through `winapi` crate pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { let heap = GetProcessHeap(); @@ -73,6 +74,7 @@ mod usable_size { HeapSize(heap, 0, ptr) as usize } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -87,11 +89,15 @@ mod usable_size { mod usable_size { use super::*; + /// Default allocators for different platforms. + /// Macos, ios and android calls jemalloc. + /// Linux call system allocator (currently malloc). extern "C" { #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] pub fn malloc_usable_size(ptr: *const c_void) -> usize; } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -106,11 +112,14 @@ mod usable_size { mod usable_size { use super::*; - /// Warning this is for compatibility only TODO dlmalloc impl + /// Warning this is for compatibility only. + /// This function does panic: `estimate-heapsize` feature needs to be activated + /// to avoid this function call. pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { panic!("Please run with `estimate-heapsize` feature") } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -121,10 +130,12 @@ mod usable_size { mod usable_size { use super::*; + /// Use of jemalloc usable size C function through jemallocator crate call. pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { jemallocator::usable_size(ptr) } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -135,11 +146,14 @@ mod usable_size { mod usable_size { use super::*; - /// Warning this is for compatibility only TODO dlmalloc impl + /// Warning this is for compatibility only. + /// Feature: `estimate-heapsize` is on with `dlmalloc-global` and this code + /// should never be called (it panics otherwhise). pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { panic!("Running estimation this code should never be reached") } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -150,11 +164,14 @@ mod usable_size { mod usable_size { use super::*; - /// Warning this is for compatibility only + /// Warning this is for compatibility only. + /// Feature: `estimate-heapsize` is on with `weealloc-global` and this code + /// should never be called (it panics otherwhise). pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { panic!("Running estimation this code should never be reached") } + /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { None @@ -170,7 +187,7 @@ pub fn new_malloc_size_ops() -> MallocSizeOfOps { ) } -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] /// Get a new instance of a MallocSizeOfOps with a haveseen ptr function pub fn new_count_malloc_size_ops(count_fn: Box) -> MallocSizeOfOps { MallocSizeOfOps::new( @@ -180,20 +197,20 @@ pub fn new_count_malloc_size_ops(count_fn: Box) -> MallocSiz ) } -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] #[cfg(feature = "std")] fn new_set() -> std::collections::HashSet { std::collections::HashSet::new() } -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] #[cfg(not(feature = "std"))] fn new_set() -> BTreeSet { BTreeSet::new() } -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] /// count function for testing purpose only (slow) pub fn test_count() -> Box bool> { let mut set = new_set(); @@ -208,9 +225,14 @@ pub fn test_count() -> Box bool> { }) } -/// Extension methods for `MallocSizeOf` +/// Extension methods for `MallocSizeOf` trait, do not implement +/// directly. +/// It allows getting heapsize without exposing `MallocSizeOfOps` +/// (a single default `MallocSizeOfOps` is used for each call). pub trait MallocSizeOfExt: MallocSizeOf { - fn m_size_of(&self) -> usize { + /// Method to launch a heapsize measurement with a + /// fresh state. + fn malloc_size_of(&self) -> usize { let mut ops = new_malloc_size_ops(); ::size_of(self, &mut ops) } @@ -219,11 +241,11 @@ pub trait MallocSizeOfExt: MallocSizeOf { impl MallocSizeOfExt for T {} /// we currently do not have use case where a conditional fn is use so -/// we default to unconditional mettering +/// we default to unconditional metering /// It would be interesting to run some test with global mutex other weak handle in ops to check /// how much we measure multiple times #[cfg(feature = "std")] -#[cfg(not(feature = "conditional-mettering"))] +#[cfg(not(feature = "conditional-metering"))] impl MallocSizeOf for std::sync::Arc { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_size_of(ops) @@ -231,7 +253,7 @@ impl MallocSizeOf for std::sync::Arc { } #[cfg(feature = "std")] -#[cfg(feature = "conditional-mettering")] +#[cfg(feature = "conditional-metering")] impl MallocSizeOf for std::sync::Arc { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.conditional_size_of(ops) diff --git a/mem/src/impls.rs b/parity-util-mem/src/impls.rs similarity index 82% rename from mem/src/impls.rs rename to parity-util-mem/src/impls.rs index 13ad066df..d682d2e42 100644 --- a/mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -14,16 +14,34 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Implementation of common types. -//! Note that the implementation (there is some macro call redundancy here) could be done in the types crate, -//! but it creates some hard to maintain/update trait dependencies. +//! Implementation of `MallocSize` for common types : +//! - etheureum types uint and fixed hash. +//! - elastic_array arrays +//! - parking_lot mutex structures extern crate elastic_array; extern crate ethereum_types; extern crate parking_lot; -use self::ethereum_types::*; -use self::elastic_array::*; +use self::ethereum_types::{ + U64, U128, U256, U512, H32, H64, + H128, H160, H256, H264, H512, H520, + Bloom +}; +use self::elastic_array::{ + ElasticArray2, + ElasticArray4, + ElasticArray8, + ElasticArray16, + ElasticArray32, + ElasticArray36, + ElasticArray64, + ElasticArray128, + ElasticArray256, + ElasticArray512, + ElasticArray1024, + ElasticArray2048, +}; use self::parking_lot::{Mutex, RwLock}; use super::{MallocSizeOf, MallocSizeOfOps}; diff --git a/mem/src/lib.rs b/parity-util-mem/src/lib.rs similarity index 88% rename from mem/src/lib.rs rename to parity-util-mem/src/lib.rs index 2129da471..8454acd4b 100644 --- a/mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -14,7 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Memory related utilities. +//! Crate for parity memory management related utilities. +//! It includes global allocator choice, heap measurement and +//! memory erasure. #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(not(feature = "std"), feature(core_intrinsics))] @@ -25,7 +27,6 @@ extern crate alloc; extern crate clear_on_drop as cod; -//extern crate malloc_size_of as malloc_size; #[macro_use] extern crate malloc_size_of_derive as malloc_size_derive; use std::ops::{Deref, DerefMut}; @@ -47,17 +48,17 @@ extern crate wee_alloc; #[cfg(feature = "jemalloc-global")] #[global_allocator] -/// global allocator +/// Global allocator pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; #[cfg(feature = "dlmalloc-global")] #[global_allocator] -/// global allocator +/// Global allocator pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; #[cfg(feature = "weealloc-global")] #[global_allocator] -/// global allocator +/// Global allocator pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; pub mod allocators; @@ -71,14 +72,14 @@ use core as std; /// This is a copy of patched crate `malloc_size_of` as a module. /// We need to have it as an inner module to be able to define our own traits implementation, /// if at some point the trait become standard enough we could use the right way of doing it -/// by implementing it in our type traits crates. At this time a move on this trait if implemented -/// at primitive types level would impact to much of the dependency to be easilly manageable. +/// by implementing it in our type traits crates. At this time moving this trait to the primitive +/// types level would impact too much of the dependencies to be easily manageable. #[macro_use] mod malloc_size; #[cfg(feature = "ethereum-impls")] pub mod impls; -/// reexport clear_on_drop crate +/// Reexport clear_on_drop crate. pub mod clear_on_drop { pub use cod::*; } diff --git a/mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs similarity index 100% rename from mem/src/malloc_size.rs rename to parity-util-mem/src/malloc_size.rs diff --git a/mem/src/sizeof.rs b/parity-util-mem/src/sizeof.rs similarity index 100% rename from mem/src/sizeof.rs rename to parity-util-mem/src/sizeof.rs From b2f170876104b62a6cf5255429625e416e4bf70a Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 11 Dec 2018 17:09:10 +0100 Subject: [PATCH 09/37] Remove conditional mettering for now (will need some testing to check if of any use). --- parity-util-mem/Cargo.toml | 3 -- parity-util-mem/README.md | 1 - parity-util-mem/src/allocators.rs | 57 ------------------------------- 3 files changed, 61 deletions(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index eb888f400..0771911bf 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -32,9 +32,6 @@ weealloc-global = ["wee_alloc", "estimate-heapsize"] jemalloc-global = ["jemallocator"] # implement additional types ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] -# Include conditional metering -# (keep state to avoid multiple measurement of counted references). -conditional-metering = [] # Include serde related struct metering. serde_only = ["serde", "serde_bytes"] # Full estimate: no call to allocator diff --git a/parity-util-mem/README.md b/parity-util-mem/README.md index be6b961b4..305f24ab7 100644 --- a/parity-util-mem/README.md +++ b/parity-util-mem/README.md @@ -6,7 +6,6 @@ Collection of memory related utilities. - volatile-erase : Not set by default, `Memzero` struct will be erasing memory throug a simple [`write_volatile`](https://doc.rust-lang.org/std/ptr/fn.write_volatile.html) call. - estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. -- conditional-metering : Try to avoid counting `Arc` twice. For test only. Others features define global allocator, see `src/alloc.rs`. diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 043691e8a..027c0bdb2 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -35,10 +35,7 @@ use malloc_size::{MallocSizeOfOps, VoidPtrToSizeFn, MallocSizeOf}; -#[cfg(feature = "conditional-metering")] -use malloc_size::{MallocConditionalSizeOf, VoidPtrToBoolFnMut}; #[cfg(feature = "std")] -#[cfg(not(feature = "conditional-metering"))] use malloc_size::MallocUnconditionalSizeOf; #[cfg(feature = "std")] use std::os::raw::c_void; @@ -46,9 +43,6 @@ use std::os::raw::c_void; use core::ffi::c_void; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet; -#[cfg(not(feature = "std"))] -#[cfg(feature = "conditional-metering")] -pub use alloc::boxed::Box; #[cfg(not(feature = "weealloc-global"))] #[cfg(not(feature = "dlmalloc-global"))] #[cfg(not(feature = "jemalloc-global"))] @@ -187,44 +181,6 @@ pub fn new_malloc_size_ops() -> MallocSizeOfOps { ) } -#[cfg(feature = "conditional-metering")] -/// Get a new instance of a MallocSizeOfOps with a haveseen ptr function -pub fn new_count_malloc_size_ops(count_fn: Box) -> MallocSizeOfOps { - MallocSizeOfOps::new( - usable_size::malloc_usable_size, - usable_size::new_enclosing_size_fn(), - Some(count_fn), - ) -} - -#[cfg(feature = "conditional-metering")] -#[cfg(feature = "std")] -fn new_set() -> std::collections::HashSet { - std::collections::HashSet::new() -} - -#[cfg(feature = "conditional-metering")] -#[cfg(not(feature = "std"))] -fn new_set() -> BTreeSet { - BTreeSet::new() -} - - -#[cfg(feature = "conditional-metering")] -/// count function for testing purpose only (slow) -pub fn test_count() -> Box bool> { - let mut set = new_set(); - Box::new(move |ptr| { - let r = if set.contains(&ptr) { - true - } else { - set.insert(ptr); - false - }; - r - }) -} - /// Extension methods for `MallocSizeOf` trait, do not implement /// directly. /// It allows getting heapsize without exposing `MallocSizeOfOps` @@ -240,22 +196,9 @@ pub trait MallocSizeOfExt: MallocSizeOf { impl MallocSizeOfExt for T {} -/// we currently do not have use case where a conditional fn is use so -/// we default to unconditional metering -/// It would be interesting to run some test with global mutex other weak handle in ops to check -/// how much we measure multiple times #[cfg(feature = "std")] -#[cfg(not(feature = "conditional-metering"))] impl MallocSizeOf for std::sync::Arc { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_size_of(ops) } } - -#[cfg(feature = "std")] -#[cfg(feature = "conditional-metering")] -impl MallocSizeOf for std::sync::Arc { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.conditional_size_of(ops) - } -} From fb127a7ecc9a15831dddebf32d76f1a8a96c2216 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 11 Dec 2018 17:13:36 +0100 Subject: [PATCH 10/37] tab --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9d7e8dbe0..8ca00c744 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,5 @@ members = [ "trie-standardmap", "triehash", "uint", - "parity-util-mem", + "parity-util-mem", ] From d5aa8991aeb396da1d922d18df8b952632383366 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 2 Jan 2019 19:41:00 +0100 Subject: [PATCH 11/37] Remove features get form malloc_size and simply delete code instead. --- parity-util-mem/slim_malloc_size_of.patch | 883 +++++++++++++--------- parity-util-mem/src/malloc_size.rs | 543 +------------ 2 files changed, 532 insertions(+), 894 deletions(-) diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index f676e3deb..3e7abf669 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,134 +1,54 @@ -diff --git a/components/malloc_size_of/Cargo.toml b/components/malloc_size_of/Cargo.toml -index 22840c89d8..dbf1208ed9 100644 ---- a/components/malloc_size_of/Cargo.toml -+++ b/components/malloc_size_of/Cargo.toml -@@ -9,6 +9,10 @@ publish = false - path = "lib.rs" - - [features] -+default = [ -+ "extra", -+ "serde_only", -+] - servo = [ - "crossbeam-channel", - "hyper", -@@ -23,29 +27,45 @@ servo = [ - "webrender_api", - "xml5ever", - ] -- -+serde_only = [ -+ "serde", -+ "serde_bytes", -+] -+extra = [ -+ "app_units", -+ "cssparser", -+ "euclid", -+ "hashglobe", -+ "selectors", -+ "servo_arc", -+ "smallbitvec", -+ "smallvec", -+ "thin-slice", -+ "void", -+] -+no_ops_shallow = [] - [dependencies] --app_units = "0.7" -+app_units = { version = "0.7", optional = true } - crossbeam-channel = { version = "0.3", optional = true } --cssparser = "0.25" --euclid = "0.19" --hashglobe = { path = "../hashglobe" } -+cssparser = { version = "0.25", optional = true } -+euclid = { version = "0.19", optional = true } -+hashglobe = { path = "../hashglobe", optional = true } - hyper = { version = "0.12", optional = true } - hyper_serde = { version = "0.9", optional = true } - keyboard-types = {version = "0.4.3", optional = true} --selectors = { path = "../selectors" } -+selectors = { path = "../selectors", optional = true } - serde = { version = "1.0.27", optional = true } - serde_bytes = { version = "0.10", optional = true } --servo_arc = { path = "../servo_arc" } --smallbitvec = "2.1.0" --smallvec = "0.6" -+servo_arc = { path = "../servo_arc", optional = true } -+smallbitvec = { version = "2.1.0", optional = true } -+smallvec = { version = "0.6", optional = true } - string_cache = { version = "0.7", optional = true } --thin-slice = "0.1.0" -+thin-slice = { version = "0.1.0", optional = true } - time = { version = "0.1.17", optional = true } - url = { version = "1.2", optional = true } - webrender_api = { git = "https://github.com/servo/webrender", features = ["ipc"], optional = true } - xml5ever = { version = "0.12", optional = true } --void = "1.0.2" -+void = { version = "1.0.2", optional = true } - - [target.'cfg(target_os = "android")'.dependencies] - mozjs = { version = "0.9.5", optional = true, features=["init_once"]} diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..75ba0909e8 100644 +index 778082b5f0..f1cc08b3e4 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs -@@ -43,11 +43,18 @@ +@@ -43,55 +43,45 @@ //! measured as well as the thing it points to. E.g. //! ` as MallocSizeOf>::size_of(field, ops)`. +-extern crate app_units; +-#[cfg(feature = "servo")] +-extern crate crossbeam_channel; +-extern crate cssparser; +-extern crate euclid; +-extern crate hashglobe; +-#[cfg(feature = "servo")] +-extern crate hyper; +-#[cfg(feature = "servo")] +-extern crate hyper_serde; +-#[cfg(feature = "servo")] +-extern crate keyboard_types; +-#[cfg(feature = "servo")] +-extern crate mozjs as js; +-extern crate selectors; +-#[cfg(feature = "servo")] + +// This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. + -+#[cfg(feature = "extra")] - extern crate app_units; - #[cfg(feature = "servo")] - extern crate crossbeam_channel; -+#[cfg(feature = "extra")] - extern crate cssparser; -+#[cfg(feature = "extra")] - extern crate euclid; -+#[cfg(feature = "extra")] - extern crate hashglobe; - #[cfg(feature = "servo")] - extern crate hyper; -@@ -57,41 +64,68 @@ extern crate hyper_serde; - extern crate keyboard_types; - #[cfg(feature = "servo")] - extern crate mozjs as js; -+#[cfg(feature = "extra")] - extern crate selectors; --#[cfg(feature = "servo")] -+#[cfg(any(feature = "servo", feature = "serde_only"))] ++#[cfg(feature = "serde_only")] extern crate serde; -#[cfg(feature = "servo")] -+#[cfg(any(feature = "servo", feature = "serde_only"))] ++#[cfg(feature = "serde_only")] extern crate serde_bytes; -+#[cfg(feature = "std")] -+#[cfg(feature = "extra")] - extern crate servo_arc; -+#[cfg(feature = "extra")] - extern crate smallbitvec; -+#[cfg(feature = "extra")] - extern crate smallvec; - #[cfg(feature = "servo")] - extern crate string_cache; -+#[cfg(feature = "extra")] - extern crate thin_slice; - #[cfg(feature = "servo")] - extern crate time; - #[cfg(feature = "url")] - extern crate url; -+#[cfg(feature = "extra")] - extern crate void; - #[cfg(feature = "webrender_api")] - extern crate webrender_api; - #[cfg(feature = "servo")] - extern crate xml5ever; - +-extern crate servo_arc; +-extern crate smallbitvec; +-extern crate smallvec; +-#[cfg(feature = "servo")] +-extern crate string_cache; +-extern crate thin_slice; +-#[cfg(feature = "servo")] +-extern crate time; +-#[cfg(feature = "url")] +-extern crate url; +-extern crate void; +-#[cfg(feature = "webrender_api")] +-extern crate webrender_api; +-#[cfg(feature = "servo")] +-extern crate xml5ever; +- -#[cfg(feature = "servo")] -use serde_bytes::ByteBuf; ++ +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; +#[cfg(not(feature = "std"))] @@ -140,10 +60,9 @@ index 778082b5f0..75ba0909e8 100644 +} + +#[cfg(feature = "std")] -+#[cfg(not(feature = "extra"))] +use std::sync as servo_arc; + -+#[cfg(any(feature = "servo", feature = "serde_only"))] ++#[cfg(feature = "serde_only")] +use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; @@ -151,12 +70,11 @@ index 778082b5f0..75ba0909e8 100644 use std::ops::{Deref, DerefMut}; +#[cfg(feature = "std")] use std::os::raw::c_void; +-use void::Void; +#[cfg(not(feature = "std"))] +use core::ffi::c_void; +#[cfg(not(feature = "std"))] +pub use alloc::boxed::Box; -+#[cfg(feature = "extra")] - use void::Void; /// A C function that takes a pointer to a heap allocation and returns its size. -type VoidPtrToSizeFn = unsafe extern "C" fn(ptr: *const c_void) -> usize; @@ -168,7 +86,7 @@ index 778082b5f0..75ba0909e8 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -216,6 +250,7 @@ pub trait MallocConditionalShallowSizeOf { +@@ -216,6 +206,7 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } @@ -176,7 +94,7 @@ index 778082b5f0..75ba0909e8 100644 impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -229,6 +264,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { +@@ -229,6 +220,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } @@ -184,23 +102,32 @@ index 778082b5f0..75ba0909e8 100644 impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } -@@ -241,6 +277,7 @@ impl MallocSizeOf for Box { - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = 0; -@@ -253,6 +290,7 @@ impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { +@@ -241,24 +233,6 @@ impl MallocSizeOf for Box { } } -+#[cfg(feature = "extra")] - impl MallocSizeOf for thin_slice::ThinBoxedSlice { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.shallow_size_of(ops) + (**self).size_of(ops) -@@ -329,6 +367,7 @@ impl MallocSizeOf for std::cell::RefCell { +-impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = 0; +- unsafe { +- n += thin_slice::ThinBoxedSlice::spilled_storage(self) +- .map_or(0, |ptr| ops.malloc_size_of(ptr)); +- n += ops.malloc_size_of(&**self); +- } +- n +- } +-} +- +-impl MallocSizeOf for thin_slice::ThinBoxedSlice { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.shallow_size_of(ops) + (**self).size_of(ops) +- } +-} +- + impl MallocSizeOf for () { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + 0 +@@ -329,6 +303,7 @@ impl MallocSizeOf for std::cell::RefCell { } } @@ -208,13 +135,13 @@ index 778082b5f0..75ba0909e8 100644 impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> where B::Owned: MallocSizeOf, -@@ -351,14 +390,15 @@ impl MallocSizeOf for [T] { +@@ -351,14 +326,15 @@ impl MallocSizeOf for [T] { } } -#[cfg(feature = "servo")] +#[cfg(not(feature = "no_ops_shallow"))] -+#[cfg(any(feature = "servo", feature = "serde_only"))] ++#[cfg(feature = "serde_only")] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -222,11 +149,11 @@ index 778082b5f0..75ba0909e8 100644 } -#[cfg(feature = "servo")] -+#[cfg(any(feature = "servo", feature = "serde_only"))] ++#[cfg(feature = "serde_only")] impl MallocSizeOf for ByteBuf { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); -@@ -369,6 +409,7 @@ impl MallocSizeOf for ByteBuf { +@@ -369,6 +345,7 @@ impl MallocSizeOf for ByteBuf { } } @@ -234,31 +161,39 @@ index 778082b5f0..75ba0909e8 100644 impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -412,6 +453,7 @@ impl MallocSizeOf for std::collections::VecDeque { - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for smallvec::SmallVec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.spilled() { -@@ -422,6 +464,7 @@ impl MallocShallowSizeOf for smallvec::SmallVec { - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for smallvec::SmallVec - where - A: smallvec::Array, -@@ -436,6 +479,7 @@ where +@@ -412,30 +389,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } +-impl MallocShallowSizeOf for smallvec::SmallVec { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- if self.spilled() { +- unsafe { ops.malloc_size_of(self.as_ptr()) } +- } else { +- 0 +- } +- } +-} +- +-impl MallocSizeOf for smallvec::SmallVec +-where +- A: smallvec::Array, +- A::Item: MallocSizeOf, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = self.shallow_size_of(ops); +- for elem in self.iter() { +- n += elem.size_of(ops); +- } +- n +- } +-} +- +#[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashSet where T: Eq + Hash, -@@ -457,6 +501,7 @@ where +@@ -457,6 +411,7 @@ where } } @@ -266,47 +201,68 @@ index 778082b5f0..75ba0909e8 100644 impl MallocSizeOf for std::collections::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -471,6 +516,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for hashglobe::hash_set::HashSet - where - T: Eq + Hash, -@@ -488,6 +534,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for hashglobe::hash_set::HashSet - where - T: Eq + Hash + MallocSizeOf, -@@ -502,6 +549,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for hashglobe::fake::HashSet - where - T: Eq + Hash, -@@ -513,6 +561,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for hashglobe::fake::HashSet - where - T: Eq + Hash + MallocSizeOf, -@@ -524,6 +573,7 @@ where - } - } - +@@ -471,59 +426,7 @@ where + } + } + +-impl MallocShallowSizeOf for hashglobe::hash_set::HashSet +-where +- T: Eq + Hash, +- S: BuildHasher, +-{ +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- // See the implementation for std::collections::HashSet for details. +- if ops.has_malloc_enclosing_size_of() { +- self.iter() +- .next() +- .map_or(0, |t| unsafe { ops.malloc_enclosing_size_of(t) }) +- } else { +- self.capacity() * (size_of::() + size_of::()) +- } +- } +-} +- +-impl MallocSizeOf for hashglobe::hash_set::HashSet +-where +- T: Eq + Hash + MallocSizeOf, +- S: BuildHasher, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = self.shallow_size_of(ops); +- for t in self.iter() { +- n += t.size_of(ops); +- } +- n +- } +-} +- +-impl MallocShallowSizeOf for hashglobe::fake::HashSet +-where +- T: Eq + Hash, +- S: BuildHasher, +-{ +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- use std::ops::Deref; +- self.deref().shallow_size_of(ops) +- } +-} +- +-impl MallocSizeOf for hashglobe::fake::HashSet +-where +- T: Eq + Hash + MallocSizeOf, +- S: BuildHasher, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- use std::ops::Deref; +- self.deref().size_of(ops) +- } +-} +- +#[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashMap where K: Eq + Hash, -@@ -541,6 +591,7 @@ where +@@ -541,6 +444,7 @@ where } } @@ -314,50 +270,74 @@ index 778082b5f0..75ba0909e8 100644 impl MallocSizeOf for std::collections::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -587,6 +638,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for hashglobe::hash_map::HashMap - where - K: Eq + Hash, -@@ -604,6 +656,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for hashglobe::hash_map::HashMap - where - K: Eq + Hash + MallocSizeOf, -@@ -620,6 +673,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocShallowSizeOf for hashglobe::fake::HashMap - where - K: Eq + Hash, -@@ -631,6 +685,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for hashglobe::fake::HashMap - where - K: Eq + Hash + MallocSizeOf, -@@ -657,21 +712,38 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -587,62 +491,6 @@ where + } + } + +-impl MallocShallowSizeOf for hashglobe::hash_map::HashMap +-where +- K: Eq + Hash, +- S: BuildHasher, +-{ +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- // See the implementation for std::collections::HashSet for details. +- if ops.has_malloc_enclosing_size_of() { +- self.values() +- .next() +- .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) +- } else { +- self.capacity() * (size_of::() + size_of::() + size_of::()) +- } +- } +-} +- +-impl MallocSizeOf for hashglobe::hash_map::HashMap +-where +- K: Eq + Hash + MallocSizeOf, +- V: MallocSizeOf, +- S: BuildHasher, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = self.shallow_size_of(ops); +- for (k, v) in self.iter() { +- n += k.size_of(ops); +- n += v.size_of(ops); +- } +- n +- } +-} +- +-impl MallocShallowSizeOf for hashglobe::fake::HashMap +-where +- K: Eq + Hash, +- S: BuildHasher, +-{ +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- use std::ops::Deref; +- self.deref().shallow_size_of(ops) +- } +-} +- +-impl MallocSizeOf for hashglobe::fake::HashMap +-where +- K: Eq + Hash + MallocSizeOf, +- V: MallocSizeOf, +- S: BuildHasher, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- use std::ops::Deref; +- self.deref().size_of(ops) +- } +-} +- + // PhantomData is always 0. + impl MallocSizeOf for std::marker::PhantomData { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +@@ -657,21 +505,31 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } +#[cfg(feature = "std")] -+#[cfg(feature = "extra")] -+fn arc_ptr(s: &servo_arc::Arc) -> * const T { -+ s.heap_ptr() -+} -+ -+#[cfg(feature = "std")] -+#[cfg(not(feature = "extra"))] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { + let sc = s.clone(); + servo_arc::Arc::into_raw(sc) @@ -387,7 +367,7 @@ index 778082b5f0..75ba0909e8 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -679,9 +751,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +537,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } @@ -399,7 +379,7 @@ index 778082b5f0..75ba0909e8 100644 0 } else { self.unconditional_size_of(ops) -@@ -695,12 +768,14 @@ impl MallocConditionalSizeOf for servo_arc::Arc { +@@ -695,203 +554,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. @@ -410,124 +390,315 @@ index 778082b5f0..75ba0909e8 100644 } } -+#[cfg(feature = "extra")] - impl MallocSizeOf for smallbitvec::SmallBitVec { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if let Some(ptr) = self.heap_ptr() { -@@ -711,30 +786,35 @@ impl MallocSizeOf for smallbitvec::SmallBitVec { - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::Length { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.0.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedScale { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.0.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedPoint2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.x.size_of(ops) + self.y.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedRect { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.origin.size_of(ops) + self.size.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedSideOffsets2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.top.size_of(ops) + -@@ -744,12 +824,14 @@ impl MallocSizeOf for euclid::TypedSideOffsets2D { - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedSize2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.width.size_of(ops) + self.height.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for euclid::TypedTransform2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.m11.size_of(ops) + -@@ -761,6 +843,7 @@ impl MallocSizeOf for euclid::TypedTransform2D MallocSizeOf for euclid::TypedTransform3D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.m11.size_of(ops) + -@@ -782,12 +865,14 @@ impl MallocSizeOf for euclid::TypedTransform3D MallocSizeOf for euclid::TypedVector2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.x.size_of(ops) + self.y.size_of(ops) - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for selectors::parser::AncestorHashes { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let selectors::parser::AncestorHashes { ref packed_hashes } = *self; -@@ -795,6 +880,7 @@ impl MallocSizeOf for selectors::parser::AncestorHashes { - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for selectors::parser::Selector - where - Impl::NonTSPseudoClass: MallocSizeOf, -@@ -815,6 +901,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for selectors::parser::Component - where - Impl::NonTSPseudoClass: MallocSizeOf, -@@ -860,6 +947,7 @@ where - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf - for selectors::attr::AttrSelectorWithOptionalNamespace - { -@@ -868,6 +956,7 @@ impl MallocSizeOf - } - } - -+#[cfg(feature = "extra")] - impl MallocSizeOf for Void { - #[inline] - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -929,8 +1018,10 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +-impl MallocSizeOf for smallbitvec::SmallBitVec { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- if let Some(ptr) = self.heap_ptr() { +- unsafe { ops.malloc_size_of(ptr) } +- } else { +- 0 +- } +- } +-} +- +-impl MallocSizeOf for euclid::Length { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.0.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedScale { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.0.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedPoint2D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.x.size_of(ops) + self.y.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedRect { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.origin.size_of(ops) + self.size.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedSideOffsets2D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.top.size_of(ops) + +- self.right.size_of(ops) + +- self.bottom.size_of(ops) + +- self.left.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedSize2D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.width.size_of(ops) + self.height.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedTransform2D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.m11.size_of(ops) + +- self.m12.size_of(ops) + +- self.m21.size_of(ops) + +- self.m22.size_of(ops) + +- self.m31.size_of(ops) + +- self.m32.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedTransform3D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.m11.size_of(ops) + +- self.m12.size_of(ops) + +- self.m13.size_of(ops) + +- self.m14.size_of(ops) + +- self.m21.size_of(ops) + +- self.m22.size_of(ops) + +- self.m23.size_of(ops) + +- self.m24.size_of(ops) + +- self.m31.size_of(ops) + +- self.m32.size_of(ops) + +- self.m33.size_of(ops) + +- self.m34.size_of(ops) + +- self.m41.size_of(ops) + +- self.m42.size_of(ops) + +- self.m43.size_of(ops) + +- self.m44.size_of(ops) +- } +-} +- +-impl MallocSizeOf for euclid::TypedVector2D { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.x.size_of(ops) + self.y.size_of(ops) +- } +-} +- +-impl MallocSizeOf for selectors::parser::AncestorHashes { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let selectors::parser::AncestorHashes { ref packed_hashes } = *self; +- packed_hashes.size_of(ops) +- } +-} +- +-impl MallocSizeOf for selectors::parser::Selector +-where +- Impl::NonTSPseudoClass: MallocSizeOf, +- Impl::PseudoElement: MallocSizeOf, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = 0; +- +- // It's OK to measure this ThinArc directly because it's the +- // "primary" reference. (The secondary references are on the +- // Stylist.) +- n += unsafe { ops.malloc_size_of(self.thin_arc_heap_ptr()) }; +- for component in self.iter_raw_match_order() { +- n += component.size_of(ops); +- } +- +- n +- } +-} +- +-impl MallocSizeOf for selectors::parser::Component +-where +- Impl::NonTSPseudoClass: MallocSizeOf, +- Impl::PseudoElement: MallocSizeOf, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- use selectors::parser::Component; +- +- match self { +- Component::AttributeOther(ref attr_selector) => attr_selector.size_of(ops), +- Component::Negation(ref components) => components.size_of(ops), +- Component::NonTSPseudoClass(ref pseudo) => (*pseudo).size_of(ops), +- Component::Slotted(ref selector) | Component::Host(Some(ref selector)) => { +- selector.size_of(ops) +- }, +- Component::PseudoElement(ref pseudo) => (*pseudo).size_of(ops), +- Component::Combinator(..) | +- Component::ExplicitAnyNamespace | +- Component::ExplicitNoNamespace | +- Component::DefaultNamespace(..) | +- Component::Namespace(..) | +- Component::ExplicitUniversalType | +- Component::LocalName(..) | +- Component::ID(..) | +- Component::Class(..) | +- Component::AttributeInNoNamespaceExists { .. } | +- Component::AttributeInNoNamespace { .. } | +- Component::FirstChild | +- Component::LastChild | +- Component::OnlyChild | +- Component::Root | +- Component::Empty | +- Component::Scope | +- Component::NthChild(..) | +- Component::NthLastChild(..) | +- Component::NthOfType(..) | +- Component::NthLastOfType(..) | +- Component::FirstOfType | +- Component::LastOfType | +- Component::OnlyOfType | +- Component::Host(None) => 0, +- } +- } +-} +- +-impl MallocSizeOf +- for selectors::attr::AttrSelectorWithOptionalNamespace +-{ +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- 0 +- } +-} +- +-impl MallocSizeOf for Void { +- #[inline] +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- void::unreachable(*self) +- } +-} +- +-#[cfg(feature = "servo")] +-impl MallocSizeOf for string_cache::Atom { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- 0 +- } +-} +- +-// This is measured properly by the heap measurement implemented in +-// SpiderMonkey. +-#[cfg(feature = "servo")] +-impl MallocSizeOf for js::jsapi::Heap { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- 0 +- } +-} +- +-/// For use on types where size_of() returns 0. + #[macro_export] + macro_rules! malloc_size_of_is_0( + ($($ty:ty),+) => ( +@@ -929,117 +598,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); -+#[cfg(feature = "extra")] - malloc_size_of_is_0!(app_units::Au); - -+#[cfg(feature = "extra")] - malloc_size_of_is_0!(cssparser::RGBA, cssparser::TokenSerializationType); - - #[cfg(feature = "url")] +-malloc_size_of_is_0!(app_units::Au); +- +-malloc_size_of_is_0!(cssparser::RGBA, cssparser::TokenSerializationType); +- +-#[cfg(feature = "url")] +-impl MallocSizeOf for url::Host { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- match *self { +- url::Host::Domain(ref s) => s.size_of(ops), +- _ => 0, +- } +- } +-} +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::BorderRadius); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::BorderStyle); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::BoxShadowClipMode); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ClipAndScrollInfo); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ColorF); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ComplexClipRegion); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ExtendMode); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::FilterOp); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ExternalScrollId); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::FontInstanceKey); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::GradientStop); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::GlyphInstance); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::NinePatchBorder); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ImageKey); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ImageRendering); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::LineStyle); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::MixBlendMode); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::NormalBorder); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::RepeatMode); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::ScrollSensitivity); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::StickyOffsetBounds); +-#[cfg(feature = "webrender_api")] +-malloc_size_of_is_0!(webrender_api::TransformStyle); +- +-#[cfg(feature = "servo")] +-impl MallocSizeOf for keyboard_types::Key { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- match self { +- keyboard_types::Key::Character(ref s) => s.size_of(ops), +- _ => 0, +- } +- } +-} +- +-#[cfg(feature = "servo")] +-malloc_size_of_is_0!(keyboard_types::Modifiers); +- +-#[cfg(feature = "servo")] +-impl MallocSizeOf for xml5ever::QualName { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.prefix.size_of(ops) + self.ns.size_of(ops) + self.local.size_of(ops) +- } +-} +- +-#[cfg(feature = "servo")] +-malloc_size_of_is_0!(time::Duration); +-#[cfg(feature = "servo")] +-malloc_size_of_is_0!(time::Tm); +- +-#[cfg(feature = "servo")] +-impl MallocSizeOf for hyper_serde::Serde +-where +- for<'de> hyper_serde::De: serde::Deserialize<'de>, +- for<'a> hyper_serde::Ser<'a, T>: serde::Serialize, +- T: MallocSizeOf, +-{ +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.0.size_of(ops) +- } +-} +- +-// Placeholder for unique case where internals of Sender cannot be measured. +-// malloc size of is 0 macro complains about type supplied! +-#[cfg(feature = "servo")] +-impl MallocSizeOf for crossbeam_channel::Sender { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- 0 +- } +-} +- +-#[cfg(feature = "servo")] +-impl MallocSizeOf for hyper::StatusCode { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- 0 +- } +-} +- + /// Measurable that defers to inner value and used to verify MallocSizeOf implementation in a + /// struct. + #[derive(Clone)] diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 75ba0909e..f1cc08b3e 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -46,51 +46,10 @@ // This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. -#[cfg(feature = "extra")] -extern crate app_units; -#[cfg(feature = "servo")] -extern crate crossbeam_channel; -#[cfg(feature = "extra")] -extern crate cssparser; -#[cfg(feature = "extra")] -extern crate euclid; -#[cfg(feature = "extra")] -extern crate hashglobe; -#[cfg(feature = "servo")] -extern crate hyper; -#[cfg(feature = "servo")] -extern crate hyper_serde; -#[cfg(feature = "servo")] -extern crate keyboard_types; -#[cfg(feature = "servo")] -extern crate mozjs as js; -#[cfg(feature = "extra")] -extern crate selectors; -#[cfg(any(feature = "servo", feature = "serde_only"))] +#[cfg(feature = "serde_only")] extern crate serde; -#[cfg(any(feature = "servo", feature = "serde_only"))] +#[cfg(feature = "serde_only")] extern crate serde_bytes; -#[cfg(feature = "std")] -#[cfg(feature = "extra")] -extern crate servo_arc; -#[cfg(feature = "extra")] -extern crate smallbitvec; -#[cfg(feature = "extra")] -extern crate smallvec; -#[cfg(feature = "servo")] -extern crate string_cache; -#[cfg(feature = "extra")] -extern crate thin_slice; -#[cfg(feature = "servo")] -extern crate time; -#[cfg(feature = "url")] -extern crate url; -#[cfg(feature = "extra")] -extern crate void; -#[cfg(feature = "webrender_api")] -extern crate webrender_api; -#[cfg(feature = "servo")] -extern crate xml5ever; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -103,10 +62,9 @@ mod std { } #[cfg(feature = "std")] -#[cfg(not(feature = "extra"))] use std::sync as servo_arc; -#[cfg(any(feature = "servo", feature = "serde_only"))] +#[cfg(feature = "serde_only")] use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; @@ -118,8 +76,6 @@ use std::os::raw::c_void; use core::ffi::c_void; #[cfg(not(feature = "std"))] pub use alloc::boxed::Box; -#[cfg(feature = "extra")] -use void::Void; /// A C function that takes a pointer to a heap allocation and returns its size. pub type VoidPtrToSizeFn = unsafe extern "C" fn(ptr: *const c_void) -> usize; @@ -277,26 +233,6 @@ impl MallocSizeOf for Box { } } -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for thin_slice::ThinBoxedSlice { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = 0; - unsafe { - n += thin_slice::ThinBoxedSlice::spilled_storage(self) - .map_or(0, |ptr| ops.malloc_size_of(ptr)); - n += ops.malloc_size_of(&**self); - } - n - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for thin_slice::ThinBoxedSlice { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.shallow_size_of(ops) + (**self).size_of(ops) - } -} - impl MallocSizeOf for () { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { 0 @@ -391,14 +327,14 @@ impl MallocSizeOf for [T] { } #[cfg(not(feature = "no_ops_shallow"))] -#[cfg(any(feature = "servo", feature = "serde_only"))] +#[cfg(feature = "serde_only")] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } } } -#[cfg(any(feature = "servo", feature = "serde_only"))] +#[cfg(feature = "serde_only")] impl MallocSizeOf for ByteBuf { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); @@ -453,32 +389,6 @@ impl MallocSizeOf for std::collections::VecDeque { } } -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for smallvec::SmallVec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.spilled() { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } else { - 0 - } - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for smallvec::SmallVec -where - A: smallvec::Array, - A::Item: MallocSizeOf, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for elem in self.iter() { - n += elem.size_of(ops); - } - n - } -} - #[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashSet where @@ -516,63 +426,6 @@ where } } -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for hashglobe::hash_set::HashSet -where - T: Eq + Hash, - S: BuildHasher, -{ - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - // See the implementation for std::collections::HashSet for details. - if ops.has_malloc_enclosing_size_of() { - self.iter() - .next() - .map_or(0, |t| unsafe { ops.malloc_enclosing_size_of(t) }) - } else { - self.capacity() * (size_of::() + size_of::()) - } - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for hashglobe::hash_set::HashSet -where - T: Eq + Hash + MallocSizeOf, - S: BuildHasher, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for t in self.iter() { - n += t.size_of(ops); - } - n - } -} - -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for hashglobe::fake::HashSet -where - T: Eq + Hash, - S: BuildHasher, -{ - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - use std::ops::Deref; - self.deref().shallow_size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for hashglobe::fake::HashSet -where - T: Eq + Hash + MallocSizeOf, - S: BuildHasher, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - use std::ops::Deref; - self.deref().size_of(ops) - } -} - #[cfg(feature = "std")] impl MallocShallowSizeOf for std::collections::HashMap where @@ -638,66 +491,6 @@ where } } -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for hashglobe::hash_map::HashMap -where - K: Eq + Hash, - S: BuildHasher, -{ - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - // See the implementation for std::collections::HashSet for details. - if ops.has_malloc_enclosing_size_of() { - self.values() - .next() - .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) - } else { - self.capacity() * (size_of::() + size_of::() + size_of::()) - } - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for hashglobe::hash_map::HashMap -where - K: Eq + Hash + MallocSizeOf, - V: MallocSizeOf, - S: BuildHasher, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for (k, v) in self.iter() { - n += k.size_of(ops); - n += v.size_of(ops); - } - n - } -} - -#[cfg(feature = "extra")] -impl MallocShallowSizeOf for hashglobe::fake::HashMap -where - K: Eq + Hash, - S: BuildHasher, -{ - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - use std::ops::Deref; - self.deref().shallow_size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for hashglobe::fake::HashMap -where - K: Eq + Hash + MallocSizeOf, - V: MallocSizeOf, - S: BuildHasher, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - use std::ops::Deref; - self.deref().size_of(ops) - } -} - // PhantomData is always 0. impl MallocSizeOf for std::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { @@ -713,13 +506,6 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocShallowSizeOf for Arc { } #[cfg(feature = "std")] -#[cfg(feature = "extra")] -fn arc_ptr(s: &servo_arc::Arc) -> * const T { - s.heap_ptr() -} - -#[cfg(feature = "std")] -#[cfg(not(feature = "extra"))] fn arc_ptr(s: &servo_arc::Arc) -> * const T { let sc = s.clone(); servo_arc::Arc::into_raw(sc) @@ -775,212 +561,6 @@ impl MallocSizeOf for std::sync::Mutex { } } -#[cfg(feature = "extra")] -impl MallocSizeOf for smallbitvec::SmallBitVec { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if let Some(ptr) = self.heap_ptr() { - unsafe { ops.malloc_size_of(ptr) } - } else { - 0 - } - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::Length { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.0.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedScale { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.0.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedPoint2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.x.size_of(ops) + self.y.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedRect { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.origin.size_of(ops) + self.size.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedSideOffsets2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.top.size_of(ops) + - self.right.size_of(ops) + - self.bottom.size_of(ops) + - self.left.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedSize2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.width.size_of(ops) + self.height.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedTransform2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.m11.size_of(ops) + - self.m12.size_of(ops) + - self.m21.size_of(ops) + - self.m22.size_of(ops) + - self.m31.size_of(ops) + - self.m32.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedTransform3D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.m11.size_of(ops) + - self.m12.size_of(ops) + - self.m13.size_of(ops) + - self.m14.size_of(ops) + - self.m21.size_of(ops) + - self.m22.size_of(ops) + - self.m23.size_of(ops) + - self.m24.size_of(ops) + - self.m31.size_of(ops) + - self.m32.size_of(ops) + - self.m33.size_of(ops) + - self.m34.size_of(ops) + - self.m41.size_of(ops) + - self.m42.size_of(ops) + - self.m43.size_of(ops) + - self.m44.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for euclid::TypedVector2D { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.x.size_of(ops) + self.y.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for selectors::parser::AncestorHashes { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let selectors::parser::AncestorHashes { ref packed_hashes } = *self; - packed_hashes.size_of(ops) - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for selectors::parser::Selector -where - Impl::NonTSPseudoClass: MallocSizeOf, - Impl::PseudoElement: MallocSizeOf, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = 0; - - // It's OK to measure this ThinArc directly because it's the - // "primary" reference. (The secondary references are on the - // Stylist.) - n += unsafe { ops.malloc_size_of(self.thin_arc_heap_ptr()) }; - for component in self.iter_raw_match_order() { - n += component.size_of(ops); - } - - n - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for selectors::parser::Component -where - Impl::NonTSPseudoClass: MallocSizeOf, - Impl::PseudoElement: MallocSizeOf, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - use selectors::parser::Component; - - match self { - Component::AttributeOther(ref attr_selector) => attr_selector.size_of(ops), - Component::Negation(ref components) => components.size_of(ops), - Component::NonTSPseudoClass(ref pseudo) => (*pseudo).size_of(ops), - Component::Slotted(ref selector) | Component::Host(Some(ref selector)) => { - selector.size_of(ops) - }, - Component::PseudoElement(ref pseudo) => (*pseudo).size_of(ops), - Component::Combinator(..) | - Component::ExplicitAnyNamespace | - Component::ExplicitNoNamespace | - Component::DefaultNamespace(..) | - Component::Namespace(..) | - Component::ExplicitUniversalType | - Component::LocalName(..) | - Component::ID(..) | - Component::Class(..) | - Component::AttributeInNoNamespaceExists { .. } | - Component::AttributeInNoNamespace { .. } | - Component::FirstChild | - Component::LastChild | - Component::OnlyChild | - Component::Root | - Component::Empty | - Component::Scope | - Component::NthChild(..) | - Component::NthLastChild(..) | - Component::NthOfType(..) | - Component::NthLastOfType(..) | - Component::FirstOfType | - Component::LastOfType | - Component::OnlyOfType | - Component::Host(None) => 0, - } - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf - for selectors::attr::AttrSelectorWithOptionalNamespace -{ - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 - } -} - -#[cfg(feature = "extra")] -impl MallocSizeOf for Void { - #[inline] - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - void::unreachable(*self) - } -} - -#[cfg(feature = "servo")] -impl MallocSizeOf for string_cache::Atom { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 - } -} - -// This is measured properly by the heap measurement implemented in -// SpiderMonkey. -#[cfg(feature = "servo")] -impl MallocSizeOf for js::jsapi::Heap { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 - } -} - -/// For use on types where size_of() returns 0. #[macro_export] macro_rules! malloc_size_of_is_0( ($($ty:ty),+) => ( @@ -1018,119 +598,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); -#[cfg(feature = "extra")] -malloc_size_of_is_0!(app_units::Au); - -#[cfg(feature = "extra")] -malloc_size_of_is_0!(cssparser::RGBA, cssparser::TokenSerializationType); - -#[cfg(feature = "url")] -impl MallocSizeOf for url::Host { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - match *self { - url::Host::Domain(ref s) => s.size_of(ops), - _ => 0, - } - } -} -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::BorderRadius); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::BorderStyle); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::BoxShadowClipMode); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ClipAndScrollInfo); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ColorF); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ComplexClipRegion); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ExtendMode); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::FilterOp); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ExternalScrollId); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::FontInstanceKey); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::GradientStop); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::GlyphInstance); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::NinePatchBorder); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ImageKey); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ImageRendering); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::LineStyle); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::MixBlendMode); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::NormalBorder); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::RepeatMode); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::ScrollSensitivity); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::StickyOffsetBounds); -#[cfg(feature = "webrender_api")] -malloc_size_of_is_0!(webrender_api::TransformStyle); - -#[cfg(feature = "servo")] -impl MallocSizeOf for keyboard_types::Key { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - match self { - keyboard_types::Key::Character(ref s) => s.size_of(ops), - _ => 0, - } - } -} - -#[cfg(feature = "servo")] -malloc_size_of_is_0!(keyboard_types::Modifiers); - -#[cfg(feature = "servo")] -impl MallocSizeOf for xml5ever::QualName { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.prefix.size_of(ops) + self.ns.size_of(ops) + self.local.size_of(ops) - } -} - -#[cfg(feature = "servo")] -malloc_size_of_is_0!(time::Duration); -#[cfg(feature = "servo")] -malloc_size_of_is_0!(time::Tm); - -#[cfg(feature = "servo")] -impl MallocSizeOf for hyper_serde::Serde -where - for<'de> hyper_serde::De: serde::Deserialize<'de>, - for<'a> hyper_serde::Ser<'a, T>: serde::Serialize, - T: MallocSizeOf, -{ - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - self.0.size_of(ops) - } -} - -// Placeholder for unique case where internals of Sender cannot be measured. -// malloc size of is 0 macro complains about type supplied! -#[cfg(feature = "servo")] -impl MallocSizeOf for crossbeam_channel::Sender { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 - } -} - -#[cfg(feature = "servo")] -impl MallocSizeOf for hyper::StatusCode { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 - } -} - /// Measurable that defers to inner value and used to verify MallocSizeOf implementation in a /// struct. #[derive(Clone)] From fd9dc921600e723832828541bc47250c263e1f11 Mon Sep 17 00:00:00 2001 From: Sergei Pepyakin Date: Wed, 2 Jan 2019 19:55:45 +0100 Subject: [PATCH 12/37] Update parity-util-mem/README.md Co-Authored-By: cheme --- parity-util-mem/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/README.md b/parity-util-mem/README.md index 305f24ab7..e01fb6ce6 100644 --- a/parity-util-mem/README.md +++ b/parity-util-mem/README.md @@ -4,7 +4,7 @@ Collection of memory related utilities. ## Features -- volatile-erase : Not set by default, `Memzero` struct will be erasing memory throug a simple [`write_volatile`](https://doc.rust-lang.org/std/ptr/fn.write_volatile.html) call. +- volatile-erase : Not set by default, `Memzero` struct will be erasing memory through a simple [`write_volatile`](https://doc.rust-lang.org/std/ptr/fn.write_volatile.html) call. - estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. Others features define global allocator, see `src/alloc.rs`. From 3df427b4253df175400fac332f7c17a840d8d7e1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Jan 2019 09:08:08 +0100 Subject: [PATCH 13/37] Missing windows import --- parity-util-mem/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index 0771911bf..40fa7e7ed 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -19,6 +19,9 @@ elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } parking_lot = { version = "*", optional = true } +[target."cfg(windows)".dependencies.winapi] +version = "0.3.4" + [features] default = ["std", "ethereum-impls"] std = [] From ef9a5b6f1f9b0a942d62b43e4dea23a21db73f4c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Jan 2019 09:46:20 +0100 Subject: [PATCH 14/37] Use self for winapi. --- parity-util-mem/src/allocators.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 027c0bdb2..2e6e63e00 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -43,16 +43,15 @@ use std::os::raw::c_void; use core::ffi::c_void; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet; + #[cfg(not(feature = "weealloc-global"))] #[cfg(not(feature = "dlmalloc-global"))] #[cfg(not(feature = "jemalloc-global"))] -#[cfg(windows)] +#[cfg(target_os = "windows")] mod usable_size { - #[cfg(target_os = "windows")] extern crate winapi; - #[cfg(target_os = "windows")] - use winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; + use self::winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; use std::os::raw::c_void; /// Get the size of a heap block. @@ -186,8 +185,8 @@ pub fn new_malloc_size_ops() -> MallocSizeOfOps { /// It allows getting heapsize without exposing `MallocSizeOfOps` /// (a single default `MallocSizeOfOps` is used for each call). pub trait MallocSizeOfExt: MallocSizeOf { - /// Method to launch a heapsize measurement with a - /// fresh state. + /// Method to launch a heapsize measurement with a + /// fresh state. fn malloc_size_of(&self) -> usize { let mut ops = new_malloc_size_ops(); ::size_of(self, &mut ops) From 1b0b2e33ef97288beccfc8d9bfb952df3dc2b0b1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Jan 2019 10:15:22 +0100 Subject: [PATCH 15/37] Correct import. --- parity-util-mem/src/allocators.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 2e6e63e00..1d48a7876 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -51,8 +51,8 @@ use alloc::collections::btree_set::BTreeSet; mod usable_size { extern crate winapi; + use super::*; use self::winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; - use std::os::raw::c_void; /// Get the size of a heap block. /// Call windows allocator through `winapi` crate From 5340912c1984304172ae1e8ba61d697e5ae05834 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 3 Jan 2019 19:10:00 +0100 Subject: [PATCH 16/37] shallow size on Arc with malloc return -1 (max val), replacing with estimate. --- parity-util-mem/get_malloc_size_src.sh | 6 +++--- parity-util-mem/slim_malloc_size_of.patch | 26 +++++++++++++++++------ parity-util-mem/src/lib.rs | 12 +++++++++++ parity-util-mem/src/malloc_size.rs | 16 ++++++++++++-- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/parity-util-mem/get_malloc_size_src.sh b/parity-util-mem/get_malloc_size_src.sh index 4559dbeb8..e717d6f4b 100755 --- a/parity-util-mem/get_malloc_size_src.sh +++ b/parity-util-mem/get_malloc_size_src.sh @@ -7,7 +7,7 @@ cd servo git checkout 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc git apply ../slim_malloc_size_of.patch #git merge master -cp components/malloc_size_of/lib.rs ../src/malloc_size.rs -cp -r components/malloc_size_of_derive .. -cd .. +#cp components/malloc_size_of/lib.rs ../src/malloc_size.rs +#cp -r components/malloc_size_of_derive .. +#cd .. #rm -rf ./servo diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index 3e7abf669..f7d63ae6b 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,5 +1,5 @@ diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..f1cc08b3e4 100644 +index 778082b5f0..6bbe400204 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,55 +43,45 @@ @@ -333,18 +333,20 @@ index 778082b5f0..f1cc08b3e4 100644 // PhantomData is always 0. impl MallocSizeOf for std::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -657,21 +505,31 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,21 +505,43 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } +#[cfg(feature = "std")] +fn arc_ptr(s: &servo_arc::Arc) -> * const T { -+ let sc = s.clone(); -+ servo_arc::Arc::into_raw(sc) ++ &(**s) as *const T +} + ++ ++// currently this seems only fine with jemalloc +#[cfg(feature = "std")] +#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.heap_ptr()) } @@ -352,6 +354,16 @@ index 778082b5f0..f1cc08b3e4 100644 } } ++#[cfg(feature = "std")] ++#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] ++impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { ++ fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ size_of::() ++ } ++} ++ ++ +#[cfg(feature = "std")] impl MallocUnconditionalSizeOf for servo_arc::Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -367,7 +379,7 @@ index 778082b5f0..f1cc08b3e4 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -679,9 +537,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +549,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } @@ -379,7 +391,7 @@ index 778082b5f0..f1cc08b3e4 100644 0 } else { self.unconditional_size_of(ops) -@@ -695,203 +554,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { +@@ -695,203 +566,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. @@ -584,7 +596,7 @@ index 778082b5f0..f1cc08b3e4 100644 #[macro_export] macro_rules! malloc_size_of_is_0( ($($ty:ty),+) => ( -@@ -929,117 +598,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,117 +610,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index 8454acd4b..948899898 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -134,3 +134,15 @@ impl> DerefMut for Memzero { &mut self.mem } } + +mod test { + use std::sync::Arc; + use super::MallocSizeOfExt; + #[test] + fn test_arc() { + let val = Arc::new("test".to_string()); + let s = val.malloc_size_of(); + assert!(s > 0); + + } +} diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index f1cc08b3e..6bbe40020 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -507,18 +507,30 @@ impl MallocSizeOf for std::marker::PhantomData { #[cfg(feature = "std")] fn arc_ptr(s: &servo_arc::Arc) -> * const T { - let sc = s.clone(); - servo_arc::Arc::into_raw(sc) + &(**s) as *const T } + +// currently this seems only fine with jemalloc #[cfg(feature = "std")] #[cfg(not(feature = "no_ops_shallow"))] +#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(arc_ptr(self)) } } } +#[cfg(feature = "std")] +#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } +} + + #[cfg(feature = "std")] impl MallocUnconditionalSizeOf for servo_arc::Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { From 8ada8148718d5c1d247aca5ba5d19135f399d284 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 7 Jan 2019 18:08:10 +0100 Subject: [PATCH 17/37] use cfg_if to makes feature choice more explicit. --- parity-util-mem/Cargo.toml | 4 +- parity-util-mem/slim_malloc_size_of.patch | 12 +- parity-util-mem/src/allocators.rs | 156 +++++++--------------- parity-util-mem/src/lib.rs | 70 +++++----- parity-util-mem/src/malloc_size.rs | 14 +- 5 files changed, 103 insertions(+), 153 deletions(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index 40fa7e7ed..87f416e53 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -8,6 +8,7 @@ license = "GPL-3.0" [dependencies] clear_on_drop = "0.2" +cfg-if = "0.1.6" malloc_size_of_derive = { path = "./malloc_size_of_derive" } dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } @@ -38,5 +39,4 @@ ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] # Include serde related struct metering. serde_only = ["serde", "serde_bytes"] # Full estimate: no call to allocator -estimate-heapsize = ["no_ops_shallow"] -no_ops_shallow = [] +estimate-heapsize = [] diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index f7d63ae6b..ba7474ed5 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -90,7 +90,7 @@ index 778082b5f0..6bbe400204 100644 fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -98,7 +98,7 @@ index 778082b5f0..6bbe400204 100644 } } -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } @@ -140,7 +140,7 @@ index 778082b5f0..6bbe400204 100644 } -#[cfg(feature = "servo")] -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(feature = "serde_only")] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -157,7 +157,7 @@ index 778082b5f0..6bbe400204 100644 } } -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -345,7 +345,7 @@ index 778082b5f0..6bbe400204 100644 + +// currently this seems only fine with jemalloc +#[cfg(feature = "std")] -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -355,7 +355,7 @@ index 778082b5f0..6bbe400204 100644 } +#[cfg(feature = "std")] -+#[cfg(not(feature = "no_ops_shallow"))] ++#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 1d48a7876..eafccefd3 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -18,19 +18,18 @@ //! Features are: //! - windows: //! - no features: default implementation from servo `heapsize` crate -//! - weealloc: return 0 for compatibility -//! - dlmalloc: enable, does not work, for compatibility only -//! - jemalloc: compile error +//! - weealloc: default to `estimate_size` +//! - dlmalloc: default to `estimate_size` +//! - jemalloc: default windows allocator is used instead //! - arch x86: //! - no features: use default alloc //! - jemalloc: use jemallocator crate -//! - weealloc: return 0 for compatibility -//! - dlmalloc: enable, does not work, for compatibility only +//! - weealloc: default to `estimate_size` +//! - dlmalloc: default to `estimate_size` //! - arch wasm32: -//! - no features: return 0 for compatibility -//! - dlmalloc: return 0 for compatibility (usable_size could be implemented if needed in -//! dlmalloc crate) -//! - weealloc: return 0 for compatibility +//! - no features: default to `estimate_size` +//! - weealloc: default to `estimate_size` +//! - dlmalloc: default to `estimate_size` //! - jemalloc: compile error @@ -44,126 +43,69 @@ use core::ffi::c_void; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet; -#[cfg(not(feature = "weealloc-global"))] -#[cfg(not(feature = "dlmalloc-global"))] -#[cfg(not(feature = "jemalloc-global"))] -#[cfg(target_os = "windows")] mod usable_size { - extern crate winapi; use super::*; - use self::winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; - /// Get the size of a heap block. - /// Call windows allocator through `winapi` crate - pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { +cfg_if! { + + if #[cfg(any( + target_arch = "wasm32", + feature = "estimate-heapsize", + feature = "weealloc-global", + feature = "dlmalloc-global", + ))] { - let heap = GetProcessHeap(); + // do not try system allocator - if HeapValidate(heap, 0, ptr) == 0 { - ptr = *(ptr as *const *const c_void).offset(-1); + /// Warning this is for compatibility only. + /// This function does panic: `estimate-heapsize` feature needs to be activated + /// to avoid this function call. + pub unsafe extern "C" fn malloc_usable_size(_ptr: *const c_void) -> usize { + unreachable!("estimate heapsize only") } - HeapSize(heap, 0, ptr) as usize - } + } else if #[cfg(target_os = "windows")] { - /// No enclosing function defined. - #[inline] - pub fn new_enclosing_size_fn() -> Option { - None - } -} + // default windows allocator + extern crate winapi; -#[cfg(not(feature = "weealloc-global"))] -#[cfg(not(feature = "dlmalloc-global"))] -#[cfg(not(feature = "jemalloc-global"))] -#[cfg(all(not(windows), not(target_arch = "wasm32")))] -// default -mod usable_size { - use super::*; + use self::winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; - /// Default allocators for different platforms. - /// Macos, ios and android calls jemalloc. - /// Linux call system allocator (currently malloc). - extern "C" { - #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] - pub fn malloc_usable_size(ptr: *const c_void) -> usize; - } + /// Get the size of a heap block. + /// Call windows allocator through `winapi` crate + pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { - /// No enclosing function defined. - #[inline] - pub fn new_enclosing_size_fn() -> Option { - None - } + let heap = GetProcessHeap(); -} - -#[cfg(not(feature = "weealloc-global"))] -#[cfg(not(feature = "dlmalloc-global"))] -#[cfg(not(feature = "jemalloc-global"))] -#[cfg(target_arch = "wasm32")] -mod usable_size { - use super::*; + if HeapValidate(heap, 0, ptr) == 0 { + ptr = *(ptr as *const *const c_void).offset(-1); + } - /// Warning this is for compatibility only. - /// This function does panic: `estimate-heapsize` feature needs to be activated - /// to avoid this function call. - pub unsafe extern "C" fn malloc_usable_size(mut ptr: *const c_void) -> usize { - panic!("Please run with `estimate-heapsize` feature") - } - - /// No enclosing function defined. - #[inline] - pub fn new_enclosing_size_fn() -> Option { - None - } -} + HeapSize(heap, 0, ptr) as usize + } -#[cfg(feature = "jemalloc-global")] -mod usable_size { - use super::*; + } else if #[cfg(feature = "jemalloc-global")] { - /// Use of jemalloc usable size C function through jemallocator crate call. - pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { - jemallocator::usable_size(ptr) - } + /// Use of jemalloc usable size C function through jemallocator crate call. + pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { + jemallocator::usable_size(ptr) + } - /// No enclosing function defined. - #[inline] - pub fn new_enclosing_size_fn() -> Option { - None - } -} + } else { -#[cfg(feature = "dlmalloc-global")] -mod usable_size { - use super::*; + // default allocator used + /// Macos, ios and android calls jemalloc. + /// Linux call system allocator (currently malloc). + extern "C" { + #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] + pub fn malloc_usable_size(ptr: *const c_void) -> usize; + } - /// Warning this is for compatibility only. - /// Feature: `estimate-heapsize` is on with `dlmalloc-global` and this code - /// should never be called (it panics otherwhise). - pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { - panic!("Running estimation this code should never be reached") } - /// No enclosing function defined. - #[inline] - pub fn new_enclosing_size_fn() -> Option { - None - } } -#[cfg(feature = "weealloc-global")] -mod usable_size { - use super::*; - - /// Warning this is for compatibility only. - /// Feature: `estimate-heapsize` is on with `weealloc-global` and this code - /// should never be called (it panics otherwhise). - pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { - panic!("Running estimation this code should never be reached") - } - /// No enclosing function defined. #[inline] pub fn new_enclosing_size_fn() -> Option { @@ -193,7 +135,7 @@ pub trait MallocSizeOfExt: MallocSizeOf { } } -impl MallocSizeOfExt for T {} +impl MallocSizeOfExt for T { } #[cfg(feature = "std")] impl MallocSizeOf for std::sync::Arc { diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index 948899898..51a59c5a2 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -22,6 +22,9 @@ #![cfg_attr(not(feature = "std"), feature(core_intrinsics))] #![cfg_attr(not(feature = "std"), feature(alloc))] +#[macro_use] +extern crate cfg_if; + #[cfg(not(feature = "std"))] extern crate alloc; @@ -37,29 +40,32 @@ use std::ptr; #[cfg(not(feature = "volatile-erase"))] pub use cod::clear::Clear; -#[cfg(feature = "jemalloc-global")] -extern crate jemallocator; - -#[cfg(feature = "dlmalloc-global")] -extern crate dlmalloc; - -#[cfg(feature = "weealloc-global")] -extern crate wee_alloc; - -#[cfg(feature = "jemalloc-global")] -#[global_allocator] -/// Global allocator -pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -#[cfg(feature = "dlmalloc-global")] -#[global_allocator] -/// Global allocator -pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; -#[cfg(feature = "weealloc-global")] -#[global_allocator] -/// Global allocator -pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; +cfg_if! { + if #[cfg(all( + feature = "jemalloc-global", + feature = "jemalloc-global", + not(target_os = "windows"), + not(target_arch = "wasm32") + ))] { + extern crate jemallocator; + #[global_allocator] + /// Global allocator + pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + } else if #[cfg(feature = "dlmalloc-global")] { + extern crate dlmalloc; + #[global_allocator] + /// Global allocator + pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; + } else if #[cfg(feature = "weealloc-global")] { + extern crate wee_alloc; + #[global_allocator] + /// Global allocator + pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; + } else { + // default allocator used + } +} pub mod allocators; @@ -135,14 +141,16 @@ impl> DerefMut for Memzero { } } +#[cfg(test)] mod test { - use std::sync::Arc; - use super::MallocSizeOfExt; - #[test] - fn test_arc() { - let val = Arc::new("test".to_string()); - let s = val.malloc_size_of(); - assert!(s > 0); - - } + use std::sync::Arc; + use super::MallocSizeOfExt; + + #[test] + fn test_arc() { + let val = Arc::new("test".to_string()); + let s = val.malloc_size_of(); + assert!(s > 0); + } + } diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 6bbe40020..b120cfd77 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -206,7 +206,7 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -220,7 +220,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } @@ -326,7 +326,7 @@ impl MallocSizeOf for [T] { } } -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] #[cfg(feature = "serde_only")] impl MallocShallowSizeOf for ByteBuf { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -345,7 +345,7 @@ impl MallocSizeOf for ByteBuf { } } -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -513,7 +513,7 @@ fn arc_ptr(s: &servo_arc::Arc) -> * const T { // currently this seems only fine with jemalloc #[cfg(feature = "std")] -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] #[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { @@ -522,10 +522,10 @@ impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { } #[cfg(feature = "std")] -#[cfg(not(feature = "no_ops_shallow"))] +#[cfg(not(feature = "estimate-heapsize"))] #[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { - fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { size_of::() } } From 61c60b04cdaa6ec4adc2674b98ec3c7df1d9d886 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jan 2019 17:40:16 +0100 Subject: [PATCH 18/37] Remove servo specific impls (unused). --- parity-util-mem/Cargo.toml | 4 -- parity-util-mem/slim_malloc_size_of.patch | 80 +++++++++++------------ parity-util-mem/src/allocators.rs | 2 +- parity-util-mem/src/impls.rs | 2 +- parity-util-mem/src/lib.rs | 2 +- parity-util-mem/src/malloc_size.rs | 25 ------- parity-util-mem/src/sizeof.rs | 4 +- 7 files changed, 43 insertions(+), 76 deletions(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index 87f416e53..068b42fc9 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -13,8 +13,6 @@ malloc_size_of_derive = { path = "./malloc_size_of_derive" } dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } -serde = { version = "1.0", optional = true } -serde_bytes = { version = "0.10", optional = true } elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } @@ -36,7 +34,5 @@ weealloc-global = ["wee_alloc", "estimate-heapsize"] jemalloc-global = ["jemallocator"] # implement additional types ethereum-impls = ["ethereum-types", "elastic-array", "parking_lot"] -# Include serde related struct metering. -serde_only = ["serde", "serde_bytes"] # Full estimate: no call to allocator estimate-heapsize = [] diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index ba7474ed5..074d6b841 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,8 +1,8 @@ diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..6bbe400204 100644 +index 778082b5f0..4c766f609b 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs -@@ -43,55 +43,45 @@ +@@ -43,55 +43,39 @@ //! measured as well as the thing it points to. E.g. //! ` as MallocSizeOf>::size_of(field, ops)`. @@ -22,14 +22,9 @@ index 778082b5f0..6bbe400204 100644 -extern crate mozjs as js; -extern crate selectors; -#[cfg(feature = "servo")] -+ -+// This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. -+ -+#[cfg(feature = "serde_only")] - extern crate serde; +-extern crate serde; -#[cfg(feature = "servo")] -+#[cfg(feature = "serde_only")] - extern crate serde_bytes; +-extern crate serde_bytes; -extern crate servo_arc; -extern crate smallbitvec; -extern crate smallvec; @@ -49,6 +44,9 @@ index 778082b5f0..6bbe400204 100644 -#[cfg(feature = "servo")] -use serde_bytes::ByteBuf; + ++// This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. ++ ++ +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; +#[cfg(not(feature = "std"))] @@ -62,8 +60,6 @@ index 778082b5f0..6bbe400204 100644 +#[cfg(feature = "std")] +use std::sync as servo_arc; + -+#[cfg(feature = "serde_only")] -+use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; use std::ops::Range; @@ -86,7 +82,7 @@ index 778082b5f0..6bbe400204 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -216,6 +206,7 @@ pub trait MallocConditionalShallowSizeOf { +@@ -216,6 +200,7 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } @@ -94,7 +90,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -229,6 +220,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { +@@ -229,6 +214,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } @@ -102,7 +98,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(&**self) } -@@ -241,24 +233,6 @@ impl MallocSizeOf for Box { +@@ -241,24 +227,6 @@ impl MallocSizeOf for Box { } } @@ -127,7 +123,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocSizeOf for () { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { 0 -@@ -329,6 +303,7 @@ impl MallocSizeOf for std::cell::RefCell { +@@ -329,6 +297,7 @@ impl MallocSizeOf for std::cell::RefCell { } } @@ -135,33 +131,33 @@ index 778082b5f0..6bbe400204 100644 impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> where B::Owned: MallocSizeOf, -@@ -351,14 +326,15 @@ impl MallocSizeOf for [T] { +@@ -351,24 +320,7 @@ impl MallocSizeOf for [T] { } } -#[cfg(feature = "servo")] -+#[cfg(not(feature = "estimate-heapsize"))] -+#[cfg(feature = "serde_only")] - impl MallocShallowSizeOf for ByteBuf { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } - } - +-impl MallocShallowSizeOf for ByteBuf { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.as_ptr()) } +- } +-} +- -#[cfg(feature = "servo")] -+#[cfg(feature = "serde_only")] - impl MallocSizeOf for ByteBuf { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); -@@ -369,6 +345,7 @@ impl MallocSizeOf for ByteBuf { - } - } - +-impl MallocSizeOf for ByteBuf { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- let mut n = self.shallow_size_of(ops); +- for elem in self.iter() { +- n += elem.size_of(ops); +- } +- n +- } +-} +- +#[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -412,30 +389,7 @@ impl MallocSizeOf for std::collections::VecDeque { +@@ -412,30 +364,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } @@ -193,7 +189,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocShallowSizeOf for std::collections::HashSet where T: Eq + Hash, -@@ -457,6 +411,7 @@ where +@@ -457,6 +386,7 @@ where } } @@ -201,7 +197,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocSizeOf for std::collections::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -471,59 +426,7 @@ where +@@ -471,59 +401,7 @@ where } } @@ -262,7 +258,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocShallowSizeOf for std::collections::HashMap where K: Eq + Hash, -@@ -541,6 +444,7 @@ where +@@ -541,6 +419,7 @@ where } } @@ -270,7 +266,7 @@ index 778082b5f0..6bbe400204 100644 impl MallocSizeOf for std::collections::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -587,62 +491,6 @@ where +@@ -587,62 +466,6 @@ where } } @@ -333,7 +329,7 @@ index 778082b5f0..6bbe400204 100644 // PhantomData is always 0. impl MallocSizeOf for std::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -657,21 +505,43 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,21 +480,43 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } @@ -358,7 +354,7 @@ index 778082b5f0..6bbe400204 100644 +#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { -+ fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } +} @@ -379,7 +375,7 @@ index 778082b5f0..6bbe400204 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -679,9 +549,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +524,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } @@ -391,7 +387,7 @@ index 778082b5f0..6bbe400204 100644 0 } else { self.unconditional_size_of(ops) -@@ -695,203 +566,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { +@@ -695,203 +541,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. @@ -596,7 +592,7 @@ index 778082b5f0..6bbe400204 100644 #[macro_export] macro_rules! malloc_size_of_is_0( ($($ty:ty),+) => ( -@@ -929,117 +610,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,117 +585,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index eafccefd3..ca8cea7b3 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// Copyright 2015-2019 Parity Technologies (UK) Ltd. // This file is part of Parity. // Parity is free software: you can redistribute it and/or modify diff --git a/parity-util-mem/src/impls.rs b/parity-util-mem/src/impls.rs index d682d2e42..19e787ba3 100644 --- a/parity-util-mem/src/impls.rs +++ b/parity-util-mem/src/impls.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// Copyright 2015-2019 Parity Technologies (UK) Ltd. // This file is part of Parity. // Parity is free software: you can redistribute it and/or modify diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index 51a59c5a2..414b00176 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// Copyright 2015-2019 Parity Technologies (UK) Ltd. // This file is part of Parity. // Parity is free software: you can redistribute it and/or modify diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index b120cfd77..4c766f609 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -46,10 +46,6 @@ // This file is patched at commit 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc DO NOT EDIT. -#[cfg(feature = "serde_only")] -extern crate serde; -#[cfg(feature = "serde_only")] -extern crate serde_bytes; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -64,8 +60,6 @@ mod std { #[cfg(feature = "std")] use std::sync as servo_arc; -#[cfg(feature = "serde_only")] -use self::serde_bytes::ByteBuf; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; use std::ops::Range; @@ -326,25 +320,6 @@ impl MallocSizeOf for [T] { } } -#[cfg(not(feature = "estimate-heapsize"))] -#[cfg(feature = "serde_only")] -impl MallocShallowSizeOf for ByteBuf { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } -} - -#[cfg(feature = "serde_only")] -impl MallocSizeOf for ByteBuf { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for elem in self.iter() { - n += elem.size_of(ops); - } - n - } -} - #[cfg(not(feature = "estimate-heapsize"))] impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { diff --git a/parity-util-mem/src/sizeof.rs b/parity-util-mem/src/sizeof.rs index 01cacb339..2b8522546 100644 --- a/parity-util-mem/src/sizeof.rs +++ b/parity-util-mem/src/sizeof.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// Copyright 2015-2019 Parity Technologies (UK) Ltd. // This file is part of Parity. // Parity is free software: you can redistribute it and/or modify @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Estimation for heapsize calculation. Usable to replace call to allocator method (for some -//! allocators or simply because we just need a determinist indicator of cunsumption). +//! allocators or simply because we just need a deterministic cunsumption measurement). #[cfg(feature = "serde_only")] extern crate serde_bytes; From 767dc908e02cf3b9a1ccc5b08cee2e9cdc097d05 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jan 2019 17:56:39 +0100 Subject: [PATCH 19/37] `estimate_heapsize` in travis. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 75601d9a2..505bf1a62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,3 +24,4 @@ script: - cd uint/ && cargo test --features=std,quickcheck --release && cd .. - cd hashdb/ && cargo test --no-default-features && cd .. - cd plain_hasher/ && cargo test --no-default-features && cd .. + - cd parity-util-mem/ && cargo test --features=estimate-heapsize && cd .. From a31851edf19473515dc27aedc9e9755ab635483f Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jan 2019 13:42:49 +0100 Subject: [PATCH 20/37] Remove last serde and remove servo_arc reference. --- parity-util-mem/slim_malloc_size_of.patch | 20 ++++++++++++-------- parity-util-mem/src/lib.rs | 1 + parity-util-mem/src/malloc_size.rs | 14 +++++++------- parity-util-mem/src/sizeof.rs | 11 ----------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index 074d6b841..61381c146 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,5 +1,5 @@ diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..4c766f609b 100644 +index 778082b5f0..7f527c930e 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,55 +43,39 @@ @@ -58,7 +58,7 @@ index 778082b5f0..4c766f609b 100644 +} + +#[cfg(feature = "std")] -+use std::sync as servo_arc; ++use std::sync::Arc; + use std::hash::{BuildHasher, Hash}; use std::mem::size_of; @@ -333,8 +333,9 @@ index 778082b5f0..4c766f609b 100644 //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } +-impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] -+fn arc_ptr(s: &servo_arc::Arc) -> * const T { ++fn arc_ptr(s: &Arc) -> * const T { + &(**s) as *const T +} + @@ -343,17 +344,18 @@ index 778082b5f0..4c766f609b 100644 +#[cfg(feature = "std")] +#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] - impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { ++impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.heap_ptr()) } + unsafe { ops.malloc_size_of(arc_ptr(self)) } } } +-impl MallocUnconditionalSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] +#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -+impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { ++impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } @@ -361,14 +363,15 @@ index 778082b5f0..4c766f609b 100644 + + +#[cfg(feature = "std")] - impl MallocUnconditionalSizeOf for servo_arc::Arc { ++impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) } } +-impl MallocConditionalShallowSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] - impl MallocConditionalShallowSizeOf for servo_arc::Arc { ++impl MallocConditionalShallowSizeOf for Arc { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if ops.have_seen_ptr(self.heap_ptr()) { + if ops.have_seen_ptr(arc_ptr(self)) { @@ -379,8 +382,9 @@ index 778082b5f0..4c766f609b 100644 } } +-impl MallocConditionalSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] - impl MallocConditionalSizeOf for servo_arc::Arc { ++impl MallocConditionalSizeOf for Arc { fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if ops.have_seen_ptr(self.heap_ptr()) { + if ops.have_seen_ptr(arc_ptr(self)) { diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index 414b00176..eafba2357 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -141,6 +141,7 @@ impl> DerefMut for Memzero { } } +#[cfg(std)] #[cfg(test)] mod test { use std::sync::Arc; diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 4c766f609..7f527c930 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -58,7 +58,7 @@ mod std { } #[cfg(feature = "std")] -use std::sync as servo_arc; +use std::sync::Arc; use std::hash::{BuildHasher, Hash}; use std::mem::size_of; @@ -481,7 +481,7 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocShallowSizeOf for Arc { } #[cfg(feature = "std")] -fn arc_ptr(s: &servo_arc::Arc) -> * const T { +fn arc_ptr(s: &Arc) -> * const T { &(**s) as *const T } @@ -490,7 +490,7 @@ fn arc_ptr(s: &servo_arc::Arc) -> * const T { #[cfg(feature = "std")] #[cfg(not(feature = "estimate-heapsize"))] #[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] -impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { +impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(arc_ptr(self)) } } @@ -499,7 +499,7 @@ impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { #[cfg(feature = "std")] #[cfg(not(feature = "estimate-heapsize"))] #[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { +impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { size_of::() } @@ -507,14 +507,14 @@ impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { #[cfg(feature = "std")] -impl MallocUnconditionalSizeOf for servo_arc::Arc { +impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) } } #[cfg(feature = "std")] -impl MallocConditionalShallowSizeOf for servo_arc::Arc { +impl MallocConditionalShallowSizeOf for Arc { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if ops.have_seen_ptr(arc_ptr(self)) { 0 @@ -525,7 +525,7 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } #[cfg(feature = "std")] -impl MallocConditionalSizeOf for servo_arc::Arc { +impl MallocConditionalSizeOf for Arc { fn conditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if ops.have_seen_ptr(arc_ptr(self)) { 0 diff --git a/parity-util-mem/src/sizeof.rs b/parity-util-mem/src/sizeof.rs index 2b8522546..405063d8a 100644 --- a/parity-util-mem/src/sizeof.rs +++ b/parity-util-mem/src/sizeof.rs @@ -17,8 +17,6 @@ //! Estimation for heapsize calculation. Usable to replace call to allocator method (for some //! allocators or simply because we just need a deterministic cunsumption measurement). -#[cfg(feature = "serde_only")] -extern crate serde_bytes; use crate::malloc_size::{ MallocSizeOf, @@ -27,8 +25,6 @@ use crate::malloc_size::{ MallocSizeOfOps }; use std::mem::{size_of, size_of_val}; -#[cfg(feature = "serde_only")] -use self::serde_bytes::ByteBuf; impl MallocShallowSizeOf for Box { fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { @@ -42,13 +38,6 @@ impl MallocSizeOf for String { } } -#[cfg(feature = "serde_only")] -impl MallocShallowSizeOf for ByteBuf { - fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - AsRef::>::as_ref(self).capacity() * size_of::() - } -} - impl MallocShallowSizeOf for Vec { fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { self.capacity() * size_of::() From 73e8c7e48a9756dd6b3a11272ea945138bc9a288 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jan 2019 14:32:38 +0100 Subject: [PATCH 21/37] Add missing version --- parity-util-mem/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index 068b42fc9..f06c169f1 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -9,7 +9,7 @@ license = "GPL-3.0" [dependencies] clear_on_drop = "0.2" cfg-if = "0.1.6" -malloc_size_of_derive = { path = "./malloc_size_of_derive" } +malloc_size_of_derive = { version = "0.0.1", path = "./malloc_size_of_derive" } dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } From 64135ac0aaf960f46ac0ee843984ae9067441e8a Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sat, 12 Jan 2019 17:43:16 +0100 Subject: [PATCH 22/37] Use malloc_size_of_derive from crates. --- parity-util-mem/Cargo.toml | 2 +- parity-util-mem/get_malloc_size_src.sh | 1 - .../malloc_size_of_derive/Cargo.toml | 15 -- .../malloc_size_of_derive/LICENSE-APACHE | 201 ------------------ .../malloc_size_of_derive/LICENSE-MIT | 23 -- parity-util-mem/malloc_size_of_derive/lib.rs | 125 ----------- 6 files changed, 1 insertion(+), 366 deletions(-) delete mode 100644 parity-util-mem/malloc_size_of_derive/Cargo.toml delete mode 100644 parity-util-mem/malloc_size_of_derive/LICENSE-APACHE delete mode 100644 parity-util-mem/malloc_size_of_derive/LICENSE-MIT delete mode 100644 parity-util-mem/malloc_size_of_derive/lib.rs diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f06c169f1..f34677aa2 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -9,7 +9,7 @@ license = "GPL-3.0" [dependencies] clear_on_drop = "0.2" cfg-if = "0.1.6" -malloc_size_of_derive = { version = "0.0.1", path = "./malloc_size_of_derive" } +malloc_size_of_derive = "0.1.0" dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } jemallocator = { version = "0.1", optional = true } diff --git a/parity-util-mem/get_malloc_size_src.sh b/parity-util-mem/get_malloc_size_src.sh index e717d6f4b..6eb52131f 100755 --- a/parity-util-mem/get_malloc_size_src.sh +++ b/parity-util-mem/get_malloc_size_src.sh @@ -8,6 +8,5 @@ git checkout 5bdea7dc1c80790a852a3fb03edfb2b8fbd403dc git apply ../slim_malloc_size_of.patch #git merge master #cp components/malloc_size_of/lib.rs ../src/malloc_size.rs -#cp -r components/malloc_size_of_derive .. #cd .. #rm -rf ./servo diff --git a/parity-util-mem/malloc_size_of_derive/Cargo.toml b/parity-util-mem/malloc_size_of_derive/Cargo.toml deleted file mode 100644 index 4cfc2ee12..000000000 --- a/parity-util-mem/malloc_size_of_derive/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "malloc_size_of_derive" -version = "0.0.1" -authors = ["The Servo Project Developers"] -license = "MIT/Apache-2.0" -publish = false - -[lib] -path = "lib.rs" -proc-macro = true - -[dependencies] -proc-macro2 = "0.4" -syn = { version = "0.15", features = ["full"] } -synstructure = "0.10" diff --git a/parity-util-mem/malloc_size_of_derive/LICENSE-APACHE b/parity-util-mem/malloc_size_of_derive/LICENSE-APACHE deleted file mode 100644 index 16fe87b06..000000000 --- a/parity-util-mem/malloc_size_of_derive/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/parity-util-mem/malloc_size_of_derive/LICENSE-MIT b/parity-util-mem/malloc_size_of_derive/LICENSE-MIT deleted file mode 100644 index 31aa79387..000000000 --- a/parity-util-mem/malloc_size_of_derive/LICENSE-MIT +++ /dev/null @@ -1,23 +0,0 @@ -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/parity-util-mem/malloc_size_of_derive/lib.rs b/parity-util-mem/malloc_size_of_derive/lib.rs deleted file mode 100644 index 0838f6cfb..000000000 --- a/parity-util-mem/malloc_size_of_derive/lib.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2016-2017 The Servo Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! A crate for deriving the MallocSizeOf trait. - -extern crate proc_macro2; -#[macro_use] -extern crate syn; -#[macro_use] -extern crate synstructure; - -#[cfg(not(test))] -decl_derive!([MallocSizeOf, attributes(ignore_malloc_size_of)] => malloc_size_of_derive); - -fn malloc_size_of_derive(s: synstructure::Structure) -> proc_macro2::TokenStream { - let match_body = s.each(|binding| { - let ignore = binding - .ast() - .attrs - .iter() - .any(|attr| match attr.interpret_meta().unwrap() { - syn::Meta::Word(ref ident) | syn::Meta::List(syn::MetaList { ref ident, .. }) - if ident == "ignore_malloc_size_of" => - { - panic!( - "#[ignore_malloc_size_of] should have an explanation, \ - e.g. #[ignore_malloc_size_of = \"because reasons\"]" - ); - } - syn::Meta::NameValue(syn::MetaNameValue { ref ident, .. }) - if ident == "ignore_malloc_size_of" => - { - true - }, - _ => false, - }); - if ignore { - None - } else if let syn::Type::Array(..) = binding.ast().ty { - Some(quote! { - for item in #binding.iter() { - sum += ::malloc_size_of::MallocSizeOf::size_of(item, ops); - } - }) - } else { - Some(quote! { - sum += ::malloc_size_of::MallocSizeOf::size_of(#binding, ops); - }) - } - }); - - let ast = s.ast(); - let name = &ast.ident; - let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl(); - let mut where_clause = where_clause.unwrap_or(&parse_quote!(where)).clone(); - for param in ast.generics.type_params() { - let ident = ¶m.ident; - where_clause - .predicates - .push(parse_quote!(#ident: ::malloc_size_of::MallocSizeOf)); - } - - let tokens = quote! { - impl #impl_generics ::malloc_size_of::MallocSizeOf for #name #ty_generics #where_clause { - #[inline] - #[allow(unused_variables, unused_mut, unreachable_code)] - fn size_of(&self, ops: &mut ::malloc_size_of::MallocSizeOfOps) -> usize { - let mut sum = 0; - match *self { - #match_body - } - sum - } - } - }; - - tokens -} - -#[test] -fn test_struct() { - let source = syn::parse_str( - "struct Foo { bar: Bar, baz: T, #[ignore_malloc_size_of = \"\"] z: Arc }", - ) - .unwrap(); - let source = synstructure::Structure::new(&source); - - let expanded = malloc_size_of_derive(source).to_string(); - let mut no_space = expanded.replace(" ", ""); - macro_rules! match_count { - ($e: expr, $count: expr) => { - assert_eq!( - no_space.matches(&$e.replace(" ", "")).count(), - $count, - "counting occurences of {:?} in {:?} (whitespace-insensitive)", - $e, - expanded - ) - }; - } - match_count!("struct", 0); - match_count!("ignore_malloc_size_of", 0); - match_count!("impl ::malloc_size_of::MallocSizeOf for Foo where T: ::malloc_size_of::MallocSizeOf {", 1); - match_count!("sum += ::malloc_size_of::MallocSizeOf::size_of(", 2); - - let source = syn::parse_str("struct Bar([Baz; 3]);").unwrap(); - let source = synstructure::Structure::new(&source); - let expanded = malloc_size_of_derive(source).to_string(); - no_space = expanded.replace(" ", ""); - match_count!("for item in", 1); -} - -#[should_panic(expected = "should have an explanation")] -#[test] -fn test_no_reason() { - let input = syn::parse_str("struct A { #[ignore_malloc_size_of] b: C }").unwrap(); - malloc_size_of_derive(synstructure::Structure::new(&input)); -} From 0437f7e63c8c38d980bc66f85ac2bd8a3ef1596b Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jan 2019 09:57:11 +0100 Subject: [PATCH 23/37] Rem heapsize constraint and deprecate impl. Add malloc_size_of impl. --- memorydb/Cargo.toml | 3 ++- memorydb/src/lib.rs | 19 ++++++++++++++++--- patricia_trie/Cargo.toml | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/memorydb/Cargo.toml b/memorydb/Cargo.toml index 91496c484..dbbc2d4db 100644 --- a/memorydb/Cargo.toml +++ b/memorydb/Cargo.toml @@ -1,12 +1,13 @@ [package] name = "memorydb" -version = "0.3.0" +version = "0.4.0" authors = ["Parity Technologies "] description = "in-memory implementation of hashdb" repository = "https://github.com/paritytech/parity-common" license = "GPL-3.0" [dependencies] +parity-util-mem = { version = "0.1", path = "../parity-util-mem" } heapsize = "0.4" hashdb = { version = "0.3.0", path = "../hashdb" } plain_hasher = { version = "0.2", path = "../plain_hasher", default-features = false } diff --git a/memorydb/src/lib.rs b/memorydb/src/lib.rs index 45678578e..64285bec6 100644 --- a/memorydb/src/lib.rs +++ b/memorydb/src/lib.rs @@ -16,6 +16,7 @@ //! Reference-counted memory-based `HashDB` implementation. extern crate hashdb; +extern crate parity_util_mem as malloc_size_of; extern crate heapsize; extern crate rlp; #[cfg(test)] extern crate keccak_hasher; @@ -23,6 +24,7 @@ extern crate rlp; #[cfg(test)] extern crate ethereum_types; use hashdb::{HashDB, Hasher as KeyHasher, AsHashDB}; +use malloc_size_of::{MallocSizeOf, MallocSizeOfOps}; use heapsize::HeapSizeOf; use rlp::NULL_RLP; use std::collections::hash_map::Entry; @@ -91,7 +93,6 @@ pub struct MemoryDB { impl<'a, H, T> Default for MemoryDB where H: KeyHasher, - H::Out: HeapSizeOf, T: From<&'a [u8]> + Clone { fn default() -> Self { Self::new() } @@ -100,7 +101,6 @@ where impl<'a, H, T> MemoryDB where H: KeyHasher, - H::Out: HeapSizeOf, T: From<&'a [u8]> + Clone, { /// Create a new instance of the memory DB. @@ -112,7 +112,6 @@ where impl MemoryDB where H: KeyHasher, - H::Out: HeapSizeOf, T: Default, { /// Remove an element and delete it from storage if reference count reaches zero. @@ -220,12 +219,26 @@ where H::Out: HeapSizeOf, T: HeapSizeOf, { + #[deprecated(since="0.4.0", note="please use `malloc_size_of` instead")] /// Returns the size of allocated heap memory pub fn mem_used(&self) -> usize { self.data.heap_size_of_children() } } +impl MallocSizeOf for MemoryDB +where +H: KeyHasher, + H::Out: MallocSizeOf, + T: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.data.size_of(ops) + + self.null_node_data.size_of(ops) + + self.hashed_null_node.size_of(ops) + } +} + impl HashDB for MemoryDB where H: KeyHasher, diff --git a/patricia_trie/Cargo.toml b/patricia_trie/Cargo.toml index 1906601c9..d7e089b78 100644 --- a/patricia_trie/Cargo.toml +++ b/patricia_trie/Cargo.toml @@ -17,7 +17,7 @@ parity-bytes = { version = "0.1", path = "../parity-bytes" } env_logger = "0.5" ethereum-types = "0.4" keccak-hash = { version = "0.1", path = "../keccak-hash" } -memorydb = { version = "0.3.0", path = "../memorydb", default-features = false } +memorydb = { version = "0.4.0", path = "../memorydb", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } trie-standardmap = { version = "0.1", path = "../trie-standardmap", default-features = false } triehash = { version = "0.3.0", path = "../triehash", default-features = false } From 81b2da10d435e710d41dc792b223c84d192028a8 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jan 2019 11:11:52 +0100 Subject: [PATCH 24/37] Remove path reference --- memorydb/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memorydb/Cargo.toml b/memorydb/Cargo.toml index dbbc2d4db..8e21bd0e9 100644 --- a/memorydb/Cargo.toml +++ b/memorydb/Cargo.toml @@ -9,7 +9,7 @@ license = "GPL-3.0" [dependencies] parity-util-mem = { version = "0.1", path = "../parity-util-mem" } heapsize = "0.4" -hashdb = { version = "0.3.0", path = "../hashdb" } +hashdb = { version = "0.3.0" } plain_hasher = { version = "0.2", path = "../plain_hasher", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } From 6c4a5fd3fa39b8c9acaf3b67a6607b3a16457c02 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jan 2019 17:39:57 +0100 Subject: [PATCH 25/37] Remove path for hashdb to avoid trait conflicts. --- patricia_trie/Cargo.toml | 4 ++-- test-support/keccak-hasher/Cargo.toml | 2 +- test-support/patricia-trie-ethereum/Cargo.toml | 2 +- triehash/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/patricia_trie/Cargo.toml b/patricia_trie/Cargo.toml index d7e089b78..3c9d636ff 100644 --- a/patricia_trie/Cargo.toml +++ b/patricia_trie/Cargo.toml @@ -10,14 +10,14 @@ license = "GPL-3.0" elastic-array = "0.10" log = "0.3" rand = "0.4" -hashdb = { version = "0.3.0", path = "../hashdb" } +hashdb = { version = "0.3.0" } parity-bytes = { version = "0.1", path = "../parity-bytes" } [dev-dependencies] env_logger = "0.5" ethereum-types = "0.4" keccak-hash = { version = "0.1", path = "../keccak-hash" } -memorydb = { version = "0.4.0", path = "../memorydb", default-features = false } +memorydb = { version = "0.3.0", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } trie-standardmap = { version = "0.1", path = "../trie-standardmap", default-features = false } triehash = { version = "0.3.0", path = "../triehash", default-features = false } diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml index 53d07d006..56bf60dbb 100644 --- a/test-support/keccak-hasher/Cargo.toml +++ b/test-support/keccak-hasher/Cargo.toml @@ -9,5 +9,5 @@ license = "GPL-3.0" [dependencies] ethereum-types = "0.4" tiny-keccak = "1.4.2" -hashdb = { version = "0.3.0", path = "../../hashdb" } +hashdb = { version = "0.3.0" } plain_hasher = { path = "../../plain_hasher" } diff --git a/test-support/patricia-trie-ethereum/Cargo.toml b/test-support/patricia-trie-ethereum/Cargo.toml index 1f987b945..6f1c48430 100644 --- a/test-support/patricia-trie-ethereum/Cargo.toml +++ b/test-support/patricia-trie-ethereum/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] patricia-trie = { version = "0.3.0", path = "../../patricia_trie" } keccak-hasher = { version = "0.1", path = "../keccak-hasher" } -hashdb = { version = "0.3.0", path = "../../hashdb" } +hashdb = { version = "0.3.0" } rlp = { version = "0.3.0", path = "../../rlp" } parity-bytes = { version = "0.1", path = "../../parity-bytes" } diff --git a/triehash/Cargo.toml b/triehash/Cargo.toml index 00f03bee5..27d1fc24e 100644 --- a/triehash/Cargo.toml +++ b/triehash/Cargo.toml @@ -7,7 +7,7 @@ repository = "https://github.com/paritytech/parity-common" license = "GPL-3.0" [dependencies] -hashdb = { version = "0.3.0", path = "../hashdb", default-features = false } +hashdb = { version = "0.3.0", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } [dev-dependencies] From a9ca6e1b46bc9d8d3b9e96eb068e467d67acaf70 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 15:01:13 +0100 Subject: [PATCH 26/37] Restore path in cargo --- memorydb/Cargo.toml | 2 +- patricia_trie/Cargo.toml | 4 ++-- test-support/keccak-hasher/Cargo.toml | 2 +- triehash/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/memorydb/Cargo.toml b/memorydb/Cargo.toml index 8e21bd0e9..dbbc2d4db 100644 --- a/memorydb/Cargo.toml +++ b/memorydb/Cargo.toml @@ -9,7 +9,7 @@ license = "GPL-3.0" [dependencies] parity-util-mem = { version = "0.1", path = "../parity-util-mem" } heapsize = "0.4" -hashdb = { version = "0.3.0" } +hashdb = { version = "0.3.0", path = "../hashdb" } plain_hasher = { version = "0.2", path = "../plain_hasher", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } diff --git a/patricia_trie/Cargo.toml b/patricia_trie/Cargo.toml index 3c9d636ff..1906601c9 100644 --- a/patricia_trie/Cargo.toml +++ b/patricia_trie/Cargo.toml @@ -10,14 +10,14 @@ license = "GPL-3.0" elastic-array = "0.10" log = "0.3" rand = "0.4" -hashdb = { version = "0.3.0" } +hashdb = { version = "0.3.0", path = "../hashdb" } parity-bytes = { version = "0.1", path = "../parity-bytes" } [dev-dependencies] env_logger = "0.5" ethereum-types = "0.4" keccak-hash = { version = "0.1", path = "../keccak-hash" } -memorydb = { version = "0.3.0", default-features = false } +memorydb = { version = "0.3.0", path = "../memorydb", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } trie-standardmap = { version = "0.1", path = "../trie-standardmap", default-features = false } triehash = { version = "0.3.0", path = "../triehash", default-features = false } diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml index 56bf60dbb..53d07d006 100644 --- a/test-support/keccak-hasher/Cargo.toml +++ b/test-support/keccak-hasher/Cargo.toml @@ -9,5 +9,5 @@ license = "GPL-3.0" [dependencies] ethereum-types = "0.4" tiny-keccak = "1.4.2" -hashdb = { version = "0.3.0" } +hashdb = { version = "0.3.0", path = "../../hashdb" } plain_hasher = { path = "../../plain_hasher" } diff --git a/triehash/Cargo.toml b/triehash/Cargo.toml index 27d1fc24e..00f03bee5 100644 --- a/triehash/Cargo.toml +++ b/triehash/Cargo.toml @@ -7,7 +7,7 @@ repository = "https://github.com/paritytech/parity-common" license = "GPL-3.0" [dependencies] -hashdb = { version = "0.3.0", default-features = false } +hashdb = { version = "0.3.0", path = "../hashdb", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } [dev-dependencies] From c315173078f3d0604b24ebda904475e4ade30563 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 15:07:08 +0100 Subject: [PATCH 27/37] Missing path --- patricia_trie/Cargo.toml | 2 +- test-support/patricia-trie-ethereum/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/patricia_trie/Cargo.toml b/patricia_trie/Cargo.toml index 1906601c9..d7e089b78 100644 --- a/patricia_trie/Cargo.toml +++ b/patricia_trie/Cargo.toml @@ -17,7 +17,7 @@ parity-bytes = { version = "0.1", path = "../parity-bytes" } env_logger = "0.5" ethereum-types = "0.4" keccak-hash = { version = "0.1", path = "../keccak-hash" } -memorydb = { version = "0.3.0", path = "../memorydb", default-features = false } +memorydb = { version = "0.4.0", path = "../memorydb", default-features = false } rlp = { version = "0.3.0", path = "../rlp", default-features = false } trie-standardmap = { version = "0.1", path = "../trie-standardmap", default-features = false } triehash = { version = "0.3.0", path = "../triehash", default-features = false } diff --git a/test-support/patricia-trie-ethereum/Cargo.toml b/test-support/patricia-trie-ethereum/Cargo.toml index 6f1c48430..1f987b945 100644 --- a/test-support/patricia-trie-ethereum/Cargo.toml +++ b/test-support/patricia-trie-ethereum/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] patricia-trie = { version = "0.3.0", path = "../../patricia_trie" } keccak-hasher = { version = "0.1", path = "../keccak-hasher" } -hashdb = { version = "0.3.0" } +hashdb = { version = "0.3.0", path = "../../hashdb" } rlp = { version = "0.3.0", path = "../../rlp" } parity-bytes = { version = "0.1", path = "../../parity-bytes" } From 71f7aa8e62f7d3c88e8b8c9f665d8f3773b680ce Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 15:27:54 +0100 Subject: [PATCH 28/37] Added test --- parity-util-mem/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index eafba2357..b289d16e3 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -141,12 +141,13 @@ impl> DerefMut for Memzero { } } -#[cfg(std)] #[cfg(test)] mod test { + #[cfg(std)] use std::sync::Arc; use super::MallocSizeOfExt; - + + #[cfg(std)] #[test] fn test_arc() { let val = Arc::new("test".to_string()); @@ -154,4 +155,10 @@ mod test { assert!(s > 0); } + #[test] + fn test_call() { + let val: Vec = vec![0;1]; + let s = val.malloc_size_of(); + assert!(s > 0); + } } From b7bcbc80cff88ab56572c6bd0c5a0eba9c519336 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 15:29:29 +0100 Subject: [PATCH 29/37] fix test --- parity-util-mem/src/lib.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index b289d16e3..254d668d0 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -141,24 +141,16 @@ impl> DerefMut for Memzero { } } +#[cfg(feature = "std")] #[cfg(test)] mod test { - #[cfg(std)] use std::sync::Arc; use super::MallocSizeOfExt; - #[cfg(std)] #[test] fn test_arc() { let val = Arc::new("test".to_string()); let s = val.malloc_size_of(); assert!(s > 0); } - - #[test] - fn test_call() { - let val: Vec = vec![0;1]; - let s = val.malloc_size_of(); - assert!(s > 0); - } } From 5e53bf3f9e73fdc7f3eb1dc48a8a1678f8131da9 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 16:02:11 +0100 Subject: [PATCH 30/37] Do not use prefixed on mac by default (same with other platform as we can activate prefix with feature). --- parity-util-mem/src/allocators.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index ca8cea7b3..025152f83 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -98,7 +98,7 @@ cfg_if! { /// Macos, ios and android calls jemalloc. /// Linux call system allocator (currently malloc). extern "C" { - #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] + #[cfg_attr(any(prefixed_jemalloc), link_name = "je_malloc_usable_size")] pub fn malloc_usable_size(ptr: *const c_void) -> usize; } From b6674365b28d19c5f52be875d7dfd0abf996a5e4 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 16:58:34 +0100 Subject: [PATCH 31/37] Have macos rely upon estimate if default allocator --- .travis.yml | 1 + parity-util-mem/src/allocators.rs | 11 +++++++---- parity-util-mem/src/lib.rs | 9 +++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 505bf1a62..ca93899ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,3 +25,4 @@ script: - cd hashdb/ && cargo test --no-default-features && cd .. - cd plain_hasher/ && cargo test --no-default-features && cd .. - cd parity-util-mem/ && cargo test --features=estimate-heapsize && cd .. + - cd parity-util-mem/ && cargo test --features=jemalloc-global && cd .. diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 025152f83..984532437 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -92,16 +92,19 @@ cfg_if! { jemallocator::usable_size(ptr) } - } else { + } else if #[cfg(target_os = "linux")] { - // default allocator used - /// Macos, ios and android calls jemalloc. /// Linux call system allocator (currently malloc). extern "C" { - #[cfg_attr(any(prefixed_jemalloc), link_name = "je_malloc_usable_size")] pub fn malloc_usable_size(ptr: *const c_void) -> usize; } + } else { + // default allocator for non linux or windows system use estimate + pub unsafe extern "C" fn malloc_usable_size(_ptr: *const c_void) -> usize { + unreachable!("estimate heapsize or feature allocator needed") + } + } } diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index 254d668d0..e63d1ece6 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -22,6 +22,12 @@ #![cfg_attr(not(feature = "std"), feature(core_intrinsics))] #![cfg_attr(not(feature = "std"), feature(alloc))] +// no direct call in macos system allocator +#![cfg_attr(all( + target_os = "macos", + not(feature = "jemalloc-global") + ), feature(estimate-heapsize))] + #[macro_use] extern crate cfg_if; @@ -43,7 +49,6 @@ pub use cod::clear::Clear; cfg_if! { if #[cfg(all( - feature = "jemalloc-global", feature = "jemalloc-global", not(target_os = "windows"), not(target_arch = "wasm32") @@ -146,7 +151,7 @@ impl> DerefMut for Memzero { mod test { use std::sync::Arc; use super::MallocSizeOfExt; - + #[test] fn test_arc() { let val = Arc::new("test".to_string()); From 85a763e0cbf0190369152ef0d3dddab34a896681 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 17:40:01 +0100 Subject: [PATCH 32/37] Need full test for macos without jemalloc --- parity-util-mem/slim_malloc_size_of.patch | 150 +++++++++++++--------- parity-util-mem/src/allocators.rs | 5 + parity-util-mem/src/lib.rs | 13 +- parity-util-mem/src/malloc_size.rs | 78 +++++------ 4 files changed, 144 insertions(+), 102 deletions(-) diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index 61381c146..30c56b7c5 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,5 +1,5 @@ diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..7f527c930e 100644 +index 778082b5f0..e13745d6af 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,55 +43,39 @@ @@ -82,23 +82,67 @@ index 778082b5f0..7f527c930e 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -216,6 +200,7 @@ pub trait MallocConditionalShallowSizeOf { +@@ -216,44 +200,62 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocSizeOf for String { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -229,6 +214,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { +-impl MallocSizeOf for String { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.as_ptr()) } ++#[cfg(not(any( ++ all( ++ target_os = "macos", ++ not(feature = "jemalloc-global"), ++ ), ++ feature = "estimate-heapsize" ++)))] ++pub mod inner_allocator_use { ++ ++use super::*; ++ ++impl MallocShallowSizeOf for Box { ++ fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(&**self) } + } + } + +-impl<'a, T: ?Sized> MallocSizeOf for &'a T { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- // Zero makes sense for a non-owning reference. +- 0 ++impl MallocShallowSizeOf for Vec { ++ fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(self.as_ptr()) } + } + } + +-impl MallocShallowSizeOf for Box { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(&**self) } ++// currently this seems only fine with jemalloc ++#[cfg(feature = "std")] ++#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] ++impl MallocUnconditionalShallowSizeOf for Arc { ++ fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(arc_ptr(self)) } } } -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocShallowSizeOf for Box { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(&**self) } -@@ -241,24 +227,6 @@ impl MallocSizeOf for Box { +-impl MallocSizeOf for Box { ++#[cfg(feature = "std")] ++#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] ++impl MallocUnconditionalShallowSizeOf for Arc { ++ fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { ++ size_of::() ++ } ++} ++ ++} ++ ++impl MallocSizeOf for String { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.shallow_size_of(ops) + (**self).size_of(ops) ++ unsafe { ops.malloc_size_of(self.as_ptr()) } } } @@ -111,19 +155,19 @@ index 778082b5f0..7f527c930e 100644 - n += ops.malloc_size_of(&**self); - } - n -- } --} -- ++impl<'a, T: ?Sized> MallocSizeOf for &'a T { ++ fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { ++ // Zero makes sense for a non-owning reference. ++ 0 + } + } + -impl MallocSizeOf for thin_slice::ThinBoxedSlice { -- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { -- self.shallow_size_of(ops) + (**self).size_of(ops) -- } --} -- - impl MallocSizeOf for () { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 -@@ -329,6 +297,7 @@ impl MallocSizeOf for std::cell::RefCell { ++impl MallocSizeOf for Box { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.shallow_size_of(ops) + (**self).size_of(ops) + } +@@ -329,6 +331,7 @@ impl MallocSizeOf for std::cell::RefCell { } } @@ -131,7 +175,7 @@ index 778082b5f0..7f527c930e 100644 impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> where B::Owned: MallocSizeOf, -@@ -351,24 +320,7 @@ impl MallocSizeOf for [T] { +@@ -351,30 +354,6 @@ impl MallocSizeOf for [T] { } } @@ -153,11 +197,16 @@ index 778082b5f0..7f527c930e 100644 - } -} - -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocShallowSizeOf for Vec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -412,30 +364,7 @@ impl MallocSizeOf for std::collections::VecDeque { +-impl MallocShallowSizeOf for Vec { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.as_ptr()) } +- } +-} +- + impl MallocSizeOf for Vec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); +@@ -412,30 +391,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } @@ -189,7 +238,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocShallowSizeOf for std::collections::HashSet where T: Eq + Hash, -@@ -457,6 +386,7 @@ where +@@ -457,6 +413,7 @@ where } } @@ -197,7 +246,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocSizeOf for std::collections::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -471,59 +401,7 @@ where +@@ -471,59 +428,7 @@ where } } @@ -258,7 +307,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocShallowSizeOf for std::collections::HashMap where K: Eq + Hash, -@@ -541,6 +419,7 @@ where +@@ -541,6 +446,7 @@ where } } @@ -266,7 +315,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocSizeOf for std::collections::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -587,62 +466,6 @@ where +@@ -587,62 +493,6 @@ where } } @@ -329,40 +378,21 @@ index 778082b5f0..7f527c930e 100644 // PhantomData is always 0. impl MallocSizeOf for std::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -657,21 +480,43 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,21 +507,22 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } -impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { +- fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.heap_ptr()) } +- } +#[cfg(feature = "std")] +fn arc_ptr(s: &Arc) -> * const T { + &(**s) as *const T -+} -+ -+ -+// currently this seems only fine with jemalloc -+#[cfg(feature = "std")] -+#[cfg(not(feature = "estimate-heapsize"))] -+#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] -+impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { -- unsafe { ops.malloc_size_of(self.heap_ptr()) } -+ unsafe { ops.malloc_size_of(arc_ptr(self)) } - } } -impl MallocUnconditionalSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] -+#[cfg(not(feature = "estimate-heapsize"))] -+#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -+impl MallocUnconditionalShallowSizeOf for Arc { -+ fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -+ size_of::() -+ } -+} -+ -+ -+#[cfg(feature = "std")] +impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) @@ -378,7 +408,7 @@ index 778082b5f0..7f527c930e 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -679,9 +524,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +530,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } @@ -391,7 +421,7 @@ index 778082b5f0..7f527c930e 100644 0 } else { self.unconditional_size_of(ops) -@@ -695,203 +541,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { +@@ -695,203 +547,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. @@ -596,7 +626,7 @@ index 778082b5f0..7f527c930e 100644 #[macro_export] macro_rules! malloc_size_of_is_0( ($($ty:ty),+) => ( -@@ -929,117 +585,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,117 +591,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index 984532437..6f63b89fb 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -26,6 +26,11 @@ //! - jemalloc: use jemallocator crate //! - weealloc: default to `estimate_size` //! - dlmalloc: default to `estimate_size` +//! - arch x86/macos: +//! - no features: use default alloc, requires using `estimate_size` +//! - jemalloc: use jemallocator crate +//! - weealloc: default to `estimate_size` +//! - dlmalloc: default to `estimate_size` //! - arch wasm32: //! - no features: default to `estimate_size` //! - weealloc: default to `estimate_size` diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index e63d1ece6..a07ff5c57 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -22,11 +22,6 @@ #![cfg_attr(not(feature = "std"), feature(core_intrinsics))] #![cfg_attr(not(feature = "std"), feature(alloc))] -// no direct call in macos system allocator -#![cfg_attr(all( - target_os = "macos", - not(feature = "jemalloc-global") - ), feature(estimate-heapsize))] #[macro_use] extern crate cfg_if; @@ -74,7 +69,13 @@ cfg_if! { pub mod allocators; -#[cfg(feature = "estimate-heapsize")] +#[cfg(any( + all( + target_os = "macos", + not(feature = "jemalloc-global"), + ), + feature = "estimate-heapsize" +))] pub mod sizeof; #[cfg(not(feature = "std"))] diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 7f527c930..e13745d6a 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -200,7 +200,48 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(not(any( + all( + target_os = "macos", + not(feature = "jemalloc-global"), + ), + feature = "estimate-heapsize" +)))] +pub mod inner_allocator_use { + +use super::*; + +impl MallocShallowSizeOf for Box { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(&**self) } + } +} + +impl MallocShallowSizeOf for Vec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } +} + +// currently this seems only fine with jemalloc +#[cfg(feature = "std")] +#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(arc_ptr(self)) } + } +} + +#[cfg(feature = "std")] +#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } +} + +} + impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -214,13 +255,6 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } -#[cfg(not(feature = "estimate-heapsize"))] -impl MallocShallowSizeOf for Box { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(&**self) } - } -} - impl MallocSizeOf for Box { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.shallow_size_of(ops) + (**self).size_of(ops) @@ -320,13 +354,6 @@ impl MallocSizeOf for [T] { } } -#[cfg(not(feature = "estimate-heapsize"))] -impl MallocShallowSizeOf for Vec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } -} - impl MallocSizeOf for Vec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); @@ -485,27 +512,6 @@ fn arc_ptr(s: &Arc) -> * const T { &(**s) as *const T } - -// currently this seems only fine with jemalloc -#[cfg(feature = "std")] -#[cfg(not(feature = "estimate-heapsize"))] -#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] -impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(arc_ptr(self)) } - } -} - -#[cfg(feature = "std")] -#[cfg(not(feature = "estimate-heapsize"))] -#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - size_of::() - } -} - - #[cfg(feature = "std")] impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { From 36822705d3a8f1e0d1026bf8020dd7971a956528 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 18:04:41 +0100 Subject: [PATCH 33/37] String in internal heap measured impl --- parity-util-mem/src/malloc_size.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index e13745d6a..a88ebc3f0 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -240,14 +240,14 @@ impl MallocUnconditionalShallowSizeOf for Arc { } } -} - impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } } } +} + impl<'a, T: ?Sized> MallocSizeOf for &'a T { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { // Zero makes sense for a non-owning reference. From 566389ec948f995e40bda1c37e2b7a58a15db256 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Jan 2019 19:04:37 +0100 Subject: [PATCH 34/37] Exclude jmalloc-sys on windows build (ci related) --- parity-util-mem/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f34677aa2..f618007e5 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -12,7 +12,6 @@ cfg-if = "0.1.6" malloc_size_of_derive = "0.1.0" dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } -jemallocator = { version = "0.1", optional = true } elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } @@ -21,6 +20,10 @@ parking_lot = { version = "*", optional = true } [target."cfg(windows)".dependencies.winapi] version = "0.3.4" +[target."cfg(not(windows))".dependencies.jemallocator] +version = "0.1" +optional = true + [features] default = ["std", "ethereum-impls"] std = [] From 08f9ed17eb089c51d037d095aea271d9df7e4bc0 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Jan 2019 12:35:31 +0100 Subject: [PATCH 35/37] Proper windows checks in cargo.toml --- parity-util-mem/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f618007e5..6293d6348 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -17,10 +17,10 @@ elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } parking_lot = { version = "*", optional = true } -[target."cfg(windows)".dependencies.winapi] +[target.'cfg(target_os = "windows")'.dependencies.winapi] version = "0.3.4" -[target."cfg(not(windows))".dependencies.jemallocator] +[target.'cfg(not(target_os = "windows"))'.dependencies.jemallocator] version = "0.1" optional = true From 18565979d9fb5e61694cfc3a1e2948a7ed306eed Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Jan 2019 15:13:15 +0100 Subject: [PATCH 36/37] Removing 'prefixed_malloc' from condition. --- parity-util-mem/slim_malloc_size_of.patch | 4 ++-- parity-util-mem/src/malloc_size.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index 30c56b7c5..7c31cf606 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -121,7 +121,7 @@ index 778082b5f0..e13745d6af 100644 - unsafe { ops.malloc_size_of(&**self) } +// currently this seems only fine with jemalloc +#[cfg(feature = "std")] -+#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] ++#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(arc_ptr(self)) } @@ -130,7 +130,7 @@ index 778082b5f0..e13745d6af 100644 -impl MallocSizeOf for Box { +#[cfg(feature = "std")] -+#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] ++#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index a88ebc3f0..8cb5edd2f 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -225,7 +225,7 @@ impl MallocShallowSizeOf for Vec { // currently this seems only fine with jemalloc #[cfg(feature = "std")] -#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(arc_ptr(self)) } @@ -233,7 +233,7 @@ impl MallocUnconditionalShallowSizeOf for Arc { } #[cfg(feature = "std")] -#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { size_of::() From df840886bf9e4e19a025642b97e09d744956bd66 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Jan 2019 19:07:40 +0100 Subject: [PATCH 37/37] arc only for jemalloc-global, escape for windows ci --- parity-util-mem/src/malloc_size.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 8cb5edd2f..ab9b0327b 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -225,7 +225,7 @@ impl MallocShallowSizeOf for Vec { // currently this seems only fine with jemalloc #[cfg(feature = "std")] -#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] +#[cfg(all(feature = "jemalloc-global", not(target_os = "windows")))] impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(arc_ptr(self)) } @@ -233,7 +233,7 @@ impl MallocUnconditionalShallowSizeOf for Arc { } #[cfg(feature = "std")] -#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] +#[cfg(not(all(feature = "jemalloc-global", not(target_os = "windows"))))] impl MallocUnconditionalShallowSizeOf for Arc { fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { size_of::()