From ca52c56e346a2a2bb042bec441b5058df3e3e289 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 12 Dec 2015 23:38:03 +0100 Subject: [PATCH 1/4] Add memchr implemenation based on rust-memchr to libstd --- src/libstd/lib.rs | 2 + src/libstd/memchr.rs | 400 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 src/libstd/memchr.rs diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index 0f8b2f6e17b25..eba0c799cd2c1 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -248,6 +248,7 @@ #![feature(link_args)] #![feature(linkage)] #![feature(macro_reexport)] +#![feature(num_bits_bytes)] #![feature(on_unimplemented)] #![feature(oom)] #![feature(optin_builtin_traits)] @@ -429,6 +430,7 @@ pub mod path; pub mod process; pub mod sync; pub mod time; +mod memchr; #[macro_use] #[path = "sys/common/mod.rs"] mod sys_common; diff --git a/src/libstd/memchr.rs b/src/libstd/memchr.rs new file mode 100644 index 0000000000000..8a2fc57575970 --- /dev/null +++ b/src/libstd/memchr.rs @@ -0,0 +1,400 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// Original implementation taken from rust-memchr +// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch + +use libc::{c_void, c_int, size_t}; + + +/// A safe interface to `memchr`. +/// +/// Returns the index corresponding to the first occurrence of `needle` in +/// `haystack`, or `None` if one is not found. +/// +/// memchr reduces to super-optimized machine code at around an order of +/// magnitude faster than `haystack.iter().position(|&b| b == needle)`. +/// (See benchmarks.) +/// +/// # Example +/// +/// This shows how to find the first position of a byte in a byte string. +/// +/// ```rust +/// use memchr::memchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memchr(b'k', haystack), Some(8)); +/// ``` +pub fn memchr(needle: u8, haystack: &[u8]) -> Option { + // libc memchr + #[cfg(any(not(target_os = "windows"), + not(any(target_pointer_width = "32", + target_pointer_width = "64"))))] + fn memchr_specific(needle: u8, haystack: &[u8]) -> Option { + use libc::memchr as libc_memchr; + + let p = unsafe { + libc_memchr( + haystack.as_ptr() as *const c_void, + needle as c_int, + haystack.len() as size_t) + }; + if p.is_null() { + None + } else { + Some(p as usize - (haystack.as_ptr() as usize)) + } + } + + // use fallback on windows, since it's faster + #[cfg(all(target_os = "windows", + any(target_pointer_width = "32", + target_pointer_width = "64")))] + fn memchr_specific(needle: u8, haystack: &[u8]) -> Option { + fallback::memchr(needle, haystack) + } + + memchr_specific(needle, haystack) +} + +/// A safe interface to `memrchr`. +/// +/// Returns the index corresponding to the last occurrence of `needle` in +/// `haystack`, or `None` if one is not found. +/// +/// # Example +/// +/// This shows how to find the last position of a byte in a byte string. +/// +/// ```rust +/// use memchr::memrchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memrchr(b'o', haystack), Some(17)); +/// ``` +pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { + + #[cfg(target_os = "linux")] + fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { + use libc::memrchr as libc_memrchr; + + // GNU's memrchr() will - unlike memchr() - error if haystack is empty. + if haystack.is_empty() {return None} + let p = unsafe { + libc_memrchr( + haystack.as_ptr() as *const c_void, + needle as c_int, + haystack.len() as size_t) + }; + if p.is_null() { + None + } else { + Some(p as usize - (haystack.as_ptr() as usize)) + } + } + + #[cfg(all(not(target_os = "linux"), + any(target_pointer_width = "32", target_pointer_width = "64")))] + fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { + fallback::memrchr(needle, haystack) + } + + // For the rare case of neither 32 bit nor 64-bit platform. + #[cfg(all(not(target_os = "linux"), + not(target_pointer_width = "32"), + not(target_pointer_width = "64")))] + fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().rposition(|&b| b == needle) + } + + memrchr_specific(needle, haystack) +} + +#[allow(dead_code)] +mod fallback { + use cmp; + use usize; + + const LO_U64: u64 = 0x0101010101010101; + const HI_U64: u64 = 0x8080808080808080; + + // use truncation + const LO_USIZE: usize = LO_U64 as usize; + const HI_USIZE: usize = HI_U64 as usize; + + /// Return `true` if `x` contains any zero byte. + /// + /// From *Matters Computational*, J. Arndt + /// + /// "The idea is to subtract one from each of the bytes and then look for + /// bytes where the borrow propagated all the way to the most significant + /// bit." + #[inline] + fn contains_zero_byte(x: usize) -> bool { + x.wrapping_sub(LO_USIZE) & !x & HI_USIZE != 0 + } + + #[cfg(target_pointer_width = "32")] + #[inline] + fn repeat_byte(b: u8) -> usize { + let mut rep = (b as usize) << 8 | b as usize; + rep = rep << 16 | rep; + rep + } + + #[cfg(target_pointer_width = "64")] + #[inline] + fn repeat_byte(b: u8) -> usize { + let mut rep = (b as usize) << 8 | b as usize; + rep = rep << 16 | rep; + rep = rep << 32 | rep; + rep + } + + /// Return the first index matching the byte `a` in `text`. + pub fn memchr(x: u8, text: &[u8]) -> Option { + // Scan for a single byte value by reading two `usize` words at a time. + // + // Split `text` in three parts + // - unaligned inital part, before the first word aligned address in text + // - body, scan by 2 words at a time + // - the last remaining part, < 2 word size + let len = text.len(); + let ptr = text.as_ptr(); + + // search up to an aligned boundary + let align = (ptr as usize) & (usize::BYTES- 1); + let mut offset; + if align > 0 { + offset = cmp::min(usize::BYTES - align, len); + if let Some(index) = text[..offset].iter().position(|elt| *elt == x) { + return Some(index); + } + } else { + offset = 0; + } + + // search the body of the text + let repeated_x = repeat_byte(x); + + if len >= 2 * usize::BYTES { + while offset <= len - 2 * usize::BYTES { + unsafe { + let u = *(ptr.offset(offset as isize) as *const usize); + let v = *(ptr.offset((offset + usize::BYTES) as isize) as *const usize); + + // break if there is a matching byte + let zu = contains_zero_byte(u ^ repeated_x); + let zv = contains_zero_byte(v ^ repeated_x); + if zu || zv { + break; + } + } + offset += usize::BYTES * 2; + } + } + + // find the byte after the point the body loop stopped + text[offset..].iter().position(|elt| *elt == x).map(|i| offset + i) + } + + /// Return the last index matching the byte `a` in `text`. + pub fn memrchr(x: u8, text: &[u8]) -> Option { + // Scan for a single byte value by reading two `usize` words at a time. + // + // Split `text` in three parts + // - unaligned tail, after the last word aligned address in text + // - body, scan by 2 words at a time + // - the first remaining bytes, < 2 word size + let len = text.len(); + let ptr = text.as_ptr(); + + // search to an aligned boundary + let end_align = (ptr as usize + len) & (usize::BYTES - 1); + let mut offset; + if end_align > 0 { + offset = len - cmp::min(usize::BYTES - end_align, len); + if let Some(index) = text[offset..].iter().rposition(|elt| *elt == x) { + return Some(offset + index); + } + } else { + offset = len; + } + + // search the body of the text + let repeated_x = repeat_byte(x); + + while offset >= 2 * usize::BYTES { + unsafe { + let u = *(ptr.offset(offset as isize - 2 * usize::BYTES as isize) as *const usize); + let v = *(ptr.offset(offset as isize - usize::BYTES as isize) as *const usize); + + // break if there is a matching byte + let zu = contains_zero_byte(u ^ repeated_x); + let zv = contains_zero_byte(v ^ repeated_x); + if zu || zv { + break; + } + } + offset -= 2 * usize::BYTES; + } + + // find the byte before the point the body loop stopped + text[..offset].iter().rposition(|elt| *elt == x) + } + + // test fallback implementations on all plattforms + #[test] + fn matches_one() { + assert_eq!(Some(0), memchr(b'a', b"a")); + } + + #[test] + fn matches_begin() { + assert_eq!(Some(0), memchr(b'a', b"aaaa")); + } + + #[test] + fn matches_end() { + assert_eq!(Some(4), memchr(b'z', b"aaaaz")); + } + + #[test] + fn matches_nul() { + assert_eq!(Some(4), memchr(b'\x00', b"aaaa\x00")); + } + + #[test] + fn matches_past_nul() { + assert_eq!(Some(5), memchr(b'z', b"aaaa\x00z")); + } + + #[test] + fn no_match_empty() { + assert_eq!(None, memchr(b'a', b"")); + } + + #[test] + fn no_match() { + assert_eq!(None, memchr(b'a', b"xyz")); + } + + #[test] + fn matches_one_reversed() { + assert_eq!(Some(0), memrchr(b'a', b"a")); + } + + #[test] + fn matches_begin_reversed() { + assert_eq!(Some(3), memrchr(b'a', b"aaaa")); + } + + #[test] + fn matches_end_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"zaaaa")); + } + + #[test] + fn matches_nul_reversed() { + assert_eq!(Some(4), memrchr(b'\x00', b"aaaa\x00")); + } + + #[test] + fn matches_past_nul_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"z\x00aaaa")); + } + + #[test] + fn no_match_empty_reversed() { + assert_eq!(None, memrchr(b'a', b"")); + } + + #[test] + fn no_match_reversed() { + assert_eq!(None, memrchr(b'a', b"xyz")); + } +} + +#[cfg(test)] +mod tests { + // test the implementations for the current plattform + use super::{memchr, memrchr}; + + #[test] + fn matches_one() { + assert_eq!(Some(0), memchr(b'a', b"a")); + } + + #[test] + fn matches_begin() { + assert_eq!(Some(0), memchr(b'a', b"aaaa")); + } + + #[test] + fn matches_end() { + assert_eq!(Some(4), memchr(b'z', b"aaaaz")); + } + + #[test] + fn matches_nul() { + assert_eq!(Some(4), memchr(b'\x00', b"aaaa\x00")); + } + + #[test] + fn matches_past_nul() { + assert_eq!(Some(5), memchr(b'z', b"aaaa\x00z")); + } + + #[test] + fn no_match_empty() { + assert_eq!(None, memchr(b'a', b"")); + } + + #[test] + fn no_match() { + assert_eq!(None, memchr(b'a', b"xyz")); + } + + #[test] + fn matches_one_reversed() { + assert_eq!(Some(0), memrchr(b'a', b"a")); + } + + #[test] + fn matches_begin_reversed() { + assert_eq!(Some(3), memrchr(b'a', b"aaaa")); + } + + #[test] + fn matches_end_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"zaaaa")); + } + + #[test] + fn matches_nul_reversed() { + assert_eq!(Some(4), memrchr(b'\x00', b"aaaa\x00")); + } + + #[test] + fn matches_past_nul_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"z\x00aaaa")); + } + + #[test] + fn no_match_empty_reversed() { + assert_eq!(None, memrchr(b'a', b"")); + } + + #[test] + fn no_match_reversed() { + assert_eq!(None, memrchr(b'a', b"xyz")); + } +} From de3e843d2467dff3ccb83efbae9260dc1b2a40bf Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 15 Dec 2015 00:03:42 +0100 Subject: [PATCH 2/4] Use memchr in libstd where possible, closes #30076 --- src/libstd/ffi/c_str.rs | 3 ++- src/libstd/io/buffered.rs | 3 ++- src/libstd/io/mod.rs | 3 ++- src/libstd/sys/unix/os.rs | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libstd/ffi/c_str.rs b/src/libstd/ffi/c_str.rs index 318ff410cbad4..3f3913471b895 100644 --- a/src/libstd/ffi/c_str.rs +++ b/src/libstd/ffi/c_str.rs @@ -19,6 +19,7 @@ use io; use iter::Iterator; use libc; use mem; +use memchr; use ops::Deref; use option::Option::{self, Some, None}; use os::raw::c_char; @@ -188,7 +189,7 @@ impl CString { } fn _new(bytes: Vec) -> Result { - match bytes.iter().position(|x| *x == 0) { + match memchr::memchr(0, &bytes) { Some(i) => Err(NulError(i, bytes)), None => Ok(unsafe { CString::from_vec_unchecked(bytes) }), } diff --git a/src/libstd/io/buffered.rs b/src/libstd/io/buffered.rs index 90a79da348354..79eedbeda2c72 100644 --- a/src/libstd/io/buffered.rs +++ b/src/libstd/io/buffered.rs @@ -18,6 +18,7 @@ use cmp; use error; use fmt; use io::{self, DEFAULT_BUF_SIZE, Error, ErrorKind, SeekFrom}; +use memchr; /// The `BufReader` struct adds buffering to any reader. /// @@ -746,7 +747,7 @@ impl LineWriter { #[stable(feature = "rust1", since = "1.0.0")] impl Write for LineWriter { fn write(&mut self, buf: &[u8]) -> io::Result { - match buf.iter().rposition(|b| *b == b'\n') { + match memchr::memrchr(b'\n', buf) { Some(i) => { let n = try!(self.inner.write(&buf[..i + 1])); if n != i + 1 { return Ok(n) } diff --git a/src/libstd/io/mod.rs b/src/libstd/io/mod.rs index efe40cf07c130..cc3f8097a88e9 100644 --- a/src/libstd/io/mod.rs +++ b/src/libstd/io/mod.rs @@ -254,6 +254,7 @@ use result; use string::String; use str; use vec::Vec; +use memchr; #[stable(feature = "rust1", since = "1.0.0")] pub use self::buffered::{BufReader, BufWriter, LineWriter}; @@ -1194,7 +1195,7 @@ fn read_until(r: &mut R, delim: u8, buf: &mut Vec) Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, Err(e) => return Err(e) }; - match available.iter().position(|x| *x == delim) { + match memchr::memchr(delim, available) { Some(i) => { buf.extend_from_slice(&available[..i + 1]); (true, i + 1) diff --git a/src/libstd/sys/unix/os.rs b/src/libstd/sys/unix/os.rs index c2bf0651cfff4..12b9d6191a05f 100644 --- a/src/libstd/sys/unix/os.rs +++ b/src/libstd/sys/unix/os.rs @@ -22,6 +22,7 @@ use io; use iter; use libc::{self, c_int, c_char, c_void}; use mem; +use memchr; use path::{self, PathBuf}; use ptr; use slice; @@ -406,7 +407,7 @@ pub fn env() -> Env { if input.is_empty() { return None; } - let pos = input[1..].iter().position(|&b| b == b'=').map(|p| p + 1); + let pos = memchr::memchr(b'=', &input[1..]).map(|p| p + 1); pos.map(|p| ( OsStringExt::from_vec(input[..p].to_vec()), OsStringExt::from_vec(input[p+1..].to_vec()), From aa1f8fd3fb6b4e5334130eb5570cea18d2c13761 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 18 Dec 2015 00:16:25 +0100 Subject: [PATCH 3/4] Update liblibc to current master --- src/liblibc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liblibc b/src/liblibc index 867c6ff0b824d..9863d5645fc4d 160000 --- a/src/liblibc +++ b/src/liblibc @@ -1 +1 @@ -Subproject commit 867c6ff0b824d6d295951ed34bb252d5e0b2467a +Subproject commit 9863d5645fc4d1be789d03bcf58c1b3cfafbba39 From a206e556d07134661eba5cbb0315fbd59c3e6717 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 18 Dec 2015 00:36:38 +0100 Subject: [PATCH 4/4] Use memrchr bindings provided by libc --- src/libstd/memchr.rs | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/libstd/memchr.rs b/src/libstd/memchr.rs index 8a2fc57575970..c654efd499b64 100644 --- a/src/libstd/memchr.rs +++ b/src/libstd/memchr.rs @@ -11,7 +11,6 @@ // Original implementation taken from rust-memchr // Copyright 2015 Andrew Gallant, bluss and Nicolas Koch -use libc::{c_void, c_int, size_t}; /// A safe interface to `memchr`. @@ -27,7 +26,7 @@ use libc::{c_void, c_int, size_t}; /// /// This shows how to find the first position of a byte in a byte string. /// -/// ```rust +/// ```rust,ignore /// use memchr::memchr; /// /// let haystack = b"the quick brown fox"; @@ -35,17 +34,15 @@ use libc::{c_void, c_int, size_t}; /// ``` pub fn memchr(needle: u8, haystack: &[u8]) -> Option { // libc memchr - #[cfg(any(not(target_os = "windows"), - not(any(target_pointer_width = "32", - target_pointer_width = "64"))))] + #[cfg(not(target_os = "windows"))] fn memchr_specific(needle: u8, haystack: &[u8]) -> Option { - use libc::memchr as libc_memchr; + use libc; let p = unsafe { - libc_memchr( - haystack.as_ptr() as *const c_void, - needle as c_int, - haystack.len() as size_t) + libc::memchr( + haystack.as_ptr() as *const libc::c_void, + needle as libc::c_int, + haystack.len() as libc::size_t) }; if p.is_null() { None @@ -55,9 +52,7 @@ pub fn memchr(needle: u8, haystack: &[u8]) -> Option { } // use fallback on windows, since it's faster - #[cfg(all(target_os = "windows", - any(target_pointer_width = "32", - target_pointer_width = "64")))] + #[cfg(target_os = "windows")] fn memchr_specific(needle: u8, haystack: &[u8]) -> Option { fallback::memchr(needle, haystack) } @@ -74,7 +69,7 @@ pub fn memchr(needle: u8, haystack: &[u8]) -> Option { /// /// This shows how to find the last position of a byte in a byte string. /// -/// ```rust +/// ```rust,ignore /// use memchr::memrchr; /// /// let haystack = b"the quick brown fox"; @@ -84,15 +79,15 @@ pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { #[cfg(target_os = "linux")] fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { - use libc::memrchr as libc_memrchr; + use libc; // GNU's memrchr() will - unlike memchr() - error if haystack is empty. if haystack.is_empty() {return None} let p = unsafe { - libc_memrchr( - haystack.as_ptr() as *const c_void, - needle as c_int, - haystack.len() as size_t) + libc::memrchr( + haystack.as_ptr() as *const libc::c_void, + needle as libc::c_int, + haystack.len() as libc::size_t) }; if p.is_null() { None @@ -101,16 +96,7 @@ pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { } } - #[cfg(all(not(target_os = "linux"), - any(target_pointer_width = "32", target_pointer_width = "64")))] - fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { - fallback::memrchr(needle, haystack) - } - - // For the rare case of neither 32 bit nor 64-bit platform. - #[cfg(all(not(target_os = "linux"), - not(target_pointer_width = "32"), - not(target_pointer_width = "64")))] + #[cfg(not(target_os = "linux"))] fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { haystack.iter().rposition(|&b| b == needle) }