Repository navigation
a bit optimize four-digit chunks in integer formatting #159130
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
b165183
25435a0
6618eb6
20ec372
c10ff54
dedf120
917aa16
af98396
c257aef
b5e581f
ea0c39c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -690,15 +690,11 @@ impl u128 { | |
| unsafe { core::hint::assert_unchecked(offset <= buf.len()) } | ||
| offset -= 4; | ||
|
|
||
| // pull two pairs | ||
| let quad = remain % 1_00_00; | ||
| remain /= 1_00_00; | ||
| let pair1 = (quad / 100) as usize; | ||
| let pair2 = (quad % 100) as usize; | ||
| buf[offset + 0].write(DECIMAL_PAIRS[pair1 * 2 + 0]); | ||
| buf[offset + 1].write(DECIMAL_PAIRS[pair1 * 2 + 1]); | ||
| buf[offset + 2].write(DECIMAL_PAIRS[pair2 * 2 + 0]); | ||
| buf[offset + 3].write(DECIMAL_PAIRS[pair2 * 2 + 1]); | ||
| // SAFETY: quad is a remainder modulo 10_000. The offset checks | ||
| // above reserve exactly four bytes in buf. | ||
| unsafe { write_quad(buf, offset, quad) }; | ||
| } | ||
|
|
||
| // Format per two digits from the lookup table. | ||
|
|
@@ -814,32 +810,65 @@ impl i128 { | |
| } | ||
| } | ||
|
|
||
| /// Writes `quad` as exactly four digits (for example: `42` becomes `"0042"`). | ||
| /// | ||
| /// # Safety | ||
| /// | ||
| /// `quad` must be below 10_000 and `buf[offset..offset + 4]` must be in bounds. | ||
| #[inline(always)] | ||
| unsafe fn write_quad(buf: &mut [MaybeUninit<u8>], offset: usize, quad: u64) { | ||
| // SAFETY: These are this function's caller-provided invariants. | ||
| unsafe { | ||
| core::hint::assert_unchecked(quad < 10_000); | ||
| core::hint::assert_unchecked(offset <= buf.len() - 4); | ||
| } | ||
|
chirizxc marked this conversation as resolved.
Resolved
|
||
|
|
||
| // For the documented range, ceil(2^19 / 100) gives an exact quotiet. | ||
| const DIV100_SHIFT: u32 = 19; | ||
| const DIV100_RECIPROCAL: u32 = (1 << DIV100_SHIFT) / 100 + 1; | ||
|
|
||
| let quad = quad as u32; | ||
| let high = (quad * DIV100_RECIPROCAL) >> DIV100_SHIFT; | ||
|
chirizxc marked this conversation as resolved.
Resolved
|
||
| let low = quad - high * 100; | ||
| let high = high as usize; | ||
| let low = low as usize; | ||
|
|
||
| // SAFETY: `high` and `low` are below 100 because quad is below 10_000. The | ||
| // destination has four bytes by the precondition, and the two source pairs | ||
| // are disjoint from it because `DECIMAL_PAIRS` is static RO storage. | ||
| unsafe { | ||
| let pairs = DECIMAL_PAIRS.as_ptr(); | ||
| let dst = buf.as_mut_ptr().add(offset).cast::<u8>(); | ||
| core::ptr::copy_nonoverlapping(pairs.add(high * 2), dst, 2); | ||
| core::ptr::copy_nonoverlapping(pairs.add(low * 2), dst.add(2), 2); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Honestly was thinking maybe this would be worded better as
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. diff --git a/library/core/src/fmt/num.rs b/library/core/src/fmt/num.rs
index d703b817b5a..f94e5433a51 100644
--- a/library/core/src/fmt/num.rs
+++ b/library/core/src/fmt/num.rs
@@ -692,11 +692,12 @@ unsafe fn _fmt_inner(self, buf: &mut [MaybeUninit<u8>]) -> usize {
let quad = remain % 1_00_00;
remain /= 1_00_00;
- // SAFETY: quad is a remainder modulo 10_000. The offset checks
- // above reserve exactly four bytes in buf.
- unsafe {
- write_quad(buf.get_unchecked_mut(offset..offset + 4), quad);
- }
+
+ write_quad(
+ // SAFETY: `offset >= 4` was asserted above.
+ unsafe { buf.get_unchecked_mut(offset..offset + 4) },
+ quad,
+ );
}
// Format per two digits from the lookup table.
@@ -813,12 +814,8 @@ pub fn format_into(self, buf: &mut NumBuffer<Self>) -> &str {
}
/// Writes `quad` as exactly four digits (for example: `42` becomes `"0042"`).
-///
-/// # Safety
-///
-/// `quad` must be below 10_000 and `buf` must contain exactly four bytes.
#[inline(always)]
-unsafe fn write_quad(buf: &mut [MaybeUninit<u8>], quad: u64) {
+fn write_quad(buf: &mut [MaybeUninit<u8>], quad: u64) {
// SAFETY: These are this function's caller-provided invariants.
unsafe {
core::hint::assert_unchecked(quad < 10_000);
@@ -834,15 +831,10 @@ unsafe fn write_quad(buf: &mut [MaybeUninit<u8>], quad: u64) {
let low = low as usize;
// SAFETY: `high` and `low` are below 100 because `quad` is below 10_000.
- // The destination has four bytes by the precondition, and the two source
- // pairs are disjoint from it because `DECIMAL_PAIRS` is static RO storage.
- unsafe {
- let pairs = DECIMAL_PAIRS.as_ptr();
- let dst = buf.as_mut_ptr().cast_init();
+ unsafe { core::hint::assert_unchecked(high < 100 && low < 100) }
- core::ptr::copy_nonoverlapping(pairs.add(high * 2), dst, 2);
- core::ptr::copy_nonoverlapping(pairs.add(low * 2), dst.add(2), 2);
- }
+ buf[0..2].write_copy_of_slice(&DECIMAL_PAIRS[high * 2..high * 2 + 2]);
+ buf[2..4].write_copy_of_slice(&DECIMAL_PAIRS[low * 2..low * 2 + 2]);
}
/// Encodes the 16 least-significant decimals of n into `buf[OFFSET .. OFFSET +
@@ -863,20 +855,20 @@ fn enc_16lsd<const OFFSET: usize>(buf: &mut [MaybeUninit<u8>], n: u64) {
let quad = remain % 1_00_00;
remain /= 1_00_00;
- // SAFETY: `OFFSET + quad_index * 4` starts one of the four
- // non-overlapping four-byte regions proven in bounds above.
- unsafe {
- write_quad(
- buf.get_unchecked_mut(OFFSET + quad_index * 4..OFFSET + (quad_index + 1) * 4),
- quad,
- );
- }
+ write_quad(
+ // SAFETY: `OFFSET + 16 <= buf.len()` and `quad_index < 4`, so this range is within `buf`.
+ unsafe {
+ buf.get_unchecked_mut(OFFSET + quad_index * 4..OFFSET + (quad_index + 1) * 4)
+ },
+ quad,
+ );
}
- // SAFETY: OFFSET starts the first four-byte region proven in bounds above.
- unsafe {
- write_quad(buf.get_unchecked_mut(OFFSET..OFFSET + 4), remain);
- }
+ write_quad(
+ // SAFETY: `OFFSET + 16 <= buf.len()` was asserted above.
+ unsafe { buf.get_unchecked_mut(OFFSET..OFFSET + 4) },
+ remain,
+ );
}
/// Euclidean division plus remainder with constant 1E16 basically consumes 16I applied this patch; it seems that now we can make unsafe { core::hint::assert_unchecked(high < 100 && low < 100) }for buf[0..2].write_copy_of_slice(&DECIMAL_PAIRS[high * 2..high * 2 + 2]);
buf[2..4].write_copy_of_slice(&DECIMAL_PAIRS[low * 2..low * 2 + 2]);
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| } | ||
| } | ||
|
|
||
| /// Encodes the 16 least-significant decimals of n into `buf[OFFSET .. OFFSET + | ||
| /// 16 ]`. | ||
| fn enc_16lsd<const OFFSET: usize>(buf: &mut [MaybeUninit<u8>], n: u64) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't know if std has some special convention, but does this function have to become unsafe?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, this should be marked as unsafe. My bad; I thought both functions were remarked as unsafe, but I guess not.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Trying to understand why I missed this, and I guess it's because GitHub has been doing a horrible job trying to show partial diffs when I review things instead of full diffs, and I thought I had reviewed the full diff, but I guess not. .-. |
||
| // Consume the least-significant decimals from a working copy. | ||
| // SAFETY: Every caller passes a remainder produced by division by 10^16, | ||
| // and every used `OFFSET` specialization reserves sixteen bytes in `buf`. | ||
| unsafe { | ||
| core::hint::assert_unchecked(n < 10_000_000_000_000_000); | ||
| core::hint::assert_unchecked(OFFSET <= buf.len() - 16); | ||
|
chirizxc marked this conversation as resolved.
Resolved
|
||
| } | ||
|
|
||
| // Peel four digits at a time from right to left (12345678 -> 1234 | 5678). | ||
| // Since 10_000 is constant, LLVM replaces each division with multiply or shift. | ||
| let mut remain = n; | ||
|
|
||
| // Format per four digits from the lookup table. | ||
| for quad_index in (1..4).rev() { | ||
| // pull two pairs | ||
| let quad = remain % 1_00_00; | ||
| remain /= 1_00_00; | ||
| let pair1 = (quad / 100) as usize; | ||
| let pair2 = (quad % 100) as usize; | ||
| buf[quad_index * 4 + OFFSET + 0].write(DECIMAL_PAIRS[pair1 * 2 + 0]); | ||
| buf[quad_index * 4 + OFFSET + 1].write(DECIMAL_PAIRS[pair1 * 2 + 1]); | ||
| buf[quad_index * 4 + OFFSET + 2].write(DECIMAL_PAIRS[pair2 * 2 + 0]); | ||
| buf[quad_index * 4 + OFFSET + 3].write(DECIMAL_PAIRS[pair2 * 2 + 1]); | ||
| // SAFETY: modulo bounds quad; OFFSET and quad_index select one of the | ||
| // four non-overlapping four-byte regions proven in bounds above. | ||
| unsafe { write_quad(buf, quad_index * 4 + OFFSET, quad) }; | ||
| } | ||
|
|
||
| // final two pairs | ||
| let pair1 = (remain / 100) as usize; | ||
| let pair2 = (remain % 100) as usize; | ||
| buf[OFFSET + 0].write(DECIMAL_PAIRS[pair1 * 2 + 0]); | ||
| buf[OFFSET + 1].write(DECIMAL_PAIRS[pair1 * 2 + 1]); | ||
| buf[OFFSET + 2].write(DECIMAL_PAIRS[pair2 * 2 + 0]); | ||
| buf[OFFSET + 3].write(DECIMAL_PAIRS[pair2 * 2 + 1]); | ||
| // SAFETY: OFFSET starts the first four-byte region proven in bounds above. | ||
| unsafe { write_quad(buf, OFFSET, remain) }; | ||
| } | ||
|
|
||
| /// Euclidean division plus remainder with constant 1E16 basically consumes 16 | ||
|
|
||

Uh oh!
There was an error while loading. Please reload this page.