Skip to content

Commit

Permalink
runtime: switch default order of hashing algorithms
Browse files Browse the repository at this point in the history 
Currently the standard hasher is memhash, which checks whether aes
instructions are available, and if so redirects to aeshash.

With this CL, we call aeshash directly, which then redirects to the
fallback hash if aes instructions are not available.

This reduces the overhead for the hash function in the common case,
as it requires just one call instead of two. On architectures which
have no assembly hasher, it's a single jump slower.

Thanks to Martin for this idea.

name         old time/op  new time/op  delta
BigKeyMap-4  22.6ns ± 1%  21.1ns ± 2%  -6.55%  (p=0.000 n=9+10)

Change-Id: Ib7ca77b63d28222eb0189bc3d7130531949d853c
Reviewed-on: https://go-review.googlesource.com/c/go/+/190998
Reviewed-by: Martin Möhrmann <[email protected]>
  • Loading branch information
@randall77
randall77 committed Aug 29, 2019
1 parent 9675f81 commit fbfb41e
Show file tree
Hide file tree
Showing 13 changed files with 146 additions and 110 deletions.
19 changes: 6 additions & 13 deletions src/runtime/alg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,14 @@ var algarray = [alg_max]typeAlg{
var useAeshash bool

// in asm_*.s
func aeshash(p unsafe.Pointer, h, s uintptr) uintptr
func aeshash32(p unsafe.Pointer, h uintptr) uintptr
func aeshash64(p unsafe.Pointer, h uintptr) uintptr
func aeshashstr(p unsafe.Pointer, h uintptr) uintptr
func memhash(p unsafe.Pointer, h, s uintptr) uintptr
func memhash32(p unsafe.Pointer, h uintptr) uintptr
func memhash64(p unsafe.Pointer, h uintptr) uintptr
func strhash(p unsafe.Pointer, h uintptr) uintptr

func strhash(a unsafe.Pointer, h uintptr) uintptr {
func strhashFallback(a unsafe.Pointer, h uintptr) uintptr {
x := (*stringStruct)(a)
return memhash(x.str, h, uintptr(x.len))
return memhashFallback(x.str, h, uintptr(x.len))
}

// NOTE: Because NaN != NaN, a map can contain any
Expand Down Expand Up @@ -305,14 +305,7 @@ func alginit() {
}

func initAlgAES() {
if GOOS == "aix" {
// runtime.algarray is immutable on AIX: see cmd/link/internal/ld/xcoff.go
return
}
useAeshash = true
algarray[alg_MEM32].hash = aeshash32
algarray[alg_MEM64].hash = aeshash64
algarray[alg_STRING].hash = aeshashstr
// Initialize with random data so hash collisions will be hard to engineer.
getRandomData(aeskeysched[:])
}
Expand Down
24 changes: 20 additions & 4 deletions src/runtime/asm_386.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -911,18 +911,26 @@ TEXT runtime·emptyfunc(SB),0,$0-0
RET

// hash function using AES hardware instructions
TEXT runtime·aeshash(SB),NOSPLIT,$0-16
TEXT runtime·memhash(SB),NOSPLIT,$0-16
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVL p+0(FP), AX // ptr to data
MOVL s+8(FP), BX // size
LEAL ret+12(FP), DX
JMP aeshashbody<>(SB)
noaes:
JMP runtime·memhashFallback(SB)

TEXT runtime·aeshashstr(SB),NOSPLIT,$0-12
TEXT runtime·strhash(SB),NOSPLIT,$0-12
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVL p+0(FP), AX // ptr to string object
MOVL 4(AX), BX // length of string
MOVL (AX), AX // string data
LEAL ret+8(FP), DX
JMP aeshashbody<>(SB)
noaes:
JMP runtime·strhashFallback(SB)

// AX: data
// BX: length
Expand Down Expand Up @@ -1108,7 +1116,9 @@ aesloop:
MOVL X4, (DX)
RET

TEXT runtime·aeshash32(SB),NOSPLIT,$0-12
TEXT runtime·memhash32(SB),NOSPLIT,$0-12
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVL p+0(FP), AX // ptr to data
MOVL h+4(FP), X0 // seed
PINSRD $1, (AX), X0 // data
Expand All @@ -1117,8 +1127,12 @@ TEXT runtime·aeshash32(SB),NOSPLIT,$0-12
AESENC runtime·aeskeysched+32(SB), X0
MOVL X0, ret+8(FP)
RET
noaes:
JMP runtime·memhash32Fallback(SB)

TEXT runtime·aeshash64(SB),NOSPLIT,$0-12
TEXT runtime·memhash64(SB),NOSPLIT,$0-12
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVL p+0(FP), AX // ptr to data
MOVQ (AX), X0 // data
PINSRD $2, h+4(FP), X0 // seed
Expand All @@ -1127,6 +1141,8 @@ TEXT runtime·aeshash64(SB),NOSPLIT,$0-12
AESENC runtime·aeskeysched+32(SB), X0
MOVL X0, ret+8(FP)
RET
noaes:
JMP runtime·memhash64Fallback(SB)

// simple mask to get rid of data in the high part of the register.
DATA masks<>+0x00(SB)/4, $0x00000000
Expand Down
36 changes: 27 additions & 9 deletions src/runtime/asm_amd64.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -885,21 +885,29 @@ done:
MOVQ AX, ret+0(FP)
RET

// func aeshash(p unsafe.Pointer, h, s uintptr) uintptr
// func memhash(p unsafe.Pointer, h, s uintptr) uintptr
// hash function using AES hardware instructions
TEXT runtime·aeshash(SB),NOSPLIT,$0-32
TEXT runtime·memhash(SB),NOSPLIT,$0-32
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVQ p+0(FP), AX // ptr to data
MOVQ s+16(FP), CX // size
LEAQ ret+24(FP), DX
JMP aeshashbody<>(SB)
noaes:
JMP runtime·memhashFallback(SB)

// func aeshashstr(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshashstr(SB),NOSPLIT,$0-24
// func strhash(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·strhash(SB),NOSPLIT,$0-24
CMPB runtime·useAeshash(SB), $0
JEQ noaes
MOVQ p+0(FP), AX // ptr to string struct
MOVQ 8(AX), CX // length of string
MOVQ (AX), AX // string data
LEAQ ret+16(FP), DX
JMP aeshashbody<>(SB)
noaes:
JMP runtime·strhashFallback(SB)

// AX: data
// CX: length
Expand Down Expand Up @@ -1232,8 +1240,11 @@ aesloop:
MOVQ X8, (DX)
RET

// func aeshash32(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshash32(SB),NOSPLIT,$0-24
// func memhash32(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·memhash32(SB),NOSPLIT,$0-24
CMPB runtime·useAeshash(SB), $0
JEQ noaes
JMP runtime·memhash32Fallback(SB)
MOVQ p+0(FP), AX // ptr to data
MOVQ h+8(FP), X0 // seed
PINSRD $2, (AX), X0 // data
Expand All @@ -1242,9 +1253,14 @@ TEXT runtime·aeshash32(SB),NOSPLIT,$0-24
AESENC runtime·aeskeysched+32(SB), X0
MOVQ X0, ret+16(FP)
RET

// func aeshash64(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshash64(SB),NOSPLIT,$0-24
noaes:
JMP runtime·memhash32Fallback(SB)

// func memhash64(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·memhash64(SB),NOSPLIT,$0-24
CMPB runtime·useAeshash(SB), $0
JEQ noaes
JMP runtime·memhash64Fallback(SB)
MOVQ p+0(FP), AX // ptr to data
MOVQ h+8(FP), X0 // seed
PINSRQ $1, (AX), X0 // data
Expand All @@ -1253,6 +1269,8 @@ TEXT runtime·aeshash64(SB),NOSPLIT,$0-24
AESENC runtime·aeskeysched+32(SB), X0
MOVQ X0, ret+16(FP)
RET
noaes:
JMP runtime·memhash64Fallback(SB)

// simple mask to get rid of data in the high part of the register.
DATA masks<>+0x00(SB)/8, $0x0000000000000000
Expand Down
20 changes: 8 additions & 12 deletions src/runtime/asm_amd64p32.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,21 +497,17 @@ TEXT runtime·cputicks(SB),NOSPLIT,$0-0
// write the implementations. Can copy and adjust the ones
// in asm_amd64.s when the time comes.

TEXT runtime·aeshash(SB),NOSPLIT,$0-20
MOVL AX, ret+16(FP)
RET
TEXT runtime·memhash(SB),NOSPLIT,$0-20
JMP runtime·memhashFallback(SB)

TEXT runtime·aeshashstr(SB),NOSPLIT,$0-12
MOVL AX, ret+8(FP)
RET
TEXT runtime·strhash(SB),NOSPLIT,$0-12
JMP runtime·strhashFallback(SB)

TEXT runtime·aeshash32(SB),NOSPLIT,$0-12
MOVL AX, ret+8(FP)
RET
TEXT runtime·memhash32(SB),NOSPLIT,$0-12
JMP runtime·memhash32Fallback(SB)

TEXT runtime·aeshash64(SB),NOSPLIT,$0-12
MOVL AX, ret+8(FP)
RET
TEXT runtime·memhash64(SB),NOSPLIT,$0-12
JMP runtime·memhash64Fallback(SB)

TEXT runtime·return0(SB), NOSPLIT, $0
MOVL $0, AX
Expand Down
20 changes: 8 additions & 12 deletions src/runtime/asm_arm.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -817,18 +817,14 @@ TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
RET

// AES hashing not implemented for ARM
TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
MOVW $0, R0
MOVW (R0), R1
TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
MOVW $0, R0
MOVW (R0), R1
TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
MOVW $0, R0
MOVW (R0), R1
TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
MOVW $0, R0
MOVW (R0), R1
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
JMP runtime·memhashFallback(SB)
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·strhashFallback(SB)
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·memhash32Fallback(SB)
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·memhash64Fallback(SB)

TEXT runtime·return0(SB),NOSPLIT,$0
MOVW $0, R0
Expand Down
36 changes: 28 additions & 8 deletions src/runtime/asm_arm64.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,8 +445,11 @@ CALLFN(·call268435456, 268435464 )
CALLFN(·call536870912, 536870920 )
CALLFN(·call1073741824, 1073741832 )

// func aeshash32(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-24
// func memhash32(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
MOVB runtime·useAeshash(SB), R0
CMP $0, R0
BEQ noaes
MOVD p+0(FP), R0
MOVD h+8(FP), R1
MOVD $ret+16(FP), R2
Expand All @@ -465,9 +468,14 @@ TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-24

VST1 [V0.D1], (R2)
RET
noaes:
B runtime·memhash32Fallback(SB)

// func aeshash64(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-24
// func memhash64(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
MOVB runtime·useAeshash(SB), R0
CMP $0, R0
BEQ noaes
MOVD p+0(FP), R0
MOVD h+8(FP), R1
MOVD $ret+16(FP), R2
Expand All @@ -486,22 +494,34 @@ TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-24

VST1 [V0.D1], (R2)
RET
noaes:
B runtime·memhash64Fallback(SB)

// func aeshash(p unsafe.Pointer, h, size uintptr) uintptr
TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-32
// func memhash(p unsafe.Pointer, h, size uintptr) uintptr
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
MOVB runtime·useAeshash(SB), R0
CMP $0, R0
BEQ noaes
MOVD p+0(FP), R0
MOVD s+16(FP), R1
MOVWU h+8(FP), R3
MOVD $ret+24(FP), R2
B aeshashbody<>(SB)
noaes:
B runtime·memhashFallback(SB)

// func aeshashstr(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-24
// func strhash(p unsafe.Pointer, h uintptr) uintptr
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
MOVB runtime·useAeshash(SB), R0
CMP $0, R0
BEQ noaes
MOVD p+0(FP), R10 // string pointer
LDP (R10), (R0, R1) //string data/ length
MOVWU h+8(FP), R3
MOVD $ret+16(FP), R2 // return adddress
B aeshashbody<>(SB)
noaes:
B runtime·strhashFallback(SB)

// R0: data
// R1: length (maximum 32 bits)
Expand Down
16 changes: 8 additions & 8 deletions src/runtime/asm_mips64x.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,14 +610,14 @@ TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
UNDEF

// AES hashing not implemented for mips64
TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
JMP runtime·memhashFallback(SB)
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·strhashFallback(SB)
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash32Fallback(SB)
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash64Fallback(SB)

TEXT runtime·return0(SB), NOSPLIT, $0
MOVW $0, R1
Expand Down
24 changes: 9 additions & 15 deletions src/runtime/asm_mipsx.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,21 +611,15 @@ TEXT setg_gcc<>(SB),NOSPLIT,$0
TEXT runtime·abort(SB),NOSPLIT,$0-0
UNDEF

// Not implemented.
TEXT runtime·aeshash(SB),NOSPLIT,$0
UNDEF

// Not implemented.
TEXT runtime·aeshash32(SB),NOSPLIT,$0
UNDEF

// Not implemented.
TEXT runtime·aeshash64(SB),NOSPLIT,$0
UNDEF

// Not implemented.
TEXT runtime·aeshashstr(SB),NOSPLIT,$0
UNDEF
// AES hashing not implemented for mips
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
JMP runtime·memhashFallback(SB)
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·strhashFallback(SB)
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·memhash32Fallback(SB)
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
JMP runtime·memhash64Fallback(SB)

TEXT runtime·return0(SB),NOSPLIT,$0
MOVW $0, R1
Expand Down
16 changes: 8 additions & 8 deletions src/runtime/asm_ppc64x.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -833,14 +833,14 @@ TEXT runtime·cputicks(SB),NOSPLIT,$0-8
RET

// AES hashing not implemented for ppc64
TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
MOVW (R0), R1
TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
JMP runtime·memhashFallback(SB)
TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·strhashFallback(SB)
TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash32Fallback(SB)
TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
JMP runtime·memhash64Fallback(SB)

TEXT runtime·return0(SB), NOSPLIT, $0
MOVW $0, R3
Expand Down
Loading

0 comments on commit fbfb41e

Please sign in to comment.