mirror of
https://github.com/golang/go.git
synced 2026-04-03 09:49:56 +09:00
internal/runtime/maps/: devirtualize hashing for specialized maps
This change improves performance of specialized maps and opens opportunity for further improvement by inlining hashing calls in the future (right now we can't inline functions from GOASM).
MapAccessBenchmarks
goos: linux
goarch: amd64
pkg: runtime
cpu: Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz
│ base-randlayout-meged.stat │ devirt-randlayout-merged.stat │
│ sec/op │ sec/op vs base │
MapAccessHit/Key=int32/Elem=int32/len=6-4 21.35n ± 1% 21.72n ± 0% +1.73% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=int32/len=64-4 25.74n ± 1% 25.37n ± 2% -1.44% (p=0.006 n=45)
MapAccessHit/Key=int32/Elem=int32/len=65536-4 41.56n ± 0% 41.07n ± 0% -1.18% (p=0.000 n=45)
MapAccessHit/Key=int64/Elem=int64/len=6-4 21.53n ± 0% 21.56n ± 0% +0.14% (p=0.000 n=45)
MapAccessHit/Key=int64/Elem=int64/len=64-4 25.56n ± 0% 25.32n ± 2% ~ (p=0.651 n=45)
MapAccessHit/Key=int64/Elem=int64/len=65536-4 44.47n ± 0% 44.42n ± 0% ~ (p=0.645 n=45)
MapAccessHit/Key=string/Elem=string/len=6-4 30.54n ± 0% 30.83n ± 0% +0.95% (p=0.000 n=45)
MapAccessHit/Key=string/Elem=string/len=64-4 33.94n ± 0% 32.58n ± 0% -4.01% (p=0.000 n=45)
MapAccessHit/Key=string/Elem=string/len=65536-4 60.75n ± 1% 59.22n ± 0% -2.52% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=bigType/len=6-4 104.7n ± 0% 104.2n ± 0% -0.48% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=bigType/len=64-4 176.9n ± 2% 169.8n ± 1% -4.01% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=bigType/len=65536-4 603.3n ± 0% 604.6n ± 0% +0.22% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=*int32/len=6-4 21.75n ± 1% 21.90n ± 0% +0.69% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=*int32/len=64-4 26.28n ± 0% 25.74n ± 0% -2.05% (p=0.000 n=45)
MapAccessHit/Key=int32/Elem=*int32/len=65536-4 45.42n ± 0% 45.10n ± 0% -0.70% (p=0.000 n=45)
MapAccessMiss/Key=int32/Elem=int32/len=6-4 23.49n ± 0% 23.56n ± 1% +0.30% (p=0.000 n=45)
MapAccessMiss/Key=int32/Elem=int32/len=64-4 24.93n ± 3% 24.85n ± 2% ~ (p=0.144 n=45)
MapAccessMiss/Key=int32/Elem=int32/len=65536-4 37.68n ± 0% 37.20n ± 0% -1.27% (p=0.000 n=45)
MapAccessMiss/Key=int64/Elem=int64/len=6-4 23.49n ± 0% 23.44n ± 1% ~ (p=0.587 n=45)
MapAccessMiss/Key=int64/Elem=int64/len=64-4 25.66n ± 2% 25.02n ± 2% -2.49% (p=0.032 n=45)
MapAccessMiss/Key=int64/Elem=int64/len=65536-4 38.19n ± 0% 37.83n ± 0% -0.94% (p=0.000 n=45)
MapAccessMiss/Key=string/Elem=string/len=6-4 31.29n ± 0% 31.55n ± 0% +0.83% (p=0.000 n=45)
MapAccessMiss/Key=string/Elem=string/len=64-4 31.28n ± 3% 30.81n ± 2% -1.50% (p=0.025 n=45)
MapAccessMiss/Key=string/Elem=string/len=65536-4 46.76n ± 0% 45.81n ± 0% -2.03% (p=0.000 n=45)
MapAccessMiss/Key=int32/Elem=bigType/len=6-4 128.0n ± 0% 128.0n ± 0% ~ (p=0.647 n=45)
MapAccessMiss/Key=int32/Elem=bigType/len=64-4 129.7n ± 0% 130.2n ± 0% ~ (p=0.069 n=45)
MapAccessMiss/Key=int32/Elem=bigType/len=65536-4 154.3n ± 0% 154.3n ± 0% ~ (p=0.058 n=45)
MapAccessMiss/Key=int32/Elem=*int32/len=6-4 23.38n ± 1% 23.87n ± 1% +2.10% (p=0.000 n=45)
MapAccessMiss/Key=int32/Elem=*int32/len=64-4 25.52n ± 2% 25.41n ± 2% ~ (p=0.321 n=45)
MapAccessMiss/Key=int32/Elem=*int32/len=65536-4 38.28n ± 0% 37.95n ± 0% -0.86% (p=0.000 n=45)
MapAccessZero/Key=int64-4 2.708n ± 0% 3.095n ± 0% +14.29% (p=0.000 n=45)*
MapAccessZero/Key=int32-4 2.708n ± 0% 3.095n ± 0% +14.29% (p=0.000 n=45)*
MapAccessZero/Key=string-4 3.481n ± 0% 3.095n ± 0% -11.09% (p=0.000 n=45)*
MapAccessEmpty/Key=int64-4 3.095n ± 0% 3.096n ± 0% ~ (p=0.087 n=45)*
MapAccessEmpty/Key=int32-4 3.095n ± 0% 3.482n ± 0% +12.50% (p=0.000 n=45)*
MapAccessEmpty/Key=string-4 3.869n ± 0% 3.483n ± 0% -9.98% (p=0.000 n=45)*
geomean 28.04n 28.00n -0.12%
* MapAccess(Empy|Zero) shows weird results due to basic block alignment changes
Fixes: #77892
Change-Id: I43913ae5dfa2d3e0cd173a766614ca4341774761
Reviewed-on: https://go-review.googlesource.com/c/go/+/750580
Auto-Submit: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
This commit is contained in:
ArsenySamoylov
committed by
Gopher Robot
Gopher Robot
parent
e7a09d1ffb
commit
252a8adbc0
@@ -14,6 +14,18 @@ import (
|
||||
)
|
||||
|
||||
// Functions below pushed from runtime.
|
||||
//
|
||||
//go:noescape
|
||||
//go:linkname memhash32 runtime.memhash32
|
||||
func memhash32(p unsafe.Pointer, h uintptr) uintptr
|
||||
|
||||
//go:noescape
|
||||
//go:linkname memhash64 runtime.memhash64
|
||||
func memhash64(p unsafe.Pointer, h uintptr) uintptr
|
||||
|
||||
//go:noescape
|
||||
//go:linkname strhash runtime.strhash
|
||||
func strhash(a unsafe.Pointer, h uintptr) uintptr
|
||||
|
||||
//go:linkname fatal
|
||||
func fatal(s string)
|
||||
|
||||
@@ -52,8 +52,19 @@ func runtime_mapaccess2_fast32(typ *abi.MapType, m *Map, key uint32) (unsafe.Poi
|
||||
return unsafe.Pointer(&zeroVal[0]), false
|
||||
}
|
||||
|
||||
// Don't pass address of the key directly to the hashing function.
|
||||
// Hashing functions are implemented in Go assembly and cannot be inlined,
|
||||
// so compiler doesn't optimize redundant address taking/dereference.
|
||||
//
|
||||
// Taking &key makes compiler treat key as address-taken, which forces it to spill on the stack
|
||||
// and reload it in the loop.
|
||||
// This is suboptimal for performance.
|
||||
//
|
||||
// Note: Even when we pass k (local copy of key), the compiler still spills the key to the stack.
|
||||
// However, from compiler's perspective, key is no longer address-taken and
|
||||
// filled back in register before the loop.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash32(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Select table.
|
||||
idx := m.directoryIndex(hash)
|
||||
@@ -141,8 +152,10 @@ func runtime_mapassign_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Point
|
||||
fatal("concurrent map writes")
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash32(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Set writing after calling Hasher, since Hasher may panic, in which
|
||||
// case we have not actually done a write.
|
||||
@@ -281,8 +294,10 @@ func runtime_mapassign_fast32ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u
|
||||
fatal("concurrent map writes")
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash32(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Set writing after calling Hasher, since Hasher may panic, in which
|
||||
// case we have not actually done a write.
|
||||
|
||||
@@ -52,8 +52,10 @@ func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Poi
|
||||
return unsafe.Pointer(&zeroVal[0]), false
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash64(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Select table.
|
||||
idx := m.directoryIndex(hash)
|
||||
@@ -142,8 +144,10 @@ func runtime_mapassign_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Point
|
||||
fatal("concurrent map writes")
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash64(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Set writing after calling Hasher, since Hasher may panic, in which
|
||||
// case we have not actually done a write.
|
||||
@@ -320,8 +324,10 @@ func runtime_mapassign_fast64ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u
|
||||
fatal("concurrent map writes")
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := memhash64(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Set writing after calling Hasher, since Hasher may panic, in which
|
||||
// case we have not actually done a write.
|
||||
|
||||
@@ -54,7 +54,11 @@ func (m *Map) getWithoutKeySmallFastStr(typ *abi.MapType, key string) unsafe.Poi
|
||||
|
||||
dohash:
|
||||
// This path will cost 1 hash and 1+ε comparisons.
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&key)), m.seed)
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := strhash(unsafe.Pointer(&k), m.seed)
|
||||
h2 := uint8(h2(hash))
|
||||
ctrls = *g.ctrls()
|
||||
slotKey = g.key(typ, 0)
|
||||
@@ -128,8 +132,10 @@ func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Po
|
||||
return elem, true
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := strhash(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Select table.
|
||||
idx := m.directoryIndex(hash)
|
||||
@@ -219,8 +225,10 @@ func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Poin
|
||||
fatal("concurrent map writes")
|
||||
}
|
||||
|
||||
// See the related comment in runtime_mapaccess2_fast32
|
||||
// for why we pass local copy of key.
|
||||
k := key
|
||||
hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed)
|
||||
hash := strhash(unsafe.Pointer(&k), m.seed)
|
||||
|
||||
// Set writing after calling Hasher, since Hasher may panic, in which
|
||||
// case we have not actually done a write.
|
||||
|
||||
Reference in New Issue
Block a user