internal/runtime/syscall/linux.Syscall6
/usr/lib/go/src/internal/runtime/syscall/linux/asm_linux_arm64.s
Total: 7.22s 7.22s (flat, cum) 10.09% 5 . . #include "textflag.h" 6 . . 7 . . // func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) 8 . . TEXT ·Syscall6(SB),NOSPLIT,$0-80 9 . . MOVD num+0(FP), R8 // syscall entry 10 20ms 20ms MOVD a1+8(FP), R0 11 . . MOVD a2+16(FP), R1 12 . . MOVD a3+24(FP), R2 13 . . MOVD a4+32(FP), R3 14 . . MOVD a5+40(FP), R4 15 . . MOVD a6+48(FP), R5 16 . . SVC 17 7.15s 7.15s CMN $4095, R0 18 10ms 10ms BCC ok 19 . . MOVD $-1, R4 20 . . MOVD R4, r1+56(FP) 21 . . MOVD ZR, r2+64(FP) 22 . . NEG R0, R0 23 . . MOVD R0, errno+72(FP) 24 . . RET 25 . . ok: 26 40ms 40ms MOVD R0, r1+56(FP) 27 . . MOVD R1, r2+64(FP) 28 . . MOVD ZR, errno+72(FP) 29 . . RET
runtime.(*moduledata).textAddr
/usr/lib/go/src/runtime/symtab.go
Total: 400ms 400ms (flat, cum) 0.56% 691 . . // 692 . . // It is nosplit because it is part of the findfunc implementation. 693 . . // 694 . . //go:nosplit 695 . . func (md *moduledata) textAddr(off32 uint32) uintptr { 696 20ms 20ms off := uintptr(off32) 697 100ms 100ms res := md.text + off 698 100ms 100ms if len(md.textsectmap) > 1 { 699 . . for i, sect := range md.textsectmap { 700 . . // For the last section, include the end address (etext), as it is included in the functab. 701 . . if off >= sect.vaddr && off < sect.end || (i == len(md.textsectmap)-1 && off == sect.end) { 702 . . res = sect.baseaddr + off - sect.vaddr 703 . . break 704 . . } 705 . . } 706 . . if res > md.etext && GOARCH != "wasm" { // on wasm, functions do not live in the same address space as the linear memory 707 . . println("runtime: textAddr", hex(res), "out of range", hex(md.text), "-", hex(md.etext)) 708 . . throw("runtime: text offset out of range") 709 . . } 710 . . } 711 . . if GOARCH == "wasm" { 712 . . // On Wasm, a text offset (e.g. in the method table) is function index, whereas 713 . . // the "PC" is function index << 16 + block index. 714 . . res <<= 16 715 . . } 716 180ms 180ms return res 717 . . } 718 . . 719 . . // textOff is the opposite of textAddr. It converts a PC to a (virtual) offset 720 . . // to md.text, and returns if the PC is in any Go text section. 721 . . //
runtime.(*moduledata).funcName
/usr/lib/go/src/runtime/symtab.go
Total: 50ms 140ms (flat, cum) 0.2% 752 . . } 753 . . return res, true 754 . . } 755 . . 756 . . // funcName returns the string at nameOff in the function name table. 757 40ms 40ms func (md *moduledata) funcName(nameOff int32) string { 758 . . if nameOff == 0 { 759 . . return "" 760 . . } 761 10ms 100ms return gostringnocopy(&md.funcnametab[nameOff]) ss := stringStruct{str: unsafe.Pointer(str), len: findnull(str)} string.go:547 ⋮ 762 . . } 763 . . 764 . . // Despite being an exported symbol, 765 . . // FuncForPC is linknamed by widely used packages. 766 . . // Notable members of the hall of shame include:
runtime.findmoduledatap
/usr/lib/go/src/runtime/symtab.go
Total: 110ms 110ms (flat, cum) 0.15% 860 . . // It is nosplit because it's part of the isgoexception 861 . . // implementation. 862 . . // 863 . . //go:nosplit 864 . . func findmoduledatap(pc uintptr) *moduledata { 865 10ms 10ms for datap := &firstmoduledata; datap != nil; datap = datap.next { 866 100ms 100ms if datap.minpc <= pc && pc < datap.maxpc { 867 . . return datap 868 . . } 869 . . } 870 . . return nil 871 . . }
runtime.funcInfo.entry
/usr/lib/go/src/runtime/symtab.go
Total: 260ms 660ms (flat, cum) 0.92% 896 . . // - github.com/phuslu/log 897 . . // 898 . . // Do not remove or change the type signature. 899 . . // See go.dev/issue/67401. 900 . . func (f funcInfo) entry() uintptr { 901 260ms 660ms return f.datap.textAddr(f.entryOff) 902 . . } 903 . . 904 . . //go:linkname badFuncInfoEntry runtime.funcInfo.entry 905 . . func badFuncInfoEntry(funcInfo) uintptr 906 . .
runtime.findfunc
/usr/lib/go/src/runtime/symtab.go
Total: 1.23s 1.23s (flat, cum) 1.72% 917 . . // Do not remove or change the type signature. 918 . . // See go.dev/issue/67401. 919 . . // 920 . . //go:nosplit 921 . . //go:linkname findfunc 922 30ms 30ms func findfunc(pc uintptr) funcInfo { 923 110ms 110ms datap := findmoduledatap(pc) if datap.minpc <= pc && pc < datap.maxpc { symtab.go:866 ⋮ ⋮ for datap := &firstmoduledata; datap != nil; datap = datap.next { symtab.go:865 ⋮ if datap.minpc <= pc && pc < datap.maxpc { symtab.go:866 ⋮ 924 . . if datap == nil { 925 . . return funcInfo{} 926 . . } 927 . . const nsub = uintptr(len(findfuncbucket{}.subbuckets)) 928 . . 929 . . pcOff, ok := datap.textOff(pc) 930 . . if !ok { 931 . . return funcInfo{} 932 . . } 933 . . 934 30ms 30ms x := uintptr(pcOff) + datap.text - datap.minpc // TODO: are datap.text and datap.minpc always equal? 935 . . if GOARCH == "wasm" { 936 . . // On Wasm, pcOff is the function index, whereas 937 . . // the "PC" is function index << 16 + block index. 938 . . x = uintptr(pcOff)<<16 + datap.text - datap.minpc 939 . . } 940 . . b := x / abi.FuncTabBucketSize 941 . . i := x % abi.FuncTabBucketSize / (abi.FuncTabBucketSize / nsub) 942 . . 943 . . ffb := (*findfuncbucket)(add(unsafe.Pointer(datap.findfunctab), b*unsafe.Sizeof(findfuncbucket{}))) 944 400ms 400ms idx := ffb.idx + uint32(ffb.subbuckets[i]) 945 . . 946 . . // Find the ftab entry. 947 530ms 530ms for datap.ftab[idx+1].entryoff <= pcOff { 948 . . idx++ 949 . . } 950 . . 951 70ms 70ms funcoff := datap.ftab[idx].funcoff 952 60ms 60ms return funcInfo{(*_func)(unsafe.Pointer(&datap.pclntable[funcoff])), datap} 953 . . } 954 . . 955 . . // A srcFunc represents a logical function in the source code. This may 956 . . // correspond to an actual symbol in the binary text, or it may correspond to a 957 . . // source function that has been inlined.
runtime.pcvalueCacheKey
/usr/lib/go/src/runtime/symtab.go
Total: 10ms 10ms (flat, cum) 0.014% 1003 . . // pcvalueCacheKey returns the outermost index in a pcvalueCache to use for targetpc. 1004 . . // It must be very cheap to calculate. 1005 . . // For now, align to goarch.PtrSize and reduce mod the number of entries. 1006 . . // In practice, this appears to be fairly randomly and evenly distributed. 1007 . . func pcvalueCacheKey(targetpc uintptr) uintptr { 1008 10ms 10ms return (targetpc / goarch.PtrSize) % uintptr(len(pcvalueCache{}.entries)) 1009 . . }
runtime.pcvalue
/usr/lib/go/src/runtime/symtab.go
Total: 2.64s 4.60s (flat, cum) 6.43% 1011 . . // Returns the PCData value, and the PC where this value starts. 1012 30ms 30ms func pcvalue(f funcInfo, off uint32, targetpc uintptr, strict bool) (int32, uintptr) { 1013 . . // If true, when we get a cache hit, still look up the data and make sure it 1014 . . // matches the cached contents. 1015 . . const debugCheckCache = false 1016 . . 1017 . . // If true, skip checking the cache entirely. 1018 . . const skipCache = false 1019 . . 1020 30ms 30ms if off == 0 { 1021 . . return -1, 0 1022 . . } 1023 . . 1024 . . // Check the cache. This speeds up walks of deep stacks, which 1025 . . // tend to have the same recursive functions over and over, 1026 . . // or repetitive stacks between goroutines. 1027 . . var checkVal int32 1028 . . var checkPC uintptr 1029 20ms 20ms ck := pcvalueCacheKey(targetpc) ⋮ return (targetpc / goarch.PtrSize) % uintptr(len(pcvalueCache{}.entries)) symtab.go:1008 1030 . . if !skipCache { 1031 50ms 50ms mp := acquirem() gp.m.locks++ runtime1.go:614 ⋮ ⋮ 1032 . . cache := &mp.pcvalueCache 1033 . . // The cache can be used by the signal handler on this M. Avoid 1034 . . // re-entrant use of the cache. The signal handler can also write inUse, 1035 . . // but will always restore its value, so we can use a regular increment 1036 . . // even if we get signaled in the middle of it. 1037 30ms 30ms cache.inUse++ 1038 40ms 40ms if cache.inUse == 1 { 1039 320ms 320ms for i := range cache.entries[ck] { 1040 . . // We check off first because we're more 1041 . . // likely to have multiple entries with 1042 . . // different offsets for the same targetpc 1043 . . // than the other way around, so we'll usually 1044 . . // fail in the first clause. 1045 160ms 160ms ent := &cache.entries[ck][i] 1046 420ms 420ms if ent.off == off && ent.targetpc == targetpc { 1047 40ms 40ms val, pc := ent.val, ent.valPC 1048 . . if debugCheckCache { 1049 . . checkVal, checkPC = ent.val, ent.valPC 1050 . . break 1051 . . } else { 1052 . . cache.inUse-- 1053 . . releasem(mp) 1054 . . return val, pc 1055 . . } 1056 . . } 1057 . . } 1058 . . } else if debugCheckCache && (cache.inUse < 1 || cache.inUse > 2) { 1059 . . // Catch accounting errors or deeply reentrant use. In principle 1060 . . // "inUse" should never exceed 2. 1061 . . throw("cache.inUse out of range") 1062 . . } 1063 . . cache.inUse-- 1064 50ms 50ms releasem(mp) if mp.locks == 0 && gp.preempt { runtime1.go:622 ⋮ 1065 . . } 1066 . . 1067 . . if !f.valid() { 1068 . . if strict && panicking.Load() == 0 { 1069 . . println("runtime: no module data for", hex(f.entry())) 1070 240ms 240ms throw("no module data") 1071 . . } 1072 . . return -1, 0 1073 . . } 1074 . . datap := f.datap 1075 100ms 100ms p := datap.pctab[off:] 1076 20ms 80ms pc := f.entry() ⋮ return f.datap.textAddr(f.entryOff) symtab.go:901 1077 . . prevpc := pc 1078 10ms 10ms val := int32(-1) 1079 10ms 10ms for { 1080 . . var ok bool 1081 320ms 2.22s p, ok = step(p, &pc, &val, pc == f.entry()) return f.datap.textAddr(f.entryOff) symtab.go:901 ⋮ ⋮ ⋮ ⋮ ⋮ return f.datap.textAddr(f.entryOff) symtab.go:901 ⋮ ⋮ return f.datap.textAddr(f.entryOff) symtab.go:901 ⋮ ⋮ 1082 30ms 30ms if !ok { 1083 . . break 1084 . . } 1085 250ms 250ms if targetpc < pc { 1086 . . // Replace a random entry in the cache. Random 1087 . . // replacement prevents a performance cliff if 1088 . . // a recursive stack's cycle is slightly 1089 . . // larger than the cache. 1090 . . // Put the new element at the beginning, 1091 . . // since it is the most likely to be newly used. 1092 . . if debugCheckCache && checkPC != 0 { 1093 . . if checkVal != val || checkPC != prevpc { 1094 . . print("runtime: table value ", val, "@", prevpc, " != cache value ", checkVal, "@", checkPC, " at PC ", targetpc, " off ", off, "\n") 1095 . . throw("bad pcvalue cache") 1096 . . } 1097 . . } else { 1098 80ms 80ms mp := acquirem() gp.m.locks++ runtime1.go:614 1099 . . cache := &mp.pcvalueCache 1100 30ms 30ms cache.inUse++ 1101 . . if cache.inUse == 1 { 1102 60ms 60ms e := &cache.entries[ck] 1103 10ms 10ms ci := cheaprandn(uint32(len(cache.entries[ck]))) return uint32((uint64(cheaprand()) * uint64(n)) >> 32) rand.go:321 hi, lo := bits.Mul32(mp.cheaprand, mp.cheaprand^0x74743c1b) rand.go:238 1104 200ms 200ms e[ci] = e[0] 1105 20ms 20ms e[0] = pcvalueCacheEnt{ 1106 . . targetpc: targetpc, 1107 . . off: off, 1108 20ms 20ms val: val, 1109 . . valPC: prevpc, 1110 . . } 1111 . . } 1112 20ms 20ms cache.inUse-- 1113 10ms 10ms releasem(mp) if mp.locks == 0 && gp.preempt { runtime1.go:622 1114 . . } 1115 . . 1116 20ms 20ms return val, prevpc 1117 . . } 1118 . . prevpc = pc 1119 . . } 1120 . . 1121 . . // If there was a table, it should have covered all program counters.
runtime.funcname
/usr/lib/go/src/runtime/symtab.go
Total: 0 140ms (flat, cum) 0.2% 1144 . . 1145 . . func funcname(f funcInfo) string { 1146 . . if !f.valid() { 1147 . . return "" 1148 . . } 1149 . 140ms return f.datap.funcName(f.nameOff) 1150 . . } 1151 . . 1152 . . func funcpkgpath(f funcInfo) string { 1153 . . name := funcNameForPrint(funcname(f)) 1154 . . i := len(name) - 1
runtime.funcspdelta
/usr/lib/go/src/runtime/symtab.go
Total: 0 2.47s (flat, cum) 3.45% 1205 . . func funcline(f funcInfo, targetpc uintptr) (file string, line int32) { 1206 . . return funcline1(f, targetpc, true) 1207 . . } 1208 . . 1209 . . func funcspdelta(f funcInfo, targetpc uintptr) int32 { 1210 . 2.47s x, _ := pcvalue(f, f.pcsp, targetpc, true) 1211 . . if debugPcln && x&(goarch.PtrSize-1) != 0 { 1212 . . print("invalid spdelta ", funcname(f), " ", hex(f.entry()), " ", hex(targetpc), " ", hex(f.pcsp), " ", x, "\n") 1213 . . throw("bad spdelta") 1214 . . } 1215 . . return x
runtime.funcMaxSPDelta
/usr/lib/go/src/runtime/symtab.go
Total: 120ms 420ms (flat, cum) 0.59% 1216 . . } 1217 . . 1218 . . // funcMaxSPDelta returns the maximum spdelta at any point in f. 1219 . . func funcMaxSPDelta(f funcInfo) int32 { 1220 . . datap := f.datap 1221 60ms 60ms p := datap.pctab[f.pcsp:] 1222 . . pc := f.entry() 1223 . . val := int32(-1) 1224 . . most := int32(0) 1225 . . for { 1226 . . var ok bool 1227 40ms 340ms p, ok = step(p, &pc, &val, pc == f.entry()) ⋮ return f.datap.textAddr(f.entryOff) symtab.go:901 ⋮ ⋮ 1228 10ms 10ms if !ok { 1229 . . return most 1230 . . } 1231 10ms 10ms most = max(most, val) 1232 . . } 1233 . . }
runtime.pcdatastart
/usr/lib/go/src/runtime/symtab.go
Total: 10ms 10ms (flat, cum) 0.014% 1234 . . 1235 . . func pcdatastart(f funcInfo, table uint32) uint32 { 1236 10ms 10ms return *(*uint32)(add(unsafe.Pointer(&f.nfuncdata), unsafe.Sizeof(f.nfuncdata)+uintptr(table)*4)) 1237 . . }
runtime.pcdatavalue
/usr/lib/go/src/runtime/symtab.go
Total: 70ms 2.13s (flat, cum) 2.98% 1239 . . func pcdatavalue(f funcInfo, table uint32, targetpc uintptr) int32 { 1240 30ms 30ms if table >= f.npcdata { 1241 . . return -1 1242 . . } 1243 20ms 2.08s r, _ := pcvalue(f, pcdatastart(f, table), targetpc, true) ⋮ return *(*uint32)(add(unsafe.Pointer(&f.nfuncdata), unsafe.Sizeof(f.nfuncdata)+uintptr(table)*4)) symtab.go:1236 ⋮ 1244 20ms 20ms return r 1245 . . } 1246 . . 1247 . . func pcdatavalue1(f funcInfo, table uint32, targetpc uintptr, strict bool) int32 {
runtime.pcdatavalue1
/usr/lib/go/src/runtime/symtab.go
Total: 0 70ms (flat, cum) 0.098% 1248 . . if table >= f.npcdata { 1249 . . return -1 1250 . . } 1251 . 70ms r, _ := pcvalue(f, pcdatastart(f, table), targetpc, strict) 1252 . . return r 1253 . . } 1254 . . 1255 . . // Like pcdatavalue, but also return the start PC of this PCData value. 1256 . . func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) {
runtime.funcdata
/usr/lib/go/src/runtime/symtab.go
Total: 220ms 220ms (flat, cum) 0.31% 1261 . . } 1262 . . 1263 . . // funcdata returns a pointer to the ith funcdata for f. 1264 . . // funcdata should be kept in sync with cmd/link:writeFuncs. 1265 . . func funcdata(f funcInfo, i uint8) unsafe.Pointer { 1266 60ms 60ms if i < 0 || i >= f.nfuncdata { 1267 . . return nil 1268 . . } 1269 . . base := f.datap.gofunc // load gofunc address early so that we calculate during cache misses 1270 20ms 20ms p := uintptr(unsafe.Pointer(&f.nfuncdata)) + unsafe.Sizeof(f.nfuncdata) + uintptr(f.npcdata)*4 + uintptr(i)*4 1271 . . off := *(*uint32)(unsafe.Pointer(p)) 1272 . . // Return off == ^uint32(0) ? 0 : f.datap.gofunc + uintptr(off), but without branches. 1273 . . // The compiler calculates mask on most architectures using conditional assignment. 1274 . . var mask uintptr 1275 . . if off == ^uint32(0) { 1276 . . mask = 1 1277 . . } 1278 . . mask-- 1279 140ms 140ms raw := base + uintptr(off) 1280 . . return unsafe.Pointer(raw & mask) 1281 . . }
runtime.step
/usr/lib/go/src/runtime/symtab.go
Total: 1.86s 1.86s (flat, cum) 2.60% 1282 . . 1283 . . // step advances to the next pc, value pair in the encoded table. 1284 120ms 120ms func step(p []byte, pc *uintptr, val *int32, first bool) (newp []byte, ok bool) { 1285 . . // For both uvdelta and pcdelta, the common case (~70%) 1286 . . // is that they are a single byte. If so, avoid calling readvarint. 1287 . . uvdelta := uint32(p[0]) 1288 860ms 860ms if uvdelta == 0 && !first { 1289 10ms 10ms return nil, false 1290 . . } 1291 . . n := uint32(1) 1292 90ms 90ms if uvdelta&0x80 != 0 { 1293 200ms 200ms n, uvdelta = readvarint(p) v |= uint32(b&0x7F) << (shift & 31) symtab.go:1314 ⋮ ⋮ for { symtab.go:1311 ⋮ ⋮ b := p[n] symtab.go:1312 ⋮ ⋮ if b&0x80 == 0 { symtab.go:1315 ⋮ v |= uint32(b&0x7F) << (shift & 31) symtab.go:1314 ⋮ n++ symtab.go:1313 ⋮ v |= uint32(b&0x7F) << (shift & 31) symtab.go:1314 ⋮ shift += 7 symtab.go:1318 ⋮ b := p[n] symtab.go:1312 1294 . . } 1295 40ms 40ms *val += int32(-(uvdelta & 1) ^ (uvdelta >> 1)) 1296 100ms 100ms p = p[n:] 1297 . . 1298 90ms 90ms pcdelta := uint32(p[0]) 1299 . . n = 1 1300 100ms 100ms if pcdelta&0x80 != 0 { 1301 60ms 60ms n, pcdelta = readvarint(p) for { symtab.go:1311 ⋮ shift += 7 symtab.go:1318 ⋮ v |= uint32(b&0x7F) << (shift & 31) symtab.go:1314 1302 . . } 1303 80ms 80ms p = p[n:] 1304 50ms 50ms *pc += uintptr(pcdelta * sys.PCQuantum) 1305 60ms 60ms return p, true 1306 . . } 1307 . .
runtime.readvarint
/usr/lib/go/src/runtime/symtab.go
Total: 280ms 280ms (flat, cum) 0.39% 1309 . . func readvarint(p []byte) (read uint32, val uint32) { 1310 . . var v, shift, n uint32 1311 120ms 120ms for { 1312 60ms 60ms b := p[n] 1313 10ms 10ms n++ 1314 60ms 60ms v |= uint32(b&0x7F) << (shift & 31) 1315 10ms 10ms if b&0x80 == 0 { 1316 . . break 1317 . . } 1318 20ms 20ms shift += 7 1319 . . } 1320 . . return n, v 1321 . . } 1322 . . 1323 . . type stackmap struct {
runtime.stackmapdata
/usr/lib/go/src/runtime/symtab.go
Total: 10ms 10ms (flat, cum) 0.014% 1332 . . // The invariant is already checked by many of stackmapdata's callers, 1333 . . // and disabling it by default allows stackmapdata to be inlined. 1334 . . if stackDebug > 0 && (n < 0 || n >= stkmap.n) { 1335 . . throw("stackmapdata: index out of range") 1336 . . } 1337 10ms 10ms return bitvector{stkmap.nbit, addb(&stkmap.bytedata[0], uintptr(n*((stkmap.nbit+7)>>3)))} 1338 . . }
runtime.usleep
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 130ms 130ms (flat, cum) 0.18% 134 . . // nanosleep(&ts, 0) 135 . . ADD $8, RSP, R0 136 . . MOVD $0, R1 137 . . MOVD $SYS_nanosleep, R8 138 . . SVC 139 130ms 130ms RET 140 . . 141 . . TEXT runtime·gettid(SB),NOSPLIT,$0-4 142 . . MOVD $SYS_gettid, R8 143 . . SVC 144 . . MOVW R0, ret+0(FP)
runtime.nanotime1
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 10ms 10ms (flat, cum) 0.014% 313 . . 314 . . MOVD R3, sec+0(FP) 315 . . MOVW R5, nsec+8(FP) 316 . . RET 317 . . 318 10ms 10ms TEXT runtime·nanotime1(SB),NOSPLIT,$24-8 319 . . #ifdef GOEXPERIMENT_runtimesecret 320 . . MOVW g_secret(g), R20 321 . . CBZ R20, nosecret 322 . . BL ·secretEraseRegisters(SB) 323 . .
runtime.nanotime1
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 10ms 10ms (flat, cum) 0.014% 402 . . MOVD 8(RSP), R1 403 . . MOVD R1, m_vdsoPC(R21) 404 . . 405 . . // sec is in R3, nsec in R5 406 . . // return nsec in R3 407 10ms 10ms MOVD $1000000000, R4 408 . . MUL R4, R3 409 . . ADD R5, R3 410 . . MOVD R3, ret+0(FP) 411 . . RET 412 . .
runtime.sysMmap
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 20ms 20ms (flat, cum) 0.028% 592 . . MOVW fd+24(FP), R4 593 . . MOVW off+28(FP), R5 594 . . 595 . . MOVD $SYS_mmap, R8 596 . . SVC 597 20ms 20ms CMN $4095, R0 598 . . BCC ok 599 . . NEG R0,R0 600 . . MOVD $0, p+32(FP) 601 . . MOVD R0, err+40(FP) 602 . . RET
runtime.madvise
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 100ms 100ms (flat, cum) 0.14% 647 . . MOVD addr+0(FP), R0 648 . . MOVD n+8(FP), R1 649 . . MOVW flags+16(FP), R2 650 . . MOVD $SYS_madvise, R8 651 . . SVC 652 100ms 100ms MOVW R0, ret+24(FP) 653 . . RET 654 . . 655 . . // int64 futex(int32 *uaddr, int32 op, int32 val, 656 . . // struct timespec *timeout, int32 *uaddr2, int32 val2); 657 . . TEXT runtime·futex(SB),NOSPLIT|NOFRAME,$0
runtime.futex
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 3.78s 3.78s (flat, cum) 5.28% 660 . . MOVW val+12(FP), R2 661 . . MOVD ts+16(FP), R3 662 . . MOVD addr2+24(FP), R4 663 . . MOVW val3+32(FP), R5 664 . . MOVD $SYS_futex, R8 665 10ms 10ms SVC 666 3.76s 3.76s MOVW R0, ret+40(FP) 667 10ms 10ms RET 668 . . 669 . . // int64 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void)); 670 . . TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0 671 . . MOVW flags+0(FP), R0 672 . . MOVD stk+8(FP), R1
runtime.osyield
/usr/lib/go/src/runtime/sys_linux_arm64.s
Total: 200ms 200ms (flat, cum) 0.28% 746 . . MOVD R0, (R0) // crash 747 . . ok: 748 . . RET 749 . . 750 . . TEXT runtime·osyield(SB),NOSPLIT|NOFRAME,$0 751 10ms 10ms MOVD $SYS_sched_yield, R8 752 . . SVC 753 190ms 190ms RET 754 . . 755 . . TEXT runtime·sched_getaffinity(SB),NOSPLIT|NOFRAME,$0 756 . . MOVD pid+0(FP), R0 757 . . MOVD len+8(FP), R1 758 . . MOVD buf+16(FP), R2
runtime.mallocgcSmallScanNoHeaderSC1
/usr/lib/go/src/runtime/malloc_generated.go
Total: 80ms 80ms (flat, cum) 0.11% 8 . . "internal/goexperiment" 9 . . "internal/runtime/sys" 10 . . "unsafe" 11 . . ) 12 . . 13 20ms 20ms func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 14 . . if doubleCheckMalloc { 15 . . if gcphase == _GCmarktermination { 16 . . throw("mallocgc called with gcphase == _GCmarktermination") 17 . . } 18 . . } 19 . . 20 . . var mp *m 21 . . 22 30ms 30ms mp = acquirem() gp.m.locks++ runtime1.go:614 ⋮ return gp.m runtime1.go:615 23 . . 24 30ms 30ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 25 . . 26 . . if forceSlowPath { 27 . . releasem(mp) 28 . . 29 . . const spc = spanClass(1<<1) | spanClass(0)
runtime.mallocgcSmallScanNoHeaderSC1
/usr/lib/go/src/runtime/malloc_generated.go
Total: 220ms 320ms (flat, cum) 0.45% 40 . . if doubleCheckMalloc { 41 . . 42 . . doubleCheckSmallScanNoHeader(size, typ, mp) 43 . . 44 . . } 45 10ms 10ms mp.mallocing = 1 46 . . 47 . . checkGCTrigger := false 48 40ms 40ms c := getMCache(mp) if pp == nil { mcache.go:143 ⋮ pp := mp.p.ptr() mcache.go:141 ⋮ c = pp.mcache mcache.go:150 49 . . const spc = spanClass(sizeclass<<1) | spanClass(0) 50 . . span := c.alloc[spc] 51 . . 52 . . var v gclinkptr 53 . . var x unsafe.Pointer 54 . . 55 . . { 56 . . 57 . . var nextFreeFastResult gclinkptr 58 90ms 90ms if span.allocCache != 0 { 59 . . theBit := sys.TrailingZeros64(span.allocCache) 60 . . result := span.freeindex + uint16(theBit) 61 . . if result < span.nelems { 62 10ms 10ms freeidx := result + 1 63 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 64 . . span.allocCache >>= uint(theBit + 1) 65 . . span.freeindex = freeidx 66 20ms 20ms span.allocCount++ 67 10ms 10ms nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 68 . . } 69 . . } 70 . . } 71 . . v = nextFreeFastResult 72 . . if v == 0 { 73 . 100ms v, span, checkGCTrigger = c.nextFree(spc) 74 . . } 75 30ms 30ms x = unsafe.Pointer(v) 76 . . } 77 . . 78 . . if span.needzero != 0 { 79 . . memclrNoHeapPointers(x, elemsize) 80 . . } 81 . . if goarch.PtrSize == 8 && elemsize == 8 { 82 . . 83 10ms 10ms c.scanAlloc += 8 84 . . } else { 85 . . dataSize := size 86 . . x := uintptr(x) 87 . . 88 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) {
runtime.mallocgcSmallScanNoHeaderSC1
/usr/lib/go/src/runtime/malloc_generated.go
Total: 40ms 40ms (flat, cum) 0.056% 145 . . 146 . . publicationBarrier() 147 . . 148 . . span.freeIndexForScan = span.freeindex 149 . . 150 10ms 10ms c.nextSample -= int64(elemsize) 151 10ms 10ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 152 . . profilealloc(mp, x, elemsize) 153 . . } 154 . . mp.mallocing = 0 155 10ms 10ms releasem(mp) if mp.locks == 0 && gp.preempt { runtime1.go:622 156 . . 157 . . if checkGCTrigger { 158 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 159 . . gcStart(t) 160 . . } 161 . . } 162 . . 163 10ms 10ms return x 164 . . }
runtime.mallocgcSmallScanNoHeaderSC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 110ms 110ms (flat, cum) 0.15% 165 . . 166 50ms 50ms func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 167 . . if doubleCheckMalloc { 168 . . if gcphase == _GCmarktermination { 169 . . throw("mallocgc called with gcphase == _GCmarktermination") 170 . . } 171 . . } 172 . . 173 . . var mp *m 174 . . 175 60ms 60ms mp = acquirem() gp := getg() runtime1.go:613 ⋮ gp.m.locks++ runtime1.go:614 ⋮ return gp.m runtime1.go:615 ⋮ gp.m.locks++ runtime1.go:614 ⋮ 176 . . 177 . . forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 178 . . 179 . . if forceSlowPath { 180 . . releasem(mp)
runtime.mallocgcSmallScanNoHeaderSC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 430ms 580ms (flat, cum) 0.81% 196 . . 197 . . } 198 . . mp.mallocing = 1 199 . . 200 . . checkGCTrigger := false 201 100ms 100ms c := getMCache(mp) c = pp.mcache mcache.go:150 ⋮ ⋮ ⋮ pp := mp.p.ptr() mcache.go:141 202 . . const spc = spanClass(sizeclass<<1) | spanClass(0) 203 . . span := c.alloc[spc] 204 . . 205 . . var v gclinkptr 206 . . var x unsafe.Pointer 207 . . 208 . . { 209 . . 210 . . var nextFreeFastResult gclinkptr 211 80ms 80ms if span.allocCache != 0 { 212 . . theBit := sys.TrailingZeros64(span.allocCache) 213 . . result := span.freeindex + uint16(theBit) 214 . . if result < span.nelems { 215 30ms 30ms freeidx := result + 1 216 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 217 . . span.allocCache >>= uint(theBit + 1) 218 . . span.freeindex = freeidx 219 10ms 10ms span.allocCount++ 220 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 221 . . } 222 . . } 223 . . } 224 . . v = nextFreeFastResult 225 40ms 40ms if v == 0 { 226 . 140ms v, span, checkGCTrigger = c.nextFree(spc) 227 . . } 228 . . x = unsafe.Pointer(v) 229 . . } 230 . . 231 . . if span.needzero != 0 { 232 . . memclrNoHeapPointers(x, elemsize) 233 . . } 234 . . if goarch.PtrSize == 8 && elemsize == 8 { 235 . . 236 . . c.scanAlloc += 8 237 . . } else { 238 . . dataSize := size 239 . . x := uintptr(x) 240 . . 241 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) { 242 . . throw("tried to write heap bits, but no heap bits in span") 243 . . } 244 . . 245 10ms 10ms src0 := readUintptr(getGCMask(typ)) if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { type.go:88 246 . . 247 . . const elemsize = 16 248 . . 249 . . var scanSize uintptr 250 . . src := src0 251 20ms 20ms if typ.Size_ == goarch.PtrSize { 252 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 253 . . 254 . . scanSize = dataSize 255 . . } else { 256 . . 257 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 258 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 259 . . } 260 . . scanSize = typ.PtrBytes 261 . . for i := typ.Size_; i < dataSize; i += typ.Size_ { 262 . . src |= src0 << (i / goarch.PtrSize) 263 . . scanSize += typ.Size_ 264 . . } 265 . . } 266 . . 267 10ms 10ms dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) return s.startAddr mheap.go:519 268 . . dst := unsafe.Pointer(dstBase) 269 . . o := (x - span.base()) / goarch.PtrSize 270 . . i := o / ptrBits 271 . . j := o % ptrBits 272 . . var bits uintptr = elemsize / goarch.PtrSize 273 . . 274 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 275 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 276 . . 277 . . bits0 := ptrBits - j 278 . . bits1 := bits - bits0 279 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 280 . . dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) 281 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 282 . . *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 283 . . } else { 284 . . 285 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 286 100ms 100ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 287 . . } 288 . . 289 . . const doubleCheck = false 290 . . if doubleCheck { 291 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 292 . . } 293 . . if doubleCheckHeapSetType { 294 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 295 . . } 296 . . c.scanAlloc += scanSize 297 . . } 298 . . 299 10ms 10ms publicationBarrier() 300 . . 301 10ms 10ms span.freeIndexForScan = span.freeindex 302 . . 303 . . c.nextSample -= int64(elemsize) 304 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 305 10ms 20ms profilealloc(mp, x, elemsize) 306 . . } 307 . . mp.mallocing = 0 308 . . releasem(mp) 309 . . 310 . . if checkGCTrigger {
runtime.mallocgcSmallScanNoHeaderSC3
/usr/lib/go/src/runtime/malloc_generated.go
Total: 40ms 40ms (flat, cum) 0.056% 314 . . } 315 . . 316 . . return x 317 . . } 318 . . 319 30ms 30ms func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 320 . . if doubleCheckMalloc { 321 . . if gcphase == _GCmarktermination { 322 . . throw("mallocgc called with gcphase == _GCmarktermination") 323 . . } 324 . . } 325 . . 326 . . var mp *m 327 . . 328 10ms 10ms mp = acquirem() return gp.m runtime1.go:615 329 . . 330 . . forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 331 . . 332 . . if forceSlowPath { 333 . . releasem(mp)
runtime.mallocgcSmallScanNoHeaderSC3
/usr/lib/go/src/runtime/malloc_generated.go
Total: 240ms 350ms (flat, cum) 0.49% 359 . . var x unsafe.Pointer 360 . . 361 . . { 362 . . 363 . . var nextFreeFastResult gclinkptr 364 60ms 60ms if span.allocCache != 0 { 365 . . theBit := sys.TrailingZeros64(span.allocCache) 366 . . result := span.freeindex + uint16(theBit) 367 . . if result < span.nelems { 368 . . freeidx := result + 1 369 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 370 . . span.allocCache >>= uint(theBit + 1) 371 . . span.freeindex = freeidx 372 20ms 20ms span.allocCount++ 373 10ms 10ms nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 374 . . } 375 . . } 376 . . } 377 . . v = nextFreeFastResult 378 . . if v == 0 { 379 . 100ms v, span, checkGCTrigger = c.nextFree(spc) 380 . . } 381 . . x = unsafe.Pointer(v) 382 . . } 383 . . 384 . . if span.needzero != 0 { 385 30ms 30ms memclrNoHeapPointers(x, elemsize) 386 . . } 387 . . if goarch.PtrSize == 8 && elemsize == 8 { 388 . . 389 . . c.scanAlloc += 8 390 . . } else { 391 . . dataSize := size 392 10ms 10ms x := uintptr(x) 393 . . 394 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) { 395 . . throw("tried to write heap bits, but no heap bits in span") 396 . . } 397 . . 398 50ms 50ms src0 := readUintptr(getGCMask(typ)) if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { type.go:88 ⋮ 399 . . 400 . . const elemsize = 24 401 . . 402 . . var scanSize uintptr 403 . . src := src0 404 10ms 10ms if typ.Size_ == goarch.PtrSize { 405 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 406 . . 407 . . scanSize = dataSize 408 . . } else { 409 . . 410 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 411 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 412 . . } 413 . . scanSize = typ.PtrBytes 414 . . for i := typ.Size_; i < dataSize; i += typ.Size_ { 415 . . src |= src0 << (i / goarch.PtrSize) 416 . . scanSize += typ.Size_ 417 . . } 418 . . } 419 . . 420 10ms 10ms dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) return s.startAddr mheap.go:519 421 . . dst := unsafe.Pointer(dstBase) 422 10ms 10ms o := (x - span.base()) / goarch.PtrSize 423 . . i := o / ptrBits 424 . . j := o % ptrBits 425 . . var bits uintptr = elemsize / goarch.PtrSize 426 . . 427 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 428 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 429 . . 430 . . bits0 := ptrBits - j 431 . . bits1 := bits - bits0 432 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 433 . . dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) 434 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 435 . . *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 436 . . } else { 437 . . 438 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 439 10ms 10ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 440 . . } 441 . . 442 . . const doubleCheck = false 443 . . if doubleCheck { 444 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 445 . . } 446 . . if doubleCheckHeapSetType { 447 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 448 . . } 449 . . c.scanAlloc += scanSize 450 . . } 451 . . 452 . . publicationBarrier() 453 . . 454 10ms 10ms span.freeIndexForScan = span.freeindex 455 . . 456 . . c.nextSample -= int64(elemsize) 457 10ms 10ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 458 . . profilealloc(mp, x, elemsize) 459 . . } 460 . . mp.mallocing = 0 461 . . releasem(mp) 462 . . 463 . . if checkGCTrigger { 464 . 10ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 465 . . gcStart(t) 466 . . } 467 . . }
runtime.mallocgcSmallScanNoHeaderSC4
/usr/lib/go/src/runtime/malloc_generated.go
Total: 90ms 90ms (flat, cum) 0.13% 469 . . return x 470 . . } 471 . . 472 30ms 30ms func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 473 . . if doubleCheckMalloc { 474 . . if gcphase == _GCmarktermination { 475 . . throw("mallocgc called with gcphase == _GCmarktermination") 476 . . } 477 . . } 478 . . 479 . . var mp *m 480 . . 481 30ms 30ms mp = acquirem() gp.m.locks++ runtime1.go:614 ⋮ 482 . . 483 30ms 30ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 484 . . 485 . . if forceSlowPath { 486 . . releasem(mp) 487 . . 488 . . const spc = spanClass(4<<1) | spanClass(0)
runtime.mallocgcSmallScanNoHeaderSC4
/usr/lib/go/src/runtime/malloc_generated.go
Total: 600ms 870ms (flat, cum) 1.22% 502 . . 503 . . } 504 . . mp.mallocing = 1 505 . . 506 . . checkGCTrigger := false 507 60ms 60ms c := getMCache(mp) c = pp.mcache mcache.go:150 ⋮ pp := mp.p.ptr() mcache.go:141 ⋮ ⋮ c = pp.mcache mcache.go:150 508 . . const spc = spanClass(sizeclass<<1) | spanClass(0) 509 . . span := c.alloc[spc] 510 . . 511 . . var v gclinkptr 512 . . var x unsafe.Pointer 513 . . 514 . . { 515 . . 516 . . var nextFreeFastResult gclinkptr 517 130ms 130ms if span.allocCache != 0 { 518 10ms 10ms theBit := sys.TrailingZeros64(span.allocCache) 519 . . result := span.freeindex + uint16(theBit) 520 . . if result < span.nelems { 521 60ms 60ms freeidx := result + 1 522 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 523 30ms 30ms span.allocCache >>= uint(theBit + 1) 524 10ms 10ms span.freeindex = freeidx 525 60ms 60ms span.allocCount++ 526 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 527 . . } 528 . . } 529 . . } 530 . . v = nextFreeFastResult 531 . . if v == 0 { 532 . 250ms v, span, checkGCTrigger = c.nextFree(spc) 533 . . } 534 10ms 10ms x = unsafe.Pointer(v) 535 . . } 536 . . 537 . . if span.needzero != 0 { 538 20ms 20ms memclrNoHeapPointers(x, elemsize) 539 . . } 540 . . if goarch.PtrSize == 8 && elemsize == 8 { 541 . . 542 . . c.scanAlloc += 8 543 . . } else { 544 . . dataSize := size 545 10ms 10ms x := uintptr(x) 546 . . 547 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) { 548 . . throw("tried to write heap bits, but no heap bits in span") 549 . . } 550 . . 551 20ms 20ms src0 := readUintptr(getGCMask(typ)) if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { type.go:88 ⋮ 552 . . 553 . . const elemsize = 32 554 . . 555 . . var scanSize uintptr 556 . . src := src0 557 10ms 10ms if typ.Size_ == goarch.PtrSize { 558 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 559 . . 560 . . scanSize = dataSize 561 . . } else { 562 . . 563 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 564 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 565 . . } 566 . . scanSize = typ.PtrBytes 567 . . for i := typ.Size_; i < dataSize; i += typ.Size_ { 568 10ms 10ms src |= src0 << (i / goarch.PtrSize) 569 . . scanSize += typ.Size_ 570 . . } 571 . . } 572 . . 573 . . dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) 574 . . dst := unsafe.Pointer(dstBase) 575 10ms 10ms o := (x - span.base()) / goarch.PtrSize 576 . . i := o / ptrBits 577 . . j := o % ptrBits 578 . . var bits uintptr = elemsize / goarch.PtrSize 579 . . 580 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 581 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 582 . . 583 . . bits0 := ptrBits - j 584 . . bits1 := bits - bits0 585 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 586 . . dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) 587 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 588 . . *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 589 . . } else { 590 . . 591 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 592 80ms 80ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 593 . . } 594 . . 595 . . const doubleCheck = false 596 . . if doubleCheck { 597 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 598 . . } 599 . . if doubleCheckHeapSetType { 600 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 601 . . } 602 . . c.scanAlloc += scanSize 603 . . } 604 . . 605 . . publicationBarrier() 606 . . 607 20ms 20ms span.freeIndexForScan = span.freeindex 608 . . 609 . . c.nextSample -= int64(elemsize) 610 10ms 10ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 611 . 10ms profilealloc(mp, x, elemsize) 612 . . } 613 . . mp.mallocing = 0 614 10ms 10ms releasem(mp) mp.locks-- runtime1.go:621 615 . . 616 . . if checkGCTrigger { 617 . 10ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 618 . . gcStart(t) 619 . . } 620 . . } 621 . . 622 30ms 30ms return x 623 . . }
runtime.mallocgcSmallScanNoHeaderSC5
/usr/lib/go/src/runtime/malloc_generated.go
Total: 250ms 820ms (flat, cum) 1.15% 624 . . 625 100ms 670ms func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 626 . . if doubleCheckMalloc { 627 . . if gcphase == _GCmarktermination { 628 . . throw("mallocgc called with gcphase == _GCmarktermination") 629 . . } 630 . . } 631 . . 632 . . var mp *m 633 . . 634 80ms 80ms mp = acquirem() gp.m.locks++ runtime1.go:614 ⋮ return gp.m runtime1.go:615 635 . . 636 70ms 70ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 637 . . 638 . . if forceSlowPath { 639 . . releasem(mp) 640 . . 641 . . const spc = spanClass(5<<1) | spanClass(0)
runtime.mallocgcSmallScanNoHeaderSC5
/usr/lib/go/src/runtime/malloc_generated.go
Total: 730ms 1.57s (flat, cum) 2.19% 655 . . 656 . . } 657 . . mp.mallocing = 1 658 . . 659 . . checkGCTrigger := false 660 100ms 100ms c := getMCache(mp) pp := mp.p.ptr() mcache.go:141 ⋮ c = pp.mcache mcache.go:150 ⋮ 661 . . const spc = spanClass(sizeclass<<1) | spanClass(0) 662 . . span := c.alloc[spc] 663 . . 664 . . var v gclinkptr 665 . . var x unsafe.Pointer 666 . . 667 . . { 668 . . 669 . . var nextFreeFastResult gclinkptr 670 100ms 100ms if span.allocCache != 0 { 671 20ms 20ms theBit := sys.TrailingZeros64(span.allocCache) 672 . . result := span.freeindex + uint16(theBit) 673 . . if result < span.nelems { 674 40ms 40ms freeidx := result + 1 675 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 676 . . span.allocCache >>= uint(theBit + 1) 677 30ms 30ms span.freeindex = freeidx 678 130ms 130ms span.allocCount++ 679 40ms 40ms nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) return s.startAddr mheap.go:519 680 . . } 681 . . } 682 . . } 683 . . v = nextFreeFastResult 684 . . if v == 0 { 685 10ms 800ms v, span, checkGCTrigger = c.nextFree(spc) 686 . . } 687 10ms 10ms x = unsafe.Pointer(v) 688 . . } 689 . . 690 . . if span.needzero != 0 { 691 20ms 20ms memclrNoHeapPointers(x, elemsize) 692 . . } 693 . . if goarch.PtrSize == 8 && elemsize == 8 { 694 . . 695 . . c.scanAlloc += 8 696 . . } else { 697 . . dataSize := size 698 10ms 10ms x := uintptr(x) 699 . . 700 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) { 701 . . throw("tried to write heap bits, but no heap bits in span") 702 . . } 703 . . 704 50ms 50ms src0 := readUintptr(getGCMask(typ)) if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { type.go:88 ⋮ 705 . . 706 . . const elemsize = 48 707 . . 708 . . var scanSize uintptr 709 . . src := src0 710 . . if typ.Size_ == goarch.PtrSize { 711 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 712 . . 713 . . scanSize = dataSize 714 . . } else { 715 . . 716 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 717 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 718 . . } 719 . . scanSize = typ.PtrBytes 720 10ms 10ms for i := typ.Size_; i < dataSize; i += typ.Size_ { 721 . . src |= src0 << (i / goarch.PtrSize) 722 . . scanSize += typ.Size_ 723 . . } 724 . . } 725 . . 726 . . dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) 727 . . dst := unsafe.Pointer(dstBase) 728 . . o := (x - span.base()) / goarch.PtrSize 729 . . i := o / ptrBits 730 20ms 20ms j := o % ptrBits 731 . . var bits uintptr = elemsize / goarch.PtrSize 732 . . 733 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 734 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 735 . . 736 . . bits0 := ptrBits - j 737 . . bits1 := bits - bits0 738 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 739 20ms 20ms dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) return unsafe.Pointer(uintptr(p) + x) stubs.go:25 740 10ms 10ms *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 741 10ms 10ms *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 742 . . } else { 743 . . 744 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 745 20ms 20ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 746 . . } 747 . . 748 . . const doubleCheck = false 749 . . if doubleCheck { 750 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 751 . . } 752 . . if doubleCheckHeapSetType { 753 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 754 . . } 755 . . c.scanAlloc += scanSize 756 . . } 757 . . 758 10ms 10ms publicationBarrier() 759 . . 760 40ms 40ms span.freeIndexForScan = span.freeindex 761 . . 762 . . c.nextSample -= int64(elemsize) 763 30ms 30ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 764 . 40ms profilealloc(mp, x, elemsize) 765 . . } 766 . . mp.mallocing = 0 767 . . releasem(mp) 768 . . 769 . . if checkGCTrigger { 770 . 10ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 771 . . gcStart(t) 772 . . } 773 . . }
runtime.mallocgcSmallScanNoHeaderSC6
/usr/lib/go/src/runtime/malloc_generated.go
Total: 30ms 30ms (flat, cum) 0.042% 775 . . return x 776 . . } 777 . . 778 20ms 20ms func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 779 . . if doubleCheckMalloc { 780 . . if gcphase == _GCmarktermination { 781 . . throw("mallocgc called with gcphase == _GCmarktermination") 782 . . } 783 . . } 784 . . 785 . . var mp *m 786 . . 787 10ms 10ms mp = acquirem() gp.m.locks++ runtime1.go:614 788 . . 789 . . forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 790 . . 791 . . if forceSlowPath { 792 . . releasem(mp)
runtime.mallocgcSmallScanNoHeaderSC6
/usr/lib/go/src/runtime/malloc_generated.go
Total: 250ms 400ms (flat, cum) 0.56% 818 . . var x unsafe.Pointer 819 . . 820 . . { 821 . . 822 . . var nextFreeFastResult gclinkptr 823 80ms 80ms if span.allocCache != 0 { 824 . . theBit := sys.TrailingZeros64(span.allocCache) 825 . . result := span.freeindex + uint16(theBit) 826 . . if result < span.nelems { 827 10ms 10ms freeidx := result + 1 828 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 829 . . span.allocCache >>= uint(theBit + 1) 830 . . span.freeindex = freeidx 831 . . span.allocCount++ 832 10ms 10ms nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 833 . . } 834 . . } 835 . . } 836 . . v = nextFreeFastResult 837 . . if v == 0 { 838 . 140ms v, span, checkGCTrigger = c.nextFree(spc) 839 . . } 840 . . x = unsafe.Pointer(v) 841 . . } 842 . . 843 10ms 10ms if span.needzero != 0 { 844 30ms 30ms memclrNoHeapPointers(x, elemsize) 845 . . } 846 . . if goarch.PtrSize == 8 && elemsize == 8 { 847 . . 848 . . c.scanAlloc += 8 849 . . } else { 850 . . dataSize := size 851 . . x := uintptr(x) 852 . . 853 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) { 854 . . throw("tried to write heap bits, but no heap bits in span") 855 . . } 856 . . 857 10ms 10ms src0 := readUintptr(getGCMask(typ)) x := *(*uintptr)(unsafe.Pointer(p)) mbitmap.go:1518 858 . . 859 . . const elemsize = 64 860 . . 861 . . var scanSize uintptr 862 . . src := src0 863 20ms 20ms if typ.Size_ == goarch.PtrSize { 864 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 865 . . 866 . . scanSize = dataSize 867 . . } else { 868 . . 869 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 870 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 871 . . } 872 . . scanSize = typ.PtrBytes 873 . . for i := typ.Size_; i < dataSize; i += typ.Size_ { 874 . . src |= src0 << (i / goarch.PtrSize) 875 . . scanSize += typ.Size_ 876 . . } 877 . . } 878 . . 879 . . dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) 880 . . dst := unsafe.Pointer(dstBase) 881 10ms 10ms o := (x - span.base()) / goarch.PtrSize 882 . . i := o / ptrBits 883 . . j := o % ptrBits 884 . . var bits uintptr = elemsize / goarch.PtrSize 885 . . 886 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 887 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 888 . . 889 . . bits0 := ptrBits - j 890 . . bits1 := bits - bits0 891 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 892 . . dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) 893 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 894 . . *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 895 . . } else { 896 . . 897 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 898 10ms 10ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 899 . . } 900 . . 901 . . const doubleCheck = false 902 . . if doubleCheck { 903 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 904 . . } 905 . . if doubleCheckHeapSetType { 906 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 907 . . } 908 . . c.scanAlloc += scanSize 909 . . } 910 . . 911 10ms 10ms publicationBarrier() 912 . . 913 10ms 10ms span.freeIndexForScan = span.freeindex 914 . . 915 . . c.nextSample -= int64(elemsize) 916 20ms 20ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 917 . 10ms profilealloc(mp, x, elemsize) 918 . . } 919 10ms 10ms mp.mallocing = 0 920 . . releasem(mp) 921 . . 922 . . if checkGCTrigger { 923 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 924 . . gcStart(t) 925 . . } 926 . . } 927 . . 928 10ms 10ms return x 929 . . }
runtime.mallocgcSmallScanNoHeaderSC7
/usr/lib/go/src/runtime/malloc_generated.go
Total: 20ms 20ms (flat, cum) 0.028% 930 . . 931 10ms 10ms func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 932 . . if doubleCheckMalloc { 933 . . if gcphase == _GCmarktermination { 934 . . throw("mallocgc called with gcphase == _GCmarktermination") 935 . . } 936 . . } 937 . . 938 . . var mp *m 939 . . 940 10ms 10ms mp = acquirem() gp.m.locks++ runtime1.go:614 941 . . 942 . . forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 943 . . 944 . . if forceSlowPath { 945 . . releasem(mp)
runtime.mallocgcSmallScanNoHeaderSC7
/usr/lib/go/src/runtime/malloc_generated.go
Total: 250ms 350ms (flat, cum) 0.49% 961 . . 962 . . } 963 . . mp.mallocing = 1 964 . . 965 . . checkGCTrigger := false 966 10ms 10ms c := getMCache(mp) c = pp.mcache mcache.go:150 967 . . const spc = spanClass(sizeclass<<1) | spanClass(0) 968 . . span := c.alloc[spc] 969 . . 970 . . var v gclinkptr 971 . . var x unsafe.Pointer 972 . . 973 . . { 974 . . 975 . . var nextFreeFastResult gclinkptr 976 50ms 50ms if span.allocCache != 0 { 977 . . theBit := sys.TrailingZeros64(span.allocCache) 978 . . result := span.freeindex + uint16(theBit) 979 . . if result < span.nelems { 980 . . freeidx := result + 1 981 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 982 . . span.allocCache >>= uint(theBit + 1) 983 . . span.freeindex = freeidx 984 . . span.allocCount++ 985 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 986 . . } 987 . . } 988 . . } 989 . . v = nextFreeFastResult 990 . . if v == 0 { 991 . 100ms v, span, checkGCTrigger = c.nextFree(spc) 992 . . } 993 10ms 10ms x = unsafe.Pointer(v) 994 . . } 995 . . 996 . . if span.needzero != 0 { 997 30ms 30ms memclrNoHeapPointers(x, elemsize) 998 . . } 999 . . if goarch.PtrSize == 8 && elemsize == 8 { 1000 . . 1001 . . c.scanAlloc += 8 1002 . . } else { 1003 . . dataSize := size 1004 . . x := uintptr(x) 1005 . . 1006 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) { 1007 . . throw("tried to write heap bits, but no heap bits in span") 1008 . . } 1009 . . 1010 40ms 40ms src0 := readUintptr(getGCMask(typ)) if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { type.go:88 1011 . . 1012 . . const elemsize = 80 1013 . . 1014 . . var scanSize uintptr 1015 . . src := src0 1016 10ms 10ms if typ.Size_ == goarch.PtrSize { 1017 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 1018 . . 1019 . . scanSize = dataSize 1020 . . } else { 1021 . . 1022 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 1023 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 1024 . . } 1025 . . scanSize = typ.PtrBytes 1026 . . for i := typ.Size_; i < dataSize; i += typ.Size_ { 1027 10ms 10ms src |= src0 << (i / goarch.PtrSize) 1028 . . scanSize += typ.Size_ 1029 . . } 1030 . . } 1031 . . 1032 10ms 10ms dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) 1033 . . dst := unsafe.Pointer(dstBase) 1034 . . o := (x - span.base()) / goarch.PtrSize 1035 . . i := o / ptrBits 1036 . . j := o % ptrBits 1037 . . var bits uintptr = elemsize / goarch.PtrSize 1038 . . 1039 . . var bitsIsPowerOfTwo = bits&(bits-1) == 0 1040 . . if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { 1041 . . 1042 10ms 10ms bits0 := ptrBits - j 1043 . . bits1 := bits - bits0 1044 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 1045 . . dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) 1046 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 1047 10ms 10ms *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 1048 . . } else { 1049 . . 1050 . . dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 1051 20ms 20ms *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) 1052 . . } 1053 . . 1054 . . const doubleCheck = false 1055 . . if doubleCheck { 1056 . . writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) 1057 . . } 1058 . . if doubleCheckHeapSetType { 1059 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 1060 . . } 1061 10ms 10ms c.scanAlloc += scanSize 1062 . . } 1063 . . 1064 . . publicationBarrier() 1065 . . 1066 . . span.freeIndexForScan = span.freeindex 1067 . . 1068 10ms 10ms c.nextSample -= int64(elemsize) 1069 10ms 10ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1070 . . profilealloc(mp, x, elemsize) 1071 . . } 1072 10ms 10ms mp.mallocing = 0 1073 . . releasem(mp) 1074 . . 1075 . . if checkGCTrigger { 1076 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1077 . . gcStart(t)
runtime.mallocgcTinySC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 310ms 370ms (flat, cum) 0.52% 1079 . . } 1080 . . 1081 . . return x 1082 . . } 1083 . . 1084 50ms 50ms func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1085 . . if doubleCheckMalloc { 1086 . . if gcphase == _GCmarktermination { 1087 . . throw("mallocgc called with gcphase == _GCmarktermination") 1088 . . } 1089 . . } 1090 . . 1091 . . var mp *m 1092 . . 1093 40ms 40ms mp = acquirem() gp.m.locks++ runtime1.go:614 ⋮ ⋮ return gp.m runtime1.go:615 1094 . . 1095 10ms 10ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 1096 . . 1097 . . if forceSlowPath { 1098 . . releasem(mp) 1099 . . 1100 . . return mallocgcTinySlowPath(size, typ, needzero) 1101 . . 1102 . . } 1103 . . 1104 . . lockRankMayQueueFinalizer() 1105 . . 1106 . . const elemsize = 16 1107 . . 1108 . . if doubleCheckMalloc { 1109 . . doubleCheckTiny(size, typ, mp) 1110 . . } 1111 . . mp.mallocing = 1 1112 . . 1113 10ms 10ms c := getMCache(mp) c = pp.mcache mcache.go:150 1114 . . off := c.tinyoffset 1115 . . 1116 30ms 30ms if size&7 == 0 { 1117 . . off = alignUp(off, 8) 1118 . . } else if goarch.PtrSize == 4 && size == 12 { 1119 . . 1120 . . off = alignUp(off, 8) 1121 . . } else if size&3 == 0 { 1122 10ms 10ms off = alignUp(off, 4) return (n + a - 1) &^ (a - 1) stubs.go:384 1123 . . } else if size&1 == 0 { 1124 . . off = alignUp(off, 2) 1125 . . } 1126 20ms 20ms if off+size <= maxTinySize && c.tiny != 0 { 1127 . . 1128 20ms 20ms x := unsafe.Pointer(c.tiny + off) 1129 . . c.tinyoffset = off + size 1130 . . c.tinyAllocs++ 1131 . . mp.mallocing = 0 1132 30ms 30ms releasem(mp) ⋮ if mp.locks == 0 && gp.preempt { runtime1.go:622 1133 . . const elemsize = 0 1134 . . 1135 . . return x 1136 . . } 1137 . . 1138 . . checkGCTrigger := false 1139 10ms 10ms span := c.alloc[tinySpanClass] 1140 . . 1141 . . const nbytes = 8192 1142 . . const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) 1143 . . var nextFreeFastResult gclinkptr 1144 40ms 40ms if span.allocCache != 0 { 1145 . . theBit := sys.TrailingZeros64(span.allocCache) 1146 . . result := span.freeindex + uint16(theBit) 1147 . . if result < nelems { 1148 . . freeidx := result + 1 1149 . . if !(freeidx%64 == 0 && freeidx != nelems) { 1150 . . span.allocCache >>= uint(theBit + 1) 1151 . . span.freeindex = freeidx 1152 . . span.allocCount++ 1153 . . nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) 1154 . . } 1155 . . } 1156 . . } 1157 . . v := nextFreeFastResult 1158 . . if v == 0 { 1159 . 60ms v, span, checkGCTrigger = c.nextFree(tinySpanClass) 1160 . . } 1161 . . x := unsafe.Pointer(v) 1162 . . (*[2]uint64)(x)[0] = 0 1163 . . (*[2]uint64)(x)[1] = 0 1164 . . 1165 . . if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { 1166 . . 1167 . . c.tiny = uintptr(x) 1168 . . c.tinyoffset = size 1169 . . } 1170 . . 1171 . . publicationBarrier() 1172 . . 1173 . . span.freeIndexForScan = span.freeindex 1174 . . 1175 . . c.nextSample -= int64(elemsize) 1176 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1177 . . profilealloc(mp, x, elemsize) 1178 . . } 1179 10ms 10ms mp.mallocing = 0 1180 . . releasem(mp) 1181 . . 1182 10ms 10ms if checkGCTrigger { 1183 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1184 . . gcStart(t) 1185 . . } 1186 . . } 1187 . . 1188 20ms 20ms return x 1189 . . } 1190 . . 1191 . . func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1192 . . if doubleCheckMalloc { 1193 . . if gcphase == _GCmarktermination {
runtime.mallocgcSmallNoScanSC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1197 . . 1198 . . var mp *m 1199 . . 1200 . . mp = acquirem() 1201 . . 1202 10ms 10ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 1203 . . 1204 . . if forceSlowPath { 1205 . . releasem(mp) 1206 . . 1207 . . const spc = spanClass(2<<1) | spanClass(1)
runtime.mallocgcSmallNoScanSC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 20ms (flat, cum) 0.028% 1241 . . } 1242 . . 1243 . . { 1244 . . 1245 . . var nextFreeFastResult gclinkptr 1246 10ms 10ms if span.allocCache != 0 { 1247 . . theBit := sys.TrailingZeros64(span.allocCache) 1248 . . result := span.freeindex + uint16(theBit) 1249 . . if result < span.nelems { 1250 . . freeidx := result + 1 1251 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 1252 . . span.allocCache >>= uint(theBit + 1) 1253 . . span.freeindex = freeidx 1254 . . span.allocCount++ 1255 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 1256 . . } 1257 . . } 1258 . . } 1259 . . v = nextFreeFastResult 1260 . . if v == 0 { 1261 . 10ms v, span, checkGCTrigger = c.nextFree(spc) 1262 . . } 1263 . . x = unsafe.Pointer(v) 1264 . . } 1265 . . 1266 . . if needzero && span.needzero != 0 {
runtime.mallocgcSmallNoScanSC2
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1282 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1283 . . gcStart(t) 1284 . . } 1285 . . } 1286 . . 1287 10ms 10ms return x 1288 . . } 1289 . . 1290 . . func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1291 . . if doubleCheckMalloc { 1292 . . if gcphase == _GCmarktermination {
runtime.mallocgcSmallNoScanSC3
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1296 . . 1297 . . var mp *m 1298 . . 1299 . . mp = acquirem() 1300 . . 1301 10ms 10ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 1302 . . 1303 . . if forceSlowPath { 1304 . . releasem(mp) 1305 . . 1306 . . const spc = spanClass(3<<1) | spanClass(1)
runtime.mallocgcSmallNoScanSC3
/usr/lib/go/src/runtime/malloc_generated.go
Total: 40ms 70ms (flat, cum) 0.098% 1344 . . var nextFreeFastResult gclinkptr 1345 . . if span.allocCache != 0 { 1346 . . theBit := sys.TrailingZeros64(span.allocCache) 1347 . . result := span.freeindex + uint16(theBit) 1348 . . if result < span.nelems { 1349 10ms 10ms freeidx := result + 1 1350 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 1351 . . span.allocCache >>= uint(theBit + 1) 1352 . . span.freeindex = freeidx 1353 . . span.allocCount++ 1354 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 1355 . . } 1356 . . } 1357 . . } 1358 . . v = nextFreeFastResult 1359 . . if v == 0 { 1360 . 30ms v, span, checkGCTrigger = c.nextFree(spc) 1361 . . } 1362 . . x = unsafe.Pointer(v) 1363 . . } 1364 . . 1365 . . if needzero && span.needzero != 0 { 1366 . . memclrNoHeapPointers(x, elemsize) 1367 . . } 1368 . . 1369 . . publicationBarrier() 1370 . . 1371 10ms 10ms span.freeIndexForScan = span.freeindex 1372 . . 1373 . . c.nextSample -= int64(elemsize) 1374 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1375 . . profilealloc(mp, x, elemsize) 1376 . . } 1377 . . mp.mallocing = 0 1378 . . releasem(mp) 1379 . . 1380 . . if checkGCTrigger { 1381 . . if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1382 . . gcStart(t) 1383 . . } 1384 . . } 1385 . . 1386 20ms 20ms return x 1387 . . }
runtime.mallocgcSmallNoScanSC4
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1388 . . 1389 10ms 10ms func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1390 . . if doubleCheckMalloc { 1391 . . if gcphase == _GCmarktermination { 1392 . . throw("mallocgc called with gcphase == _GCmarktermination") 1393 . . } 1394 . . }
runtime.mallocgcSmallNoScanSC4
/usr/lib/go/src/runtime/malloc_generated.go
Total: 50ms 60ms (flat, cum) 0.084% 1439 . . } 1440 . . 1441 . . { 1442 . . 1443 . . var nextFreeFastResult gclinkptr 1444 30ms 30ms if span.allocCache != 0 { 1445 . . theBit := sys.TrailingZeros64(span.allocCache) 1446 . . result := span.freeindex + uint16(theBit) 1447 . . if result < span.nelems { 1448 . . freeidx := result + 1 1449 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 1450 . . span.allocCache >>= uint(theBit + 1) 1451 . . span.freeindex = freeidx 1452 . . span.allocCount++ 1453 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 1454 . . } 1455 . . } 1456 . . } 1457 . . v = nextFreeFastResult 1458 . . if v == 0 { 1459 . 10ms v, span, checkGCTrigger = c.nextFree(spc) 1460 . . } 1461 . . x = unsafe.Pointer(v) 1462 . . } 1463 . . 1464 . . if needzero && span.needzero != 0 { 1465 . . memclrNoHeapPointers(x, elemsize) 1466 . . } 1467 . . 1468 . . publicationBarrier() 1469 . . 1470 10ms 10ms span.freeIndexForScan = span.freeindex 1471 . . 1472 10ms 10ms c.nextSample -= int64(elemsize) 1473 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1474 . . profilealloc(mp, x, elemsize) 1475 . . } 1476 . . mp.mallocing = 0 1477 . . releasem(mp)
runtime.mallocgcSmallNoScanSC5
/usr/lib/go/src/runtime/malloc_generated.go
Total: 30ms 30ms (flat, cum) 0.042% 1483 . . } 1484 . . 1485 . . return x 1486 . . } 1487 . . 1488 30ms 30ms func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1489 . . if doubleCheckMalloc { 1490 . . if gcphase == _GCmarktermination { 1491 . . throw("mallocgc called with gcphase == _GCmarktermination") 1492 . . } 1493 . . }
runtime.mallocgcSmallNoScanSC5
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1515 . . if doubleCheckMalloc { 1516 . . 1517 . . doubleCheckSmallNoScan(typ, mp) 1518 . . 1519 . . } 1520 10ms 10ms mp.mallocing = 1 1521 . . 1522 . . checkGCTrigger := false 1523 . . c := getMCache(mp) 1524 . . const spc = spanClass(sizeclass<<1) | spanClass(1) 1525 . . span := c.alloc[spc]
runtime.mallocgcSmallNoScanSC5
/usr/lib/go/src/runtime/malloc_generated.go
Total: 40ms 70ms (flat, cum) 0.098% 1538 . . } 1539 . . 1540 . . { 1541 . . 1542 . . var nextFreeFastResult gclinkptr 1543 20ms 20ms if span.allocCache != 0 { 1544 10ms 10ms theBit := sys.TrailingZeros64(span.allocCache) 1545 . . result := span.freeindex + uint16(theBit) 1546 . . if result < span.nelems { 1547 10ms 10ms freeidx := result + 1 1548 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 1549 . . span.allocCache >>= uint(theBit + 1) 1550 . . span.freeindex = freeidx 1551 . . span.allocCount++ 1552 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 1553 . . } 1554 . . } 1555 . . } 1556 . . v = nextFreeFastResult 1557 . . if v == 0 { 1558 . 30ms v, span, checkGCTrigger = c.nextFree(spc) 1559 . . } 1560 . . x = unsafe.Pointer(v) 1561 . . } 1562 . . 1563 . . if needzero && span.needzero != 0 {
runtime.mallocgcSmallNoScanSC6
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1593 . . 1594 . . var mp *m 1595 . . 1596 . . mp = acquirem() 1597 . . 1598 10ms 10ms forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0) 1599 . . 1600 . . if forceSlowPath { 1601 . . releasem(mp) 1602 . . 1603 . . const spc = spanClass(6<<1) | spanClass(1)
runtime.mallocgcSmallNoScanSC6
/usr/lib/go/src/runtime/malloc_generated.go
Total: 40ms 40ms (flat, cum) 0.056% 1637 . . } 1638 . . 1639 . . { 1640 . . 1641 . . var nextFreeFastResult gclinkptr 1642 20ms 20ms if span.allocCache != 0 { 1643 . . theBit := sys.TrailingZeros64(span.allocCache) 1644 . . result := span.freeindex + uint16(theBit) 1645 . . if result < span.nelems { 1646 . . freeidx := result + 1 1647 . . if !(freeidx%64 == 0 && freeidx != span.nelems) { 1648 . . span.allocCache >>= uint(theBit + 1) 1649 . . span.freeindex = freeidx 1650 . . span.allocCount++ 1651 . . nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base()) 1652 . . } 1653 . . } 1654 . . } 1655 . . v = nextFreeFastResult 1656 10ms 10ms if v == 0 { 1657 . . v, span, checkGCTrigger = c.nextFree(spc) 1658 . . } 1659 . . x = unsafe.Pointer(v) 1660 . . } 1661 . . 1662 . . if needzero && span.needzero != 0 { 1663 . . memclrNoHeapPointers(x, elemsize) 1664 . . } 1665 . . 1666 . . publicationBarrier() 1667 . . 1668 10ms 10ms span.freeIndexForScan = span.freeindex 1669 . . 1670 . . c.nextSample -= int64(elemsize) 1671 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1672 . . profilealloc(mp, x, elemsize) 1673 . . }
runtime.mallocgcSmallNoScanSC7
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1681 . . } 1682 . . 1683 . . return x 1684 . . } 1685 . . 1686 10ms 10ms func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1687 . . if doubleCheckMalloc { 1688 . . if gcphase == _GCmarktermination { 1689 . . throw("mallocgc called with gcphase == _GCmarktermination") 1690 . . } 1691 . . }
runtime.mallocgcSmallNoScanSC7
/usr/lib/go/src/runtime/malloc_generated.go
Total: 10ms 10ms (flat, cum) 0.014% 1736 . . } 1737 . . 1738 . . { 1739 . . 1740 . . var nextFreeFastResult gclinkptr 1741 10ms 10ms if span.allocCache != 0 { 1742 . . theBit := sys.TrailingZeros64(span.allocCache) 1743 . . result := span.freeindex + uint16(theBit) 1744 . . if result < span.nelems { 1745 . . freeidx := result + 1 1746 . . if !(freeidx%64 == 0 && freeidx != span.nelems) {
runtime.gopark
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 467 . . mp.waitlock = lock 468 . . mp.waitunlockf = unlockf 469 . . gp.waitreason = reason 470 . . mp.waitTraceBlockReason = traceReason 471 . . mp.waitTraceSkip = traceskip 472 10ms 10ms releasem(mp) 473 . . // can't do anything that might move the G between Ms here. 474 10ms 10ms mcall(park_m) 475 . . } 476 . .
runtime.goparkunlock
/usr/lib/go/src/runtime/proc.go
Total: 0 10ms (flat, cum) 0.014% 478 . . // The goroutine can be made runnable again by calling goready(gp). 479 . . func goparkunlock(lock *mutex, reason waitReason, traceReason traceBlockReason, traceskip int) { 480 . 10ms gopark(parkunlock_c, unsafe.Pointer(lock), reason, traceReason, traceskip) 481 . . } 482 . . 483 . . // goready should be an internal detail, 484 . . // but widely used packages access it using linkname. 485 . . // Notable members of the hall of shame include:
runtime.goready
/usr/lib/go/src/runtime/proc.go
Total: 0 410ms (flat, cum) 0.57% 489 . . // Do not remove or change the type signature. 490 . . // See go.dev/issue/67401. 491 . . // 492 . . //go:linkname goready 493 . . func goready(gp *g, traceskip int) { 494 . 410ms systemstack(func() {
_.goready.func1
/usr/lib/go/src/runtime/proc.go
Total: 0 410ms (flat, cum) 0.57% 495 . 410ms ready(gp, traceskip, true) 496 . . }) 497 . . } 498 . . 499 . . //go:nosplit 500 . . func acquireSudog() *sudog {
runtime.releaseSudog
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 553 . . } 554 . . if s.c.get() != nil { 555 . . throw("runtime: sudog with non-nil c") 556 . . } 557 . . gp := getg() 558 10ms 10ms if gp.param != nil { 559 . . throw("runtime: releaseSudog with non-nil gp.param") 560 . . } 561 . . mp := acquirem() // avoid rescheduling to another P 562 . . pp := mp.p.ptr() 563 . . if len(pp.sudogcache) == cap(pp.sudogcache) {
runtime.(*m).becomeSpinning
/usr/lib/go/src/runtime/proc.go
Total: 30ms 30ms (flat, cum) 0.042% 1083 . . //go:linkname pprof_makeProfStack 1084 . . func pprof_makeProfStack() []uintptr { return makeProfStack() } 1085 . . 1086 . . func (mp *m) becomeSpinning() { 1087 . . mp.spinning = true 1088 30ms 30ms sched.nmspinning.Add(1) return Xaddint32(&i.value, delta) types.go:56 1089 . . sched.needspinning.Store(0) 1090 . . } 1091 . . 1092 . . // Take a snapshot of allp, for use after dropping the P. 1093 . . //
runtime.(*m).clearAllpSnapshot
/usr/lib/go/src/runtime/proc.go
Total: 40ms 40ms (flat, cum) 0.056% 1105 . . // no longer required. 1106 . . // 1107 . . // Must be called after reacquiring a P, as it requires a write barrier. 1108 . . // 1109 . . //go:yeswritebarrierrec 1110 30ms 30ms func (mp *m) clearAllpSnapshot() { 1111 . . mp.allpSnapshot = nil 1112 10ms 10ms } 1113 . . 1114 . . func (mp *m) hasCgoOnStack() bool { 1115 . . return mp.ncgo > 0 || mp.isextra 1116 . . } 1117 . .
runtime.ready
/usr/lib/go/src/runtime/proc.go
Total: 0 410ms (flat, cum) 0.57% 1140 . . throw("bad g->status in ready") 1141 . . } 1142 . . 1143 . . // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq 1144 . . trace := traceAcquire() 1145 . 10ms casgstatus(gp, _Gwaiting, _Grunnable) 1146 . . if trace.ok() { 1147 . . trace.GoUnpark(gp, traceskip) 1148 . . traceRelease(trace) 1149 . . } 1150 . 30ms runqput(mp.p.ptr(), gp, next) 1151 . 370ms wakep() 1152 . . releasem(mp) 1153 . . } 1154 . . 1155 . . // freezeStopWait is a large value that freezetheworld sets 1156 . . // sched.stopwait to in order to request that all Gs permanently stop.
runtime.casgstatus
/usr/lib/go/src/runtime/proc.go
Total: 420ms 420ms (flat, cum) 0.59% 1285 . . // and casfrom_Gscanstatus instead. 1286 . . // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that 1287 . . // put it in the Gscan state is finished. 1288 . . // 1289 . . //go:nosplit 1290 10ms 10ms func casgstatus(gp *g, oldval, newval uint32) { 1291 30ms 30ms if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { 1292 . . systemstack(func() { 1293 . . // Call on the systemstack to prevent print and throw from counting 1294 . . // against the nosplit stack reservation. 1295 . . print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") 1296 . . throw("casgstatus: bad incoming values") 1297 . . }) 1298 . . } 1299 . . 1300 . . lockWithRankMayAcquire(nil, lockRankGscan) 1301 . . 1302 . . // See https://golang.org/cl/21503 for justification of the yield delay. 1303 . . const yieldDelay = 5 * 1000 1304 . . var nextYield int64 1305 . . 1306 . . // loop if gp->atomicstatus is in a scan state giving 1307 . . // GC time to finish and change the state to oldval. 1308 380ms 380ms for i := 0; !gp.atomicstatus.CompareAndSwap(oldval, newval); i++ { return Cas(&u.value, old, new) types.go:236 ⋮ ⋮ ⋮ ⋮ return Cas(&u.value, old, new) types.go:236 1309 . . if oldval == _Gwaiting && gp.atomicstatus.Load() == _Grunnable { 1310 . . systemstack(func() { 1311 . . // Call on the systemstack to prevent throw from counting 1312 . . // against the nosplit stack reservation. 1313 . . throw("casgstatus: waiting for Gwaiting but is Grunnable")
runtime.casgstatus
/usr/lib/go/src/runtime/proc.go
Total: 80ms 80ms (flat, cum) 0.11% 1324 . . osyield() 1325 . . nextYield = nanotime() + yieldDelay/2 1326 . . } 1327 . . } 1328 . . 1329 20ms 20ms if gp.bubble != nil { 1330 . . systemstack(func() { 1331 . . gp.bubble.changegstatus(gp, oldval, newval) 1332 . . }) 1333 . . } 1334 . . 1335 . . if (oldval == _Grunning || oldval == _Gsyscall) && (newval != _Grunning && newval != _Gsyscall) { 1336 . . // Track every gTrackingPeriod time a goroutine transitions out of _Grunning or _Gsyscall. 1337 . . // Do not track _Grunning <-> _Gsyscall transitions, since they're two very similar states. 1338 10ms 10ms if casgstatusAlwaysTrack || gp.trackingSeq%gTrackingPeriod == 0 { 1339 . . gp.tracking = true 1340 . . } 1341 20ms 20ms gp.trackingSeq++ 1342 . . } 1343 10ms 10ms if !gp.tracking { 1344 10ms 10ms return 1345 . . } 1346 . . 1347 . . // Handle various kinds of tracking. 1348 . . // 1349 . . // Currently: 1350 . . // - Time spent in runnable. 1351 . . // - Time spent blocked on a sync.Mutex or sync.RWMutex. 1352 . . switch oldval { 1353 . . case _Grunnable: 1354 . . // We transitioned out of runnable, so measure how much 1355 . . // time we spent in this state and add it to 1356 . . // runnableTime. 1357 10ms 10ms now := nanotime() return nanotime1() time_nofake.go:33 1358 . . gp.runnableTime += now - gp.trackingStamp 1359 . . gp.trackingStamp = 0 1360 . . case _Gwaiting: 1361 . . if !gp.waitreason.isMutexWait() { 1362 . . // Not blocking on a lock.
runtime.casgstatus
/usr/lib/go/src/runtime/proc.go
Total: 30ms 150ms (flat, cum) 0.21% 1381 . . now := nanotime() 1382 . . gp.trackingStamp = now 1383 . . case _Grunnable: 1384 . . // We just transitioned into runnable, so record what 1385 . . // time that happened. 1386 20ms 20ms now := nanotime() return nanotime1() time_nofake.go:33 1387 10ms 10ms gp.trackingStamp = now 1388 . . case _Grunning: 1389 . . // We're transitioning into running, so turn off 1390 . . // tracking and record how much time we spent in 1391 . . // runnable. 1392 . . gp.tracking = false 1393 . 120ms sched.timeToRun.record(gp.runnableTime) 1394 . . gp.runnableTime = 0 1395 . . } 1396 . . } 1397 . . 1398 . . // casGToWaiting transitions gp from old to _Gwaiting, and sets the wait reason.
runtime.mPark
/usr/lib/go/src/runtime/proc.go
Total: 10ms 1.01s (flat, cum) 1.41% 1980 . . // This M might stay parked through an entire GC cycle. 1981 . . // Erase any leftovers on the signal stack. 1982 . . if goexperiment.RuntimeSecret { 1983 . . eraseSecretsSignalStk() 1984 . . } 1985 . 1s notesleep(&gp.m.park) 1986 10ms 10ms noteclear(&gp.m.park) 1987 . . } 1988 . . 1989 . . // mexit tears down and exits the current thread. 1990 . . // 1991 . . // Don't call this directly to exit the thread, since it must run at
runtime.forEachP
/usr/lib/go/src/runtime/proc.go
Total: 0 10ms (flat, cum) 0.014% 2125 . . // memory barrier. GC uses this as a "ragged barrier." 2126 . . // 2127 . . // The caller must hold worldsema. fn must not refer to any 2128 . . // part of the current goroutine's stack, since the GC may move it. 2129 . . func forEachP(reason waitReason, fn func(*p)) { 2130 . 10ms systemstack(func() { 2131 . . gp := getg().m.curg 2132 . . // Mark the user stack as preemptible so that it may be scanned 2133 . . // by the GC or observed by the execution tracer. Otherwise, our 2134 . . // attempt to force all P's to a safepoint could result in a 2135 . . // deadlock as we attempt to preempt a goroutine that's trying
runtime.forEachP.func1
/usr/lib/go/src/runtime/proc.go
Total: 0 10ms (flat, cum) 0.014% 2139 . . // stack shrink, effectively pinning the stack in memory for the duration. 2140 . . // 2141 . . // N.B. The execution tracer is not aware of this status transition and 2142 . . // handles it specially based on the wait reason. 2143 . . casGToWaitingForSuspendG(gp, _Grunning, reason) 2144 . 10ms forEachPInternal(fn) 2145 . . casgstatus(gp, _Gwaiting, _Grunning) 2146 . . }) 2147 . . } 2148 . . 2149 . . // forEachPInternal calls fn(p) for every P p when p reaches a GC safe point.
runtime.forEachPInternal
/usr/lib/go/src/runtime/proc.go
Total: 0 10ms (flat, cum) 0.014% 2180 . . 2181 . . // Run safe point function for all idle Ps. sched.pidle will 2182 . . // not change because we hold sched.lock. 2183 . . for p := sched.pidle.ptr(); p != nil; p = p.link.ptr() { 2184 . . if atomic.Cas(&p.runSafePointFn, 1, 0) { 2185 . 10ms fn(p) 2186 . . sched.safePointWait-- 2187 . . } 2188 . . } 2189 . . 2190 . . wait := sched.safePointWait > 0
runtime.stopm
/usr/lib/go/src/runtime/proc.go
Total: 30ms 1.21s (flat, cum) 1.69% 3015 . . } 3016 . . if gp.m.spinning { 3017 . . throw("stopm spinning") 3018 . . } 3019 . . 3020 . 30ms lock(&sched.lock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 3021 . 50ms mput(gp.m) 3022 . 30ms unlock(&sched.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 3023 10ms 1.01s mPark() notesleep(&gp.m.park) proc.go:1985 noteclear(&gp.m.park) proc.go:1986 3024 10ms 80ms acquirep(gp.m.nextp.ptr()) 3025 10ms 10ms gp.m.nextp = 0 3026 . . } 3027 . . 3028 . . func mspinning() { 3029 . . // startm's caller incremented nmspinning. Set the new M's spinning. 3030 . . getg().m.spinning = true
runtime.startm
/usr/lib/go/src/runtime/proc.go
Total: 0 60ms (flat, cum) 0.084% 3082 . . } 3083 . . releasem(mp) 3084 . . return 3085 . . } 3086 . . } 3087 . 60ms nmp := mget() mp := (*m)(sched.midle.pop()) proc.go:7267 3088 . . if nmp == nil { 3089 . . // No M is available, we must drop sched.lock and call newm. 3090 . . // However, we already own a P to assign to the M. 3091 . . // 3092 . . // Once sched.lock is released, another G (e.g., in a syscall),
runtime.startm
/usr/lib/go/src/runtime/proc.go
Total: 0 2.65s (flat, cum) 3.70% 3117 . . // Preemption is now safe. 3118 . . releasem(mp) 3119 . . return 3120 . . } 3121 . . if !lockheld { 3122 . 70ms unlock(&sched.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 3123 . . } 3124 . . if nmp.spinning { 3125 . . throw("startm: m is spinning") 3126 . . } 3127 . . if nmp.nextp != 0 { 3128 . . throw("startm: m has p") 3129 . . } 3130 . . if spinning && !runqempty(pp) { 3131 . . throw("startm: p has runnable gs") 3132 . . } 3133 . . // The caller incremented nmspinning, so set m.spinning in the new M. 3134 . . nmp.spinning = spinning 3135 . . nmp.nextp.set(pp) 3136 . 2.58s notewakeup(&nmp.park) 3137 . . // Ownership transfer of pp committed by wakeup. Preemption is now 3138 . . // safe. 3139 . . releasem(mp) 3140 . . } 3141 . .
runtime.wakep
/usr/lib/go/src/runtime/proc.go
Total: 90ms 2.99s (flat, cum) 4.18% 3225 . . // 3226 . . //go:linkname wakep 3227 . . func wakep() { 3228 . . // Be conservative about spinning threads, only start one if none exist 3229 . . // already. 3230 70ms 70ms if sched.nmspinning.Load() != 0 || !sched.nmspinning.CompareAndSwap(0, 1) { return Loadint32(&i.value) types.go:21 ⋮ return Casint32(&i.value, old, new) types.go:37 ⋮ 3231 . . return 3232 . . } 3233 . . 3234 . . // Disable preemption until ownership of pp transfers to the next M in 3235 . . // startm. Otherwise preemption here would leave pp stuck waiting to 3236 . . // enter _Pgcstop. 3237 . . // 3238 . . // See preemption comment on acquirem in startm for more details. 3239 . . mp := acquirem() 3240 . . 3241 . . var pp *p 3242 . 30ms lock(&sched.lock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 3243 . 160ms pp, _ = pidlegetSpinning(0) 3244 . . if pp == nil { 3245 20ms 20ms if sched.nmspinning.Add(-1) < 0 { return Xaddint32(&i.value, delta) types.go:56 3246 . . throw("wakep: negative nmspinning") 3247 . . } 3248 . . unlock(&sched.lock) 3249 . . releasem(mp) 3250 . . return 3251 . . } 3252 . . // Since we always have a P, the race in the "No M is available" 3253 . . // comment in startm doesn't apply during the small window between the 3254 . . // unlock here and lock in startm. A checkdead in between will always 3255 . . // see at least one running M (ours). 3256 . . unlock(&sched.lock) 3257 . . 3258 . 2.71s startm(pp, true, false) 3259 . . 3260 . . releasem(mp) 3261 . . } 3262 . . 3263 . . // Stops execution of the current m that is locked to a g until the g is runnable again.
runtime.execute
/usr/lib/go/src/runtime/proc.go
Total: 80ms 310ms (flat, cum) 0.43% 3344 . . // 3345 . . //go:yeswritebarrierrec 3346 . . func execute(gp *g, inheritTime bool) { 3347 . . mp := getg().m 3348 . . 3349 10ms 10ms if goroutineProfile.active { 3350 . . // Make sure that gp has had its stack written out to the goroutine 3351 . . // profile, exactly as it was when the goroutine profiler first stopped 3352 . . // the world. 3353 . . tryRecordGoroutineProfile(gp, nil, osyield) 3354 . . } 3355 . . 3356 . . // Assign gp.m before entering _Grunning so running Gs have an M. 3357 20ms 20ms mp.curg = gp 3358 20ms 20ms gp.m = mp 3359 . . gp.syncSafePoint = false // Clear the flag, which may have been set by morestack. 3360 . 230ms casgstatus(gp, _Grunnable, _Grunning) 3361 . . gp.waitsince = 0 3362 10ms 10ms gp.preempt = false 3363 . . gp.stackguard0 = gp.stack.lo + stackGuard 3364 . . if !inheritTime { 3365 20ms 20ms mp.p.ptr().schedtick++ 3366 . . } 3367 . . 3368 . . if sys.DITSupported && debug.dataindependenttiming != 1 { 3369 . . if gp.ditWanted && !mp.ditEnabled { 3370 . . // The current M doesn't have DIT enabled, but the goroutine we're
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 350ms 440ms (flat, cum) 0.62% 3399 . . 3400 . . // Finds a runnable goroutine to execute. 3401 . . // Tries to steal from other P's, get g from local or global queue, poll network. 3402 . . // tryWakeP indicates that the returned goroutine is not normal (GC worker, trace 3403 . . // reader) so the caller should try to wake a P. 3404 10ms 10ms func findRunnable() (gp *g, inheritTime, tryWakeP bool) { 3405 . . mp := getg().m 3406 . . 3407 . . // The conditions here and in handoffp must agree: if 3408 . . // findRunnable would return a G to run, handoffp must start 3409 . . // an M. 3410 . . 3411 . . top: 3412 . . // We may have collected an allp snapshot below. The snapshot is only 3413 . . // required in each loop iteration. Clear it to all GC to collect the 3414 . . // slice. 3415 20ms 50ms mp.clearAllpSnapshot() 3416 . . 3417 . . pp := mp.p.ptr() 3418 30ms 30ms if sched.gcwaiting.Load() { return b.u.Load() != 0 types.go:168 3419 . . gcstopm() 3420 . . goto top 3421 . . } 3422 20ms 20ms if pp.runSafePointFn != 0 { 3423 . . runSafePointFn() 3424 . . } 3425 . . 3426 . . // now and pollUntil are saved for work stealing later, 3427 . . // which may steal timers. It's important that between now 3428 . . // and then, nothing blocks, so these numbers remain mostly 3429 . . // relevant. 3430 . 50ms now, pollUntil, _ := pp.timers.check(0, nil) 3431 . . 3432 . . // Try to schedule the trace reader. 3433 . . if traceEnabled() || traceShuttingDown() { 3434 . . gp := traceReader() 3435 . . if gp != nil { 3436 . . trace := traceAcquire() 3437 . . casgstatus(gp, _Gwaiting, _Grunnable) 3438 . . if trace.ok() { 3439 . . trace.GoUnpark(gp, 0) 3440 . . traceRelease(trace) 3441 . . } 3442 . . return gp, false, true 3443 . . } 3444 . . } 3445 . . 3446 . . // Try to schedule a GC worker. 3447 20ms 20ms if gcBlackenEnabled != 0 { 3448 . . gp, tnow := gcController.findRunnableGCWorker(pp, now) 3449 . . if gp != nil { 3450 . . return gp, false, true 3451 . . } 3452 . . now = tnow 3453 . . } 3454 . . 3455 . . // Check the global runnable queue once in a while to ensure fairness. 3456 . . // Otherwise two goroutines can completely occupy the local runqueue 3457 . . // by constantly respawning each other. 3458 10ms 10ms if pp.schedtick%61 == 0 && !sched.runq.empty() { 3459 . . lock(&sched.lock) 3460 . . gp := globrunqget() 3461 . . unlock(&sched.lock) 3462 . . if gp != nil { 3463 . . return gp, false, false 3464 . . } 3465 . . } 3466 . . 3467 . . // Wake up the finalizer G. 3468 . . if fingStatus.Load()&(fingWait|fingWake) == fingWait|fingWake { 3469 . . if gp := wakefing(); gp != nil { 3470 . . ready(gp, 0, true) 3471 . . } 3472 . . } 3473 . . 3474 . . // Wake up one or more cleanup Gs. 3475 40ms 40ms if gcCleanups.needsWake() { return q.workUnits.Load() > 0 && (q.asleep.Load() > 0 || q.ng.Load() < maxCleanupGs()) mcleanup.go:589 3476 . . gcCleanups.wake() 3477 . . } 3478 . . 3479 30ms 30ms if *cgo_yield != nil { 3480 . . asmcgocall(*cgo_yield, nil) 3481 . . } 3482 . . 3483 . . // local runq 3484 140ms 140ms if gp, inheritTime := runqget(pp); gp != nil { if atomic.CasRel(&pp.runqhead, h, h+1) { // cas-release, commits consume proc.go:7666 ⋮ next := pp.runnext proc.go:7651 ⋮ t := pp.runqtail proc.go:7661 ⋮ if next != 0 && pp.runnext.cas(next, 0) { proc.go:7655 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) runtime2.go:252 ⋮ h := atomic.LoadAcq(&pp.runqhead) // load-acquire, synchronize with other consumers proc.go:7660 ⋮ gp := pp.runq[h%uint32(len(pp.runq))].ptr() proc.go:7665 3485 . . return gp, inheritTime, false 3486 . . } 3487 . . 3488 . . // global runq 3489 . . if !sched.runq.empty() { 3490 . . lock(&sched.lock) 3491 . . gp, q := globrunqgetbatch(int32(len(pp.runq)) / 2) 3492 . 10ms unlock(&sched.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 3493 . . if gp != nil { 3494 . . if runqputbatch(pp, &q); !q.empty() { 3495 . . throw("Couldn't put Gs into empty local runq") 3496 . . } 3497 . . return gp, false, false 3498 . . } 3499 . . } 3500 . . 3501 . . // Poll network. 3502 . . // This netpoll is only an optimization before we resort to stealing. 3503 . . // We can safely skip it if there are no waiters or a thread is blocked 3504 . . // in netpoll already. If there is any kind of logical race with that 3505 . . // blocked thread (e.g. it has already returned from netpoll, but does 3506 . . // not set lastpoll yet), this thread will do blocking netpoll below 3507 . . // anyway. 3508 . . // We only poll from one thread at a time to avoid kernel contention 3509 . . // on machines with many cores. 3510 30ms 30ms if netpollinited() && netpollAnyWaiters() && sched.lastpoll.Load() != 0 && sched.pollingNet.Swap(1) == 0 { ⋮ return netpollWaiters.Load() > 0 netpoll.go:678 ⋮ return Load(&u.value) types.go:194 3511 . . list, delta := netpoll(0) 3512 . . sched.pollingNet.Store(0) 3513 . . if !list.empty() { // non-blocking 3514 . . gp := list.pop() 3515 . . injectglist(&list)
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 100ms 830ms (flat, cum) 1.16% 3527 . . // Spinning Ms: steal work from other Ps. 3528 . . // 3529 . . // Limit the number of spinning Ms to half the number of busy Ps. 3530 . . // This is necessary to prevent excessive CPU consumption when 3531 . . // GOMAXPROCS>>1 but the program parallelism is low. 3532 50ms 50ms if mp.spinning || 2*sched.nmspinning.Load() < gomaxprocs-sched.npidle.Load() { 3533 . . if !mp.spinning { 3534 30ms 30ms mp.becomeSpinning() sched.nmspinning.Add(1) proc.go:1088 return Xaddint32(&i.value, delta) types.go:56 3535 . . } 3536 . . 3537 . 730ms gp, inheritTime, tnow, w, newWork := stealWork(now) 3538 . . if gp != nil { 3539 . . // Successfully stole. 3540 . . return gp, inheritTime, false 3541 . . } 3542 10ms 10ms if newWork { 3543 . . // There may be new timer or GC work; restart to 3544 . . // discover. 3545 . . goto top 3546 . . } 3547 . . 3548 . . now = tnow 3549 . . if w != 0 && (pollUntil == 0 || w < pollUntil) { 3550 . . // Earlier timer to wait for. 3551 . . pollUntil = w 3552 . . } 3553 . . } 3554 . . 3555 . . // We have nothing to do. 3556 . . // 3557 . . // If we're in the GC mark phase, can safely scan and blacken objects, 3558 . . // and have work to do, run idle-time marking rather than give up the P. 3559 10ms 10ms if gcBlackenEnabled != 0 && gcShouldScheduleWorker(pp) && gcController.addIdleMarkWorker() { 3560 . . node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop()) 3561 . . if node != nil { 3562 . . pp.gcMarkWorkerMode = gcMarkWorkerIdleMode 3563 . . gp := node.gp.ptr() 3564 . .
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 0 60ms (flat, cum) 0.084% 3604 . . // len to change out from under us. 3605 . . idlepMaskSnapshot := idlepMask 3606 . . timerpMaskSnapshot := timerpMask 3607 . . 3608 . . // return P and block 3609 . 60ms lock(&sched.lock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 3610 . . if sched.gcwaiting.Load() || pp.runSafePointFn != 0 { 3611 . . unlock(&sched.lock) 3612 . . goto top 3613 . . } 3614 . . if !sched.runq.empty() {
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 0 130ms (flat, cum) 0.18% 3626 . . // See "Delicate dance" comment below. 3627 . . mp.becomeSpinning() 3628 . . unlock(&sched.lock) 3629 . . goto top 3630 . . } 3631 . 30ms if releasep() != pp { 3632 . . throw("findRunnable: wrong p") 3633 . . } 3634 . 40ms now = pidleput(pp, now) 3635 . 60ms unlock(&sched.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 3636 . . 3637 . . // Delicate dance: thread transitions from spinning to non-spinning 3638 . . // state, potentially concurrently with submission of new work. We must 3639 . . // drop nmspinning first and then check all sources again (with 3640 . . // #StoreLoad memory barrier in between). If we do it the other way
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 3671 . . // Also see "Worker thread parking/unparking" comment at the top of the 3672 . . // file. 3673 . . wasSpinning := mp.spinning 3674 . . if mp.spinning { 3675 . . mp.spinning = false 3676 10ms 10ms if sched.nmspinning.Add(-1) < 0 { return Xaddint32(&i.value, delta) types.go:56 3677 . . throw("findRunnable: negative nmspinning") 3678 . . } 3679 . . 3680 . . // Note the for correctness, only the last M transitioning from 3681 . . // spinning to non-spinning must perform these rechecks to
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 0 20ms (flat, cum) 0.028% 3713 . . mp.becomeSpinning() 3714 . . goto top 3715 . . } 3716 . . 3717 . . // Check for idle-priority GC work again. 3718 . 20ms pp, gp := checkIdleGCNoP() 3719 . . if pp != nil { 3720 . . acquirep(pp) 3721 . . mp.becomeSpinning() 3722 . . 3723 . . // Run the idle worker.
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 0 20ms (flat, cum) 0.028% 3735 . . // transitioning from spinning to non-spinning. 3736 . . // 3737 . . // Note that we cannot use checkTimers here because it calls 3738 . . // adjusttimers which may need to allocate memory, and that isn't 3739 . . // allowed when we don't have an active P. 3740 . 20ms pollUntil = checkTimersNoP(allpSnapshot, timerpMaskSnapshot, pollUntil) 3741 . . } 3742 . . 3743 . . // We don't need allp anymore at this pointer, but can't clear the 3744 . . // snapshot without a P for the write barrier.. 3745 . .
runtime.findRunnable
/usr/lib/go/src/runtime/proc.go
Total: 0 1.21s (flat, cum) 1.69% 3806 . . pollerPollUntil := sched.pollUntil.Load() 3807 . . if pollerPollUntil == 0 || pollerPollUntil > pollUntil { 3808 . . netpollBreak() 3809 . . } 3810 . . } 3811 . 1.21s stopm() 3812 . . goto top 3813 . . } 3814 . . 3815 . . // pollWork reports whether there is non-background work this P could 3816 . . // be doing. This is a fairly lightweight check to be used for
runtime.stealWork
/usr/lib/go/src/runtime/proc.go
Total: 230ms 340ms (flat, cum) 0.48% 3838 . . // 3839 . . // If newWork is true, new work may have been readied. 3840 . . // 3841 . . // If now is not 0 it is the current time. stealWork returns the passed time or 3842 . . // the current time if now was passed as 0. 3843 10ms 10ms func stealWork(now int64) (gp *g, inheritTime bool, rnow, pollUntil int64, newWork bool) { 3844 . . pp := getg().m.p.ptr() 3845 . . 3846 . . ranTimer := false 3847 . . 3848 . . const stealTries = 4 3849 10ms 10ms for i := 0; i < stealTries; i++ { 3850 40ms 40ms stealTimersOrRunNextG := i == stealTries-1 3851 . . 3852 60ms 60ms for enum := stealOrder.start(cheaprand()); !enum.done(); enum.next() { ⋮ enum.pos = (enum.pos + enum.inc) % enum.count proc.go:8086 ⋮ ⋮ return enum.i == enum.count proc.go:8081 ⋮ enum.pos = (enum.pos + enum.inc) % enum.count proc.go:8086 ⋮ mp.cheaprand += 0x53c5ca59 rand.go:237 3853 . . if sched.gcwaiting.Load() { 3854 . . // GC work may be available. 3855 . . return nil, false, now, pollUntil, true 3856 . . } 3857 30ms 30ms p2 := allp[enum.position()] ⋮ ⋮ return enum.pos proc.go:8090 3858 60ms 60ms if pp == p2 { 3859 . . continue 3860 . . } 3861 . . 3862 . . // Steal timers from p2. This call to checkTimers is the only place 3863 . . // where we might hold a lock on a different P's timers. We do this 3864 . . // once on the last pass before checking runnext because stealing 3865 . . // from the other P's runnext should be the last resort, so if there 3866 . . // are timers to steal do that first. 3867 . . // 3868 . . // We only check timers on one of the stealing iterations because 3869 . . // the time stored in now doesn't change in this loop and checking 3870 . . // the timers for each P more than once with the same value of now 3871 . . // is probably a waste of time. 3872 . . // 3873 . . // timerpMask tells us whether the P may have timers at all. If it 3874 . . // can't, no need to check at all. 3875 20ms 20ms if stealTimersOrRunNextG && timerpMask.read(enum.position()) { return (atomic.Load(&p[word]) & mask) != 0 proc.go:7370 ⋮ 3876 . 110ms tnow, w, ran := p2.timers.check(now, nil) 3877 . . now = tnow 3878 . . if w != 0 && (pollUntil == 0 || w < pollUntil) { 3879 . . pollUntil = w 3880 . . } 3881 . . if ran {
runtime.stealWork
/usr/lib/go/src/runtime/proc.go
Total: 20ms 390ms (flat, cum) 0.55% 3893 . . ranTimer = true 3894 . . } 3895 . . } 3896 . . 3897 . . // Don't bother to attempt to steal if p2 is idle. 3898 20ms 20ms if !idlepMask.read(enum.position()) { return (atomic.Load(&p[word]) & mask) != 0 proc.go:7370 ⋮ return enum.pos proc.go:8090 3899 . 370ms if gp := runqsteal(pp, p2, stealTimersOrRunNextG); gp != nil { 3900 . . return gp, false, now, pollUntil, ranTimer 3901 . . } 3902 . . } 3903 . . } 3904 . . }
runtime.checkTimersNoP
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 3934 . . } 3935 . . 3936 . . // Check all Ps for a timer expiring sooner than pollUntil. 3937 . . // 3938 . . // Returns updated pollUntil value. 3939 10ms 10ms func checkTimersNoP(allpSnapshot []*p, timerpMaskSnapshot pMask, pollUntil int64) int64 { 3940 . . for id, p2 := range allpSnapshot { 3941 . . if timerpMaskSnapshot.read(uint32(id)) { 3942 . . w := p2.timers.wakeTime() 3943 . . if w != 0 && (pollUntil == 0 || w < pollUntil) { 3944 . . pollUntil = w 3945 . . } 3946 . . } 3947 . . } 3948 . . 3949 10ms 10ms return pollUntil 3950 . . } 3951 . . 3952 . . // Check for idle-priority GC, without a P on entry.
runtime.checkIdleGCNoP
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 3953 . . // 3954 . . // If some GC work, a P, and a worker G are all available, the P and G will be 3955 . . // returned. The returned P has not been wired yet. 3956 10ms 10ms func checkIdleGCNoP() (*p, *g) { 3957 . . // N.B. Since we have no P, gcBlackenEnabled may change at any time; we 3958 . . // must check again after acquiring a P. As an optimization, we also check 3959 . . // if an idle mark worker is needed at all. This is OK here, because if we 3960 . . // observe that one isn't needed, at least one is currently running. Even if 3961 . . // it stops running, its own journey into the scheduler should schedule it 3962 . . // again, if need be (at which point, this check will pass, if relevant). 3963 10ms 10ms if atomic.Load(&gcBlackenEnabled) == 0 || !gcController.needIdleMarkWorker() { 3964 . . return nil, nil 3965 . . } 3966 . . if !gcShouldScheduleWorker(nil) { 3967 . . return nil, nil 3968 . . }
runtime.resetspinning
/usr/lib/go/src/runtime/proc.go
Total: 60ms 2.10s (flat, cum) 2.94% 4031 . . wakep() 4032 . . } 4033 . . } 4034 . . } 4035 . . 4036 30ms 30ms func resetspinning() { 4037 . . gp := getg() 4038 . . if !gp.m.spinning { 4039 . . throw("resetspinning: not a spinning m") 4040 . . } 4041 . . gp.m.spinning = false 4042 30ms 30ms nmspinning := sched.nmspinning.Add(-1) return Xaddint32(&i.value, delta) types.go:56 4043 . . if nmspinning < 0 { 4044 . . throw("findRunnable: negative nmspinning") 4045 . . } 4046 . . // M wakeup policy is deliberately somewhat conservative, so check if we 4047 . . // need to wakeup another P here. See "Worker thread parking/unparking" 4048 . . // comment at the top of the file for details. 4049 . 2.04s wakep() 4050 . . } 4051 . . 4052 . . // injectglist adds each runnable G on the list to some run queue, 4053 . . // and clears glist. If there is no current P, they are added to the 4054 . . // global queue, and up to npidle M's are started to run them.
runtime.schedule
/usr/lib/go/src/runtime/proc.go
Total: 100ms 4.94s (flat, cum) 6.91% 4145 . . wakep() 4146 . . } 4147 . . 4148 . . // One round of scheduler: find a runnable goroutine and execute it. 4149 . . // Never returns. 4150 50ms 50ms func schedule() { 4151 . . mp := getg().m 4152 . . 4153 . . if mp.locks != 0 { 4154 . . throw("schedule: holding locks") 4155 . . } 4156 . . 4157 10ms 10ms if mp.lockedg != 0 { 4158 . . stoplockedm() 4159 . . execute(mp.lockedg.ptr(), false) // Never returns. 4160 . . } 4161 . . 4162 . . // We should not schedule away from a g that is executing a cgo call, 4163 . . // since the cgo call is using the m's g0 stack. 4164 20ms 20ms if mp.incgo { 4165 . . throw("schedule: in cgo") 4166 . . } 4167 . . 4168 . . top: 4169 . . pp := mp.p.ptr() 4170 . . pp.preempt = false 4171 . . 4172 . . // Safety check: if we are spinning, the run queue should be empty. 4173 . . // Check this before calling checkTimers, as that might call 4174 . . // goready to put a ready goroutine on the local run queue. 4175 . . if mp.spinning && (pp.runnext != 0 || pp.runqhead != pp.runqtail) { 4176 . . throw("schedule: spinning with local work") 4177 . . } 4178 . . 4179 . 2.72s gp, inheritTime, tryWakeP := findRunnable() // blocks until work is available 4180 . . 4181 . . // May be on a new P. 4182 . . pp = mp.p.ptr() 4183 . . 4184 . . // findRunnable may have collected an allp snapshot. The snapshot is 4185 . . // only required within findRunnable. Clear it to all GC to collect the 4186 . . // slice. 4187 . 10ms mp.clearAllpSnapshot() 4188 . . 4189 . . // If the P was assigned a next GC mark worker but findRunnable 4190 . . // selected anything else, release the worker so another P may run it. 4191 . . // 4192 . . // N.B. If this occurs because a higher-priority goroutine was selected 4193 . . // (trace reader), then tryWakeP is set, which will wake another P to 4194 . . // run the worker. If this occurs because the GC is no longer active, 4195 . . // there is no need to wakep. 4196 . 10ms gcController.releaseNextGCMarkWorker(pp) 4197 . . 4198 10ms 10ms if debug.dontfreezetheworld > 0 && freezing.Load() { 4199 . . // See comment in freezetheworld. We don't want to perturb 4200 . . // scheduler state, so we didn't gcstopm in findRunnable, but 4201 . . // also don't want to allow new goroutines to run. 4202 . . // 4203 . . // Deadlock here rather than in the findRunnable loop so if 4204 . . // findRunnable is stuck in a loop we don't perturb that 4205 . . // either. 4206 . . lock(&deadlock) 4207 . . lock(&deadlock) 4208 . . } 4209 . . 4210 . . // This thread is going to run a goroutine and is not spinning anymore, 4211 . . // so if it was marked as spinning we need to reset it now and potentially 4212 . . // start a new spinning M. 4213 10ms 10ms if mp.spinning { 4214 . 2.10s resetspinning() 4215 . . } 4216 . . 4217 . . if sched.disable.user && !schedEnabled(gp) { 4218 . . // Scheduling of this goroutine is disabled. Put it on 4219 . . // the list of pending runnable goroutines for when we
runtime.schedule
/usr/lib/go/src/runtime/proc.go
Total: 90ms 400ms (flat, cum) 0.56% 4233 . . // If about to schedule a not-normal goroutine (a GCworker or tracereader), 4234 . . // wake a P if there is one. 4235 . . if tryWakeP { 4236 . . wakep() 4237 . . } 4238 90ms 90ms if gp.lockedm != 0 { 4239 . . // Hands off own p to the locked m, 4240 . . // then blocks waiting for a new p. 4241 . . startlockedm(gp) 4242 . . goto top 4243 . . } 4244 . . 4245 . 310ms execute(gp, inheritTime) 4246 . . } 4247 . . 4248 . . // dropg removes the association between m and the current goroutine m->curg (gp for short). 4249 . . // Typically a caller sets gp's status away from Grunning and then 4250 . . // immediately calls dropg to finish the job. The caller is also responsible
runtime.park_m
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 4263 . . unlock((*mutex)(lock)) 4264 . . return true 4265 . . } 4266 . . 4267 . . // park continuation on g0. 4268 20ms 20ms func park_m(gp *g) { 4269 . . mp := getg().m 4270 . . 4271 . . trace := traceAcquire() 4272 . . 4273 . . // If g is in a synctest group, we don't want to let the group
runtime.park_m
/usr/lib/go/src/runtime/proc.go
Total: 10ms 630ms (flat, cum) 0.88% 4285 . . // transition anymore. 4286 . . trace.GoPark(mp.waitTraceBlockReason, mp.waitTraceSkip) 4287 . . } 4288 . . // N.B. Not using casGToWaiting here because the waitreason is 4289 . . // set by park_m's caller. 4290 . 20ms casgstatus(gp, _Grunning, _Gwaiting) 4291 . . if trace.ok() { 4292 . . traceRelease(trace) 4293 . . } 4294 . . 4295 . . dropg() 4296 . . 4297 . . if fn := mp.waitunlockf; fn != nil { 4298 . 10ms ok := fn(gp, mp.waitlock) 4299 10ms 10ms mp.waitunlockf = nil 4300 . . mp.waitlock = nil 4301 . . if !ok { 4302 . . trace := traceAcquire() 4303 . . casgstatus(gp, _Gwaiting, _Grunnable) 4304 . . if bubble != nil { 4305 . . bubble.decActive() 4306 . . } 4307 . . if trace.ok() { 4308 . . trace.GoUnpark(gp, 2) 4309 . . traceRelease(trace) 4310 . . } 4311 . . execute(gp, true) // Schedule it back, never returns. 4312 . . } 4313 . . } 4314 . . 4315 . . if bubble != nil { 4316 . . bubble.decActive() 4317 . . } 4318 . . 4319 . 590ms schedule() 4320 . . } 4321 . . 4322 . . func goschedImpl(gp *g, preempted bool) { 4323 . . pp := gp.m.p.ptr() 4324 . . trace := traceAcquire()
runtime.goschedImpl
/usr/lib/go/src/runtime/proc.go
Total: 0 650ms (flat, cum) 0.91% 4335 . . trace.GoPreempt() 4336 . . } else { 4337 . . trace.GoSched() 4338 . . } 4339 . . } 4340 . 10ms casgstatus(gp, _Grunning, _Grunnable) 4341 . . if trace.ok() { 4342 . . traceRelease(trace) 4343 . . } 4344 . . 4345 . . dropg() 4346 . . if preempted && sched.gcwaiting.Load() { 4347 . . // If preempted for STW, keep the G on the local P in runnext 4348 . . // so it can keep running immediately after the STW. 4349 . . runqput(pp, gp, true) 4350 . . } else { 4351 . 10ms lock(&sched.lock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 4352 . . globrunqput(gp) 4353 . 40ms unlock(&sched.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 4354 . . } 4355 . . 4356 . . if mainStarted { 4357 . 330ms wakep() 4358 . . } 4359 . . 4360 . 260ms schedule() 4361 . . } 4362 . .
runtime.gosched_m
/usr/lib/go/src/runtime/proc.go
Total: 0 650ms (flat, cum) 0.91% 4363 . . // Gosched continuation on g0. 4364 . . func gosched_m(gp *g) { 4365 . 650ms goschedImpl(gp, false) 4366 . . } 4367 . . 4368 . . // goschedguarded is a forbidden-states-avoided version of gosched_m. 4369 . . func goschedguarded_m(gp *g) { 4370 . . if !canPreemptM(gp.m) {
runtime.goexit1
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 4492 . . if gp := getg(); gp.bubble != nil { 4493 . . racereleasemergeg(gp, gp.bubble.raceaddr()) 4494 . . } 4495 . . racegoend() 4496 . . } 4497 10ms 10ms trace := traceAcquire() if !traceEnabled() { traceruntime.go:189 4498 . . if trace.ok() { 4499 . . trace.GoEnd() 4500 . . traceRelease(trace) 4501 . . } 4502 10ms 10ms mcall(goexit0) 4503 . . } 4504 . . 4505 . . // goexit continuation on g0. 4506 . . func goexit0(gp *g) { 4507 . . if goexperiment.RuntimeSecret && gp.secret > 0 {
runtime.goexit0
/usr/lib/go/src/runtime/proc.go
Total: 0 4.91s (flat, cum) 6.86% 4509 . . // runtime.Goexit is called from within a runtime/secret.Do call. 4510 . . memclrNoHeapPointers(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 4511 . . // Since this is running on g0, our registers are already zeroed from going through 4512 . . // mcall in secret mode. 4513 . . } 4514 . 420ms gdestroy(gp) 4515 . 4.49s schedule() 4516 . . } 4517 . .
runtime.gdestroy
/usr/lib/go/src/runtime/proc.go
Total: 30ms 260ms (flat, cum) 0.36% 4518 . . func gdestroy(gp *g) { 4519 . . mp := getg().m 4520 10ms 10ms pp := mp.p.ptr() func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } runtime2.go:272 4521 . . 4522 . 90ms casgstatus(gp, _Grunning, _Gdead) 4523 20ms 20ms gcController.addScannableStack(pp, -int64(gp.stack.hi-gp.stack.lo)) c.maxStackScan.Add(pp.maxStackScanDelta) mgcpacer.go:981 return Xadd64(&u.value, delta) types.go:344 4524 . 140ms if isSystemGoroutine(gp, false) { 4525 . . sched.ngsys.Add(-1) 4526 . . } 4527 . . gp.m = nil 4528 . . locked := gp.lockedm != 0 4529 . . gp.lockedm = 0
runtime.gdestroy
/usr/lib/go/src/runtime/proc.go
Total: 0 160ms (flat, cum) 0.22% 4563 . . if mp.isextra { 4564 . . throw("runtime.Goexit called in a thread that was not created by the Go runtime") 4565 . . } 4566 . . throw("exited a goroutine internally locked to the OS thread") 4567 . . } 4568 . 160ms gfput(pp, gp) 4569 . . if locked { 4570 . . // The goroutine may have locked this thread because 4571 . . // it put it in an unusual kernel state. Kill it 4572 . . // rather than returning it to the thread pool. 4573 . .
runtime.save
/usr/lib/go/src/runtime/proc.go
Total: 30ms 30ms (flat, cum) 0.042% 4589 . . // save must not have write barriers because invoking a write barrier 4590 . . // can clobber getg().sched. 4591 . . // 4592 . . //go:nosplit 4593 . . //go:nowritebarrierrec 4594 10ms 10ms func save(pc, sp, bp uintptr) { 4595 . . gp := getg() 4596 . . 4597 20ms 20ms if gp == gp.m.g0 || gp == gp.m.gsignal { 4598 . . // m.g0.sched is special and must describe the context 4599 . . // for exiting the thread. mstart1 writes to it directly. 4600 . . // m.gsignal.sched should not be used at all. 4601 . . // This check makes sure save calls do not accidentally 4602 . . // run in contexts where they'd write to system g's.
runtime.reentersyscall
/usr/lib/go/src/runtime/proc.go
Total: 70ms 70ms (flat, cum) 0.098% 4637 . . // from a function further up in the call stack than the parent, as g->syscallsp 4638 . . // must always point to a valid stack frame. entersyscall below is the normal 4639 . . // entry point for syscalls, which obtains the SP and PC from the caller. 4640 . . // 4641 . . //go:nosplit 4642 30ms 30ms func reentersyscall(pc, sp, bp uintptr) { 4643 . . gp := getg() 4644 . . 4645 . . // Disable preemption because during this function g is in Gsyscall status, 4646 . . // but can have inconsistent g->sched, do not let GC observe it. 4647 40ms 40ms gp.m.locks++ 4648 . . 4649 . . // This M may have a signal stack that is dirtied with secret information 4650 . . // (see package "runtime/secret"). Since it's about to go into a syscall for 4651 . . // an arbitrary amount of time and the G that put the secret info there 4652 . . // might have returned from secret.Do, we have to zero it out now, lest we
runtime.reentersyscall
/usr/lib/go/src/runtime/proc.go
Total: 110ms 140ms (flat, cum) 0.2% 4676 . . // will trip any stack check and leaving a flag to tell newstack to die. 4677 . . gp.stackguard0 = stackPreempt 4678 . . gp.throwsplit = true 4679 . . 4680 . . // Copy the syscalltick over so we can identify if the P got stolen later. 4681 30ms 30ms gp.m.syscalltick = gp.m.p.ptr().syscalltick 4682 . . 4683 . . pp := gp.m.p.ptr() 4684 . . if pp.runSafePointFn != 0 { 4685 . . // runSafePointFn may stack split if run on this stack 4686 . . systemstack(runSafePointFn) 4687 . . } 4688 10ms 10ms gp.m.oldp.set(pp) 4689 . . 4690 . . // Leave SP around for GC and traceback. 4691 . 30ms save(pc, sp, bp) 4692 . . gp.syscallsp = sp 4693 . . gp.syscallpc = pc 4694 10ms 10ms gp.syscallbp = bp 4695 . . 4696 . . // Double-check sp and bp. 4697 . . if gp.syscallsp < gp.stack.lo || gp.stack.hi < gp.syscallsp { 4698 . . systemstack(func() { 4699 . . print("entersyscall inconsistent sp ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") 4700 . . throw("entersyscall") 4701 . . }) 4702 . . } 4703 . . if gp.syscallbp != 0 && gp.syscallbp < gp.stack.lo || gp.stack.hi < gp.syscallbp { 4704 . . systemstack(func() { 4705 . . print("entersyscall inconsistent bp ", hex(gp.syscallbp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") 4706 . . throw("entersyscall") 4707 . . }) 4708 . . } 4709 10ms 10ms trace := traceAcquire() 4710 . . if trace.ok() { 4711 . . // Emit a trace event. Notably, actually emitting the event must happen before 4712 . . // the casgstatus because it mutates the P, but the traceLocker must be held 4713 . . // across the casgstatus since we're transitioning out of _Grunning 4714 . . // (see trace.go invariants). 4715 . . systemstack(func() { 4716 . . trace.GoSysCall() 4717 . . }) 4718 . . // systemstack clobbered gp.sched, so restore it. 4719 . . save(pc, sp, bp) 4720 . . } 4721 10ms 10ms if sched.gcwaiting.Load() { return b.u.Load() != 0 types.go:168 4722 . . // Optimization: If there's a pending STW, do the equivalent of 4723 . . // entersyscallblock here at the last minute and immediately give 4724 . . // away our P. 4725 . . systemstack(func() { 4726 . . entersyscallHandleGCWait(trace) 4727 . . }) 4728 . . // systemstack clobbered gp.sched, so restore it. 4729 . . save(pc, sp, bp) 4730 . . } 4731 . . // As soon as we switch to _Gsyscall, we are in danger of losing our P. 4732 . . // We must not touch it after this point. 4733 . . // 4734 . . // Try to do a quick CAS to avoid calling into casgstatus in the common case. 4735 . . // If we have a bubble, we need to fall into casgstatus. 4736 10ms 10ms if gp.bubble != nil || !gp.atomicstatus.CompareAndSwap(_Grunning, _Gsyscall) { return Cas(&u.value, old, new) types.go:236 4737 . . casgstatus(gp, _Grunning, _Gsyscall) 4738 . . } 4739 . . if staticLockRanking { 4740 . . // casgstatus clobbers gp.sched via systemstack under staticLockRanking. Restore it. 4741 . . save(pc, sp, bp) 4742 . . } 4743 20ms 20ms if trace.ok() { 4744 . . // N.B. We don't need to go on the systemstack because traceRelease is very 4745 . . // carefully recursively nosplit. This also means we don't need to worry 4746 . . // about clobbering gp.sched. 4747 . . traceRelease(trace) 4748 . . } 4749 10ms 10ms if sched.sysmonwait.Load() { return b.u.Load() != 0 types.go:168 4750 . . systemstack(entersyscallWakeSysmon) 4751 . . // systemstack clobbered gp.sched, so restore it. 4752 . . save(pc, sp, bp) 4753 . . } 4754 . . gp.m.locks--
runtime.entersyscall
/usr/lib/go/src/runtime/proc.go
Total: 10ms 270ms (flat, cum) 0.38% 4771 . . // Do not remove or change the type signature. 4772 . . // See go.dev/issue/67401. 4773 . . // 4774 . . //go:nosplit 4775 . . //go:linkname entersyscall 4776 10ms 10ms func entersyscall() { 4777 . . // N.B. getcallerfp cannot be written directly as argument in the call 4778 . . // to reentersyscall because it forces spilling the other arguments to 4779 . . // the stack. This results in exceeding the nosplit stack requirements 4780 . . // on some platforms. 4781 . 50ms fp := getcallerfp() 4782 . 210ms reentersyscall(sys.GetCallerPC(), sys.GetCallerSP(), fp) 4783 . . } 4784 . . 4785 . . func entersyscallWakeSysmon() { 4786 . . lock(&sched.lock) 4787 . . if sched.sysmonwait.Load() {
runtime.exitsyscall
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 4917 . . // See go.dev/issue/67401. 4918 . . // 4919 . . //go:nosplit 4920 . . //go:nowritebarrierrec 4921 . . //go:linkname exitsyscall 4922 10ms 10ms func exitsyscall() { 4923 . . gp := getg() 4924 . . 4925 10ms 10ms gp.m.locks++ // see comment in entersyscall 4926 . . if sys.GetCallerSP() > gp.syscallsp { 4927 . . throw("exitsyscall: syscall frame is no longer valid") 4928 . . } 4929 . . gp.waitsince = 0 4930 . .
runtime.exitsyscall
/usr/lib/go/src/runtime/proc.go
Total: 90ms 90ms (flat, cum) 0.13% 4947 . . // the tracer because we're non-preemptible and in the runtime. It can't stop 4948 . . // us to read a bad status. 4949 . . // 4950 . . // Try to do a quick CAS to avoid calling into casgstatus in the common case. 4951 . . // If we have a bubble, we need to fall into casgstatus. 4952 70ms 70ms if gp.bubble != nil || !gp.atomicstatus.CompareAndSwap(_Gsyscall, _Grunning) { return Cas(&u.value, old, new) types.go:236 4953 . . casgstatus(gp, _Gsyscall, _Grunning) 4954 . . } 4955 . . 4956 . . // Caution: we're in a window where we may be in _Grunning without a P. 4957 . . // Either we will grab a P or call exitsyscall0, where we'll switch to 4958 . . // _Grunnable. 4959 . . if debugExtendGrunningNoP { 4960 . . usleep(10) 4961 . . } 4962 . . 4963 . . // Grab and clear our old P. 4964 20ms 20ms oldp := gp.m.oldp.ptr() 4965 . . gp.m.oldp.set(nil) 4966 . . 4967 . . // Check if we still have a P, and if not, try to acquire an idle P. 4968 . . pp := gp.m.p.ptr() 4969 . . if pp != nil {
runtime.exitsyscall
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 5013 . . pp = gp.m.p.ptr() 5014 . . } 5015 . . 5016 . . // If we have a P, clean up and exit. 5017 . . if pp != nil { 5018 10ms 10ms if goroutineProfile.active { 5019 . . // Make sure that gp has had its stack written out to the goroutine 5020 . . // profile, exactly as it was when the goroutine profiler first 5021 . . // stopped the world. 5022 . . systemstack(func() { 5023 . . tryRecordGoroutineProfileWB(gp)
runtime.newproc.func1
/usr/lib/go/src/runtime/proc.go
Total: 30ms 2.60s (flat, cum) 3.63% 5332 . . // Put it on the queue of g's waiting to run. 5333 . . // The compiler turns a go statement into a call to this. 5334 . . func newproc(fn *funcval) { 5335 . . gp := getg() 5336 . . pc := sys.GetCallerPC() 5337 20ms 1.32s systemstack(func() { 5338 10ms 970ms newg := newproc1(fn, gp, pc, false, waitReasonZero) 5339 . . 5340 . . pp := getg().m.p.ptr() 5341 . 60ms runqput(pp, newg, true) 5342 . . 5343 . . if mainStarted { 5344 . 250ms wakep() 5345 . . } 5346 . . }) 5347 . . } 5348 . .
runtime.newproc1
/usr/lib/go/src/runtime/proc.go
Total: 60ms 610ms (flat, cum) 0.85% 5349 . . // Create a new g in state _Grunnable (or _Gwaiting if parked is true), starting at fn. 5350 . . // callerpc is the address of the go statement that created this. The caller is responsible 5351 . . // for adding the new g to the scheduler. If parked is true, waitreason must be non-zero. 5352 . . func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreason waitReason) *g { 5353 50ms 50ms if fn == nil { 5354 . . fatal("go of nil func value") 5355 . . } 5356 . . 5357 . . mp := acquirem() // disable preemption because we hold M and P in local vars. 5358 . . pp := mp.p.ptr() 5359 . 550ms newg := gfget(pp) 5360 . . if newg == nil { 5361 . . newg = malg(stackMin) 5362 . . casgstatus(newg, _Gidle, _Gdead) 5363 . . allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. 5364 . . } 5365 . . if newg.stack.hi == 0 { 5366 . . throw("newproc1: newg missing stack") 5367 . . } 5368 . . 5369 10ms 10ms if readgstatus(newg) != _Gdead { 5370 . . throw("newproc1: new g is not Gdead") 5371 . . } 5372 . . 5373 . . totalSize := uintptr(4*goarch.PtrSize + sys.MinFrameSize) // extra space in case of reads slightly beyond frame 5374 . . totalSize = alignUp(totalSize, sys.StackAlign)
runtime.newproc1
/usr/lib/go/src/runtime/proc.go
Total: 10ms 210ms (flat, cum) 0.29% 5386 . . memclrNoHeapPointers(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched)) 5387 . . newg.sched.sp = sp 5388 . . newg.stktopsp = sp 5389 . . newg.sched.pc = abi.FuncPCABI0(goexit) + sys.PCQuantum // +PCQuantum so that previous instruction is in same function 5390 . . newg.sched.g = guintptr(unsafe.Pointer(newg)) 5391 . 40ms gostartcallfn(&newg.sched, fn) 5392 . . newg.parentGoid = callergp.goid 5393 . . newg.gopc = callerpc 5394 . 10ms newg.ancestors = saveAncestors(callergp) 5395 10ms 10ms newg.startpc = fn.fn 5396 . . newg.runningCleanups.Store(false) 5397 . 150ms if isSystemGoroutine(newg, false) { 5398 . . sched.ngsys.Add(1) 5399 . . } else { 5400 . . // Only user goroutines inherit synctest groups and pprof labels. 5401 . . newg.bubble = callergp.bubble 5402 . . if mp.curg != nil {
runtime.newproc1
/usr/lib/go/src/runtime/proc.go
Total: 60ms 140ms (flat, cum) 0.2% 5414 . . // Track initial transition? 5415 . . newg.trackingSeq = uint8(cheaprand()) 5416 . . if newg.trackingSeq%gTrackingPeriod == 0 { 5417 . . newg.tracking = true 5418 . . } 5419 40ms 40ms gcController.addScannableStack(pp, int64(newg.stack.hi-newg.stack.lo)) c.maxStackScan.Add(pp.maxStackScanDelta) mgcpacer.go:981 return Xadd64(&u.value, delta) types.go:344 5420 . . 5421 . . // Get a goid and switch to runnable. This needs to happen under traceAcquire 5422 . . // since it's a goroutine transition. See tracer invariants in trace.go. 5423 . . trace := traceAcquire() 5424 . . var status uint32 = _Grunnable 5425 . . if parked { 5426 . . status = _Gwaiting 5427 . . newg.waitreason = waitreason 5428 . . } 5429 . . if pp.goidcache == pp.goidcacheend { 5430 . . // Sched.goidgen is the last allocated id, 5431 . . // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. 5432 . . // At startup sched.goidgen=0, so main goroutine receives goid=1. 5433 10ms 10ms pp.goidcache = sched.goidgen.Add(_GoidCacheBatch) return Xadd64(&u.value, delta) types.go:344 5434 . . pp.goidcache -= _GoidCacheBatch - 1 5435 . . pp.goidcacheend = pp.goidcache + _GoidCacheBatch 5436 . . } 5437 . . newg.goid = pp.goidcache 5438 . 80ms casgstatus(newg, _Gdead, status) 5439 10ms 10ms pp.goidcache++ 5440 . . newg.trace.reset() 5441 . . if trace.ok() { 5442 . . trace.GoCreate(newg, newg.startpc, parked) 5443 . . traceRelease(trace) 5444 . . }
runtime.saveAncestors
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 5481 . . // includes info for the current caller into a new set of tracebacks for 5482 . . // a g being created. 5483 . . func saveAncestors(callergp *g) *[]ancestorInfo { 5484 . . // Copy all prior info, except for the root goroutine (goid 0). 5485 . . if debug.tracebackancestors <= 0 || callergp.goid == 0 { 5486 10ms 10ms return nil 5487 . . } 5488 . . var callerAncestors []ancestorInfo 5489 . . if callergp.ancestors != nil { 5490 . . callerAncestors = *callergp.ancestors 5491 . . }
runtime.gfput
/usr/lib/go/src/runtime/proc.go
Total: 70ms 160ms (flat, cum) 0.22% 5512 . . } 5513 . . 5514 . . // Put on gfree list. 5515 . . // If local list is too long, transfer a batch to the global list. 5516 . . func gfput(pp *p, gp *g) { 5517 10ms 10ms if readgstatus(gp) != _Gdead { 5518 . . throw("gfput: bad status (not Gdead)") 5519 . . } 5520 . . 5521 . . stksize := gp.stack.hi - gp.stack.lo 5522 . . 5523 . . if stksize != uintptr(startingStackSize) { 5524 . . // non-standard stack size - free it. 5525 . 90ms stackfree(gp.stack) 5526 . . gp.stack.lo = 0 5527 . . gp.stack.hi = 0 5528 . . gp.stackguard0 = 0 5529 . . if valgrindenabled { 5530 . . valgrindDeregisterStack(gp.valgrindStackID) 5531 . . gp.valgrindStackID = 0 5532 . . } 5533 . . } 5534 . . 5535 . . pp.gFree.push(gp) 5536 . . if pp.gFree.size >= 64 { 5537 . . var ( 5538 . . stackQ gQueue 5539 . . noStackQ gQueue 5540 . . ) 5541 . . for pp.gFree.size >= 32 { 5542 50ms 50ms gp := pp.gFree.pop() l.head = gp.schedlink proc.go:7904 5543 10ms 10ms if gp.stack.lo == 0 { 5544 . . noStackQ.push(gp) 5545 . . } else { 5546 . . stackQ.push(gp) 5547 . . } 5548 . . }
runtime.gfget
/usr/lib/go/src/runtime/proc.go
Total: 400ms 550ms (flat, cum) 0.77% 5553 . . } 5554 . . } 5555 . . 5556 . . // Get from gfree list. 5557 . . // If local list is empty, grab a batch from global list. 5558 20ms 20ms func gfget(pp *p) *g { 5559 . . retry: 5560 20ms 20ms if pp.gFree.empty() && (!sched.gFree.stack.empty() || !sched.gFree.noStack.empty()) { return l.head == 0 proc.go:7881 ⋮ 5561 . . lock(&sched.gFree.lock) 5562 . . // Move a batch of free Gs to the P. 5563 . . for pp.gFree.size < 32 { 5564 . . // Prefer Gs with stacks. 5565 . . gp := sched.gFree.stack.pop() 5566 . . if gp == nil { 5567 50ms 50ms gp = sched.gFree.noStack.pop() l.head = gp.schedlink proc.go:7904 5568 . . if gp == nil { 5569 . . break 5570 . . } 5571 . . } 5572 . . pp.gFree.push(gp) 5573 . . } 5574 . . unlock(&sched.gFree.lock) 5575 . . goto retry 5576 . . } 5577 260ms 260ms gp := pp.gFree.pop() l.head = gp.schedlink proc.go:7904 5578 . . if gp == nil { 5579 . . return nil 5580 . . } 5581 50ms 50ms if gp.stack.lo != 0 && gp.stack.hi-gp.stack.lo != uintptr(startingStackSize) { 5582 . . // Deallocate old stack. We kept it in gfput because it was the 5583 . . // right size when the goroutine was put on the free list, but 5584 . . // the right size has changed since then. 5585 . . systemstack(func() { 5586 . . stackfree(gp.stack) 5587 . . gp.stack.lo = 0 5588 . . gp.stack.hi = 0 5589 . . gp.stackguard0 = 0 5590 . . if valgrindenabled { 5591 . . valgrindDeregisterStack(gp.valgrindStackID) 5592 . . gp.valgrindStackID = 0 5593 . . } 5594 . . }) 5595 . . } 5596 . . if gp.stack.lo == 0 { 5597 . . // Stack was deallocated in gfput or just above. Allocate a new one. 5598 . 150ms systemstack(func() {
runtime.gfget.func2
/usr/lib/go/src/runtime/proc.go
Total: 0 150ms (flat, cum) 0.21% 5599 . 150ms gp.stack = stackalloc(startingStackSize) 5600 . . if valgrindenabled { 5601 . . gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(gp.stack.lo), unsafe.Pointer(gp.stack.hi)) 5602 . . } 5603 . . }) 5604 . . gp.stackguard0 = gp.stack.lo + stackGuard
runtime.acquirep
/usr/lib/go/src/runtime/proc.go
Total: 10ms 70ms (flat, cum) 0.098% 6305 . . // 6306 . . // This function is allowed to have write barriers even if the caller 6307 . . // isn't because it immediately acquires pp. 6308 . . // 6309 . . //go:yeswritebarrierrec 6310 10ms 10ms func acquirep(pp *p) { 6311 . . // Do the work. 6312 . 60ms acquirepNoTrace(pp) 6313 . . 6314 . . // Emit the event. 6315 . . trace := traceAcquire() 6316 . . if trace.ok() { 6317 . . trace.ProcStart()
runtime.acquirepNoTrace
/usr/lib/go/src/runtime/proc.go
Total: 10ms 60ms (flat, cum) 0.084% 6322 . . // Internals of acquirep, just skipping the trace events. 6323 . . // 6324 . . //go:yeswritebarrierrec 6325 . . func acquirepNoTrace(pp *p) { 6326 . . // Do the part that isn't allowed to have write barriers. 6327 . 40ms wirep(pp) 6328 . . 6329 . . // Have p; write barriers now allowed. 6330 . . 6331 . . // The M we're associating with will be the old M after the next 6332 . . // releasep. We must set this here because write barriers are not 6333 . . // allowed in releasep. 6334 10ms 10ms pp.oldm = pp.m.ptr().self 6335 . . 6336 . . // Perform deferred mcache flush before this P can allocate 6337 . . // from a potentially stale mcache. 6338 . 10ms pp.mcache.prepareForSweep() 6339 . . } 6340 . . 6341 . . // wirep is the first step of acquirep, which actually associates the 6342 . . // current M to pp. This is broken out so we can disallow write 6343 . . // barriers for this part, since we don't yet have a P.
runtime.wirep
/usr/lib/go/src/runtime/proc.go
Total: 40ms 40ms (flat, cum) 0.056% 6345 . . //go:nowritebarrierrec 6346 . . //go:nosplit 6347 . . func wirep(pp *p) { 6348 . . gp := getg() 6349 . . 6350 10ms 10ms if gp.m.p != 0 { 6351 . . // Call on the systemstack to avoid a nosplit overflow build failure 6352 . . // on some platforms when built with -N -l. See #64113. 6353 . . systemstack(func() { 6354 . . throw("wirep: already in go") 6355 . . }) 6356 . . } 6357 20ms 20ms if pp.m != 0 || pp.status != _Pidle { 6358 . . // Call on the systemstack to avoid a nosplit overflow build failure 6359 . . // on some platforms when built with -N -l. See #64113. 6360 . . systemstack(func() { 6361 . . id := int64(0) 6362 . . if pp.m != 0 { 6363 . . id = pp.m.ptr().id 6364 . . } 6365 . . print("wirep: p->m=", pp.m, "(", id, ") p->status=", pp.status, "\n") 6366 . . throw("wirep: invalid p state") 6367 . . }) 6368 . . } 6369 10ms 10ms gp.m.p.set(pp) 6370 . . pp.m.set(gp.m) 6371 . . pp.status = _Prunning
runtime.releasep
/usr/lib/go/src/runtime/proc.go
Total: 10ms 30ms (flat, cum) 0.042% 6373 . . 6374 . . // Disassociate p and the current m. 6375 10ms 10ms func releasep() *p { 6376 . . trace := traceAcquire() 6377 . . if trace.ok() { 6378 . . trace.ProcStop(getg().m.p.ptr()) 6379 . . traceRelease(trace) 6380 . . } 6381 . 20ms return releasepNoTrace() 6382 . . } 6383 . . 6384 . . // Disassociate p and the current m without tracing an event. 6385 . . func releasepNoTrace() *p { 6386 . . gp := getg()
runtime.releasepNoTrace
/usr/lib/go/src/runtime/proc.go
Total: 0 20ms (flat, cum) 0.028% 6393 . . print("releasep: m=", gp.m, " m->p=", gp.m.p.ptr(), " p->m=", hex(pp.m), " p->status=", pp.status, "\n") 6394 . . throw("releasep: invalid p state") 6395 . . } 6396 . . 6397 . . // P must clear if nextGCMarkWorker if it stops. 6398 . 20ms gcController.releaseNextGCMarkWorker(pp) 6399 . . 6400 . . gp.m.p = 0 6401 . . pp.m = 0 6402 . . pp.status = _Pidle 6403 . . return pp
runtime.checkdead
/usr/lib/go/src/runtime/proc.go
Total: 30ms 30ms (flat, cum) 0.042% 6438 . . // If we are not running under cgo, but we have an extra M then account 6439 . . // for it. (It is possible to have an extra M on Windows without cgo to 6440 . . // accommodate callbacks created by syscall.NewCallback. See issue #6751 6441 . . // for details.) 6442 . . var run0 int32 6443 20ms 20ms if !iscgo && cgoHasExtraM && extraMLength.Load() > 0 { 6444 . . run0 = 1 6445 . . } 6446 . . 6447 . . run := mcount() - sched.nmidle - sched.nmidlelocked - sched.nmsys 6448 . . if run > run0 { 6449 10ms 10ms return 6450 . . } 6451 . . if run < 0 { 6452 . . print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", mcount(), " nmsys=", sched.nmsys, "\n") 6453 . . unlock(&sched.lock) 6454 . . throw("checkdead: inconsistent counts")
runtime.mput
/usr/lib/go/src/runtime/proc.go
Total: 0 50ms (flat, cum) 0.07% 7249 . . // 7250 . . //go:nowritebarrierrec 7251 . . func mput(mp *m) { 7252 . . assertLockHeld(&sched.lock) 7253 . . 7254 . 20ms sched.midle.push(unsafe.Pointer(mp)) 7255 . . sched.nmidle++ 7256 . 30ms checkdead() 7257 . . } 7258 . . 7259 . . // Try to get an m from midle list. 7260 . . // sched.lock must be held. 7261 . . // May run during STW, so write barriers are not allowed.
runtime.mget
/usr/lib/go/src/runtime/proc.go
Total: 0 60ms (flat, cum) 0.084% 7262 . . // 7263 . . //go:nowritebarrierrec 7264 . . func mget() *m { 7265 . . assertLockHeld(&sched.lock) 7266 . . 7267 . 60ms mp := (*m)(sched.midle.pop()) 7268 . . if mp != nil { 7269 . . sched.nmidle-- 7270 . . } 7271 . . return mp 7272 . . }
runtime.pMask.read
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 7365 . . 7366 . . // read returns true if P id's bit is set. 7367 . . func (p pMask) read(id uint32) bool { 7368 . . word := id / 32 7369 . . mask := uint32(1) << (id % 32) 7370 20ms 20ms return (atomic.Load(&p[word]) & mask) != 0 7371 . . } 7372 . .
runtime.pMask.set
/usr/lib/go/src/runtime/proc.go
Total: 40ms 40ms (flat, cum) 0.056% 7373 . . // set sets P id's bit. 7374 . . func (p pMask) set(id int32) { 7375 40ms 40ms word := id / 32 7376 . . mask := uint32(1) << (id % 32) 7377 . . atomic.Or(&p[word], mask) 7378 . . } 7379 . . 7380 . . // clear clears P id's bit.
runtime.pidleput
/usr/lib/go/src/runtime/proc.go
Total: 40ms 40ms (flat, cum) 0.056% 7426 . . 7427 . . if !runqempty(pp) { 7428 . . throw("pidleput: P has non-empty run queue") 7429 . . } 7430 . . if now == 0 { 7431 20ms 20ms now = nanotime() return nanotime1() time_nofake.go:33 7432 . . } 7433 . . if pp.timers.len.Load() == 0 { 7434 . . timerpMask.clear(pp.id) 7435 . . } 7436 20ms 20ms idlepMask.set(pp.id) 7437 . . pp.link = sched.pidle 7438 . . sched.pidle.set(pp) 7439 . . sched.npidle.Add(1) 7440 . . if !pp.limiterEvent.start(limiterEventIdle, now) { 7441 . . throw("must be able to track idle limiter event")
runtime.pidleget
/usr/lib/go/src/runtime/proc.go
Total: 90ms 150ms (flat, cum) 0.21% 7455 . . 7456 . . pp := sched.pidle.ptr() 7457 . . if pp != nil { 7458 . . // Timer may get added at any time now. 7459 . . if now == 0 { 7460 30ms 30ms now = nanotime() return nanotime1() time_nofake.go:33 7461 . . } 7462 40ms 40ms timerpMask.set(pp.id) word := id / 32 proc.go:7375 7463 10ms 10ms idlepMask.clear(pp.id) 7464 . . sched.pidle = pp.link 7465 10ms 10ms sched.npidle.Add(-1) return Xaddint32(&i.value, delta) types.go:56 7466 . 60ms pp.limiterEvent.stop(limiterEventIdle, now) 7467 . . } 7468 . . return pp, now 7469 . . } 7470 . . 7471 . . // pidlegetSpinning tries to get a p from the _Pidle list, acquiring ownership.
runtime.pidlegetSpinning
/usr/lib/go/src/runtime/proc.go
Total: 10ms 160ms (flat, cum) 0.22% 7476 . . // sched.lock must be held. 7477 . . // 7478 . . // May run during STW, so write barriers are not allowed. 7479 . . // 7480 . . //go:nowritebarrierrec 7481 10ms 10ms func pidlegetSpinning(now int64) (*p, int64) { 7482 . . assertLockHeld(&sched.lock) 7483 . . 7484 . 150ms pp, now := pidleget(now) 7485 . . if pp == nil { 7486 . . // See "Delicate dance" comment in findRunnable. We found work 7487 . . // that we cannot take, we must synchronize with non-spinning 7488 . . // Ms that may be preparing to drop their P. 7489 . . sched.needspinning.Store(1)
runtime.runqput
/usr/lib/go/src/runtime/proc.go
Total: 90ms 90ms (flat, cum) 0.13% 7542 . . next = false 7543 . . } 7544 . . 7545 . . if next { 7546 . . retryNext: 7547 20ms 20ms oldnext := pp.runnext 7548 60ms 60ms if !pp.runnext.cas(oldnext, guintptr(unsafe.Pointer(gp))) { return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) runtime2.go:252 ⋮ 7549 . . goto retryNext 7550 . . } 7551 10ms 10ms if oldnext == 0 { 7552 . . return 7553 . . } 7554 . . // Kick the old runnext out to the regular run queue. 7555 . . gp = oldnext.ptr() 7556 . . }
runtime.runqget
/usr/lib/go/src/runtime/proc.go
Total: 140ms 140ms (flat, cum) 0.2% 7646 . . // If inheritTime is true, gp should inherit the remaining time in the 7647 . . // current time slice. Otherwise, it should start a new time slice. 7648 . . // Executed only by the owner P. 7649 . . func runqget(pp *p) (gp *g, inheritTime bool) { 7650 . . // If there's a runnext, it's the next G to run. 7651 10ms 10ms next := pp.runnext 7652 . . // If the runnext is non-0 and the CAS fails, it could only have been stolen by another P, 7653 . . // because other Ps can race to set runnext to 0, but only the current P can set it to non-0. 7654 . . // Hence, there's no need to retry this CAS if it fails. 7655 20ms 20ms if next != 0 && pp.runnext.cas(next, 0) { return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) runtime2.go:252 7656 . . return next.ptr(), true 7657 . . } 7658 . . 7659 . . for { 7660 10ms 10ms h := atomic.LoadAcq(&pp.runqhead) // load-acquire, synchronize with other consumers 7661 60ms 60ms t := pp.runqtail 7662 . . if t == h { 7663 . . return nil, false 7664 . . } 7665 10ms 10ms gp := pp.runq[h%uint32(len(pp.runq))].ptr() 7666 30ms 30ms if atomic.CasRel(&pp.runqhead, h, h+1) { // cas-release, commits consume 7667 . . return gp, false 7668 . . } 7669 . . } 7670 . . } 7671 . .
runtime.runqgrab
/usr/lib/go/src/runtime/proc.go
Total: 140ms 140ms (flat, cum) 0.2% 7708 . . 7709 . . // Grabs a batch of goroutines from pp's runnable queue into batch. 7710 . . // Batch is a ring buffer starting at batchHead. 7711 . . // Returns number of grabbed goroutines. 7712 . . // Can be executed by any P. 7713 10ms 10ms func runqgrab(pp *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 { 7714 . . for { 7715 100ms 100ms h := atomic.LoadAcq(&pp.runqhead) // load-acquire, synchronize with other consumers 7716 20ms 20ms t := atomic.LoadAcq(&pp.runqtail) // load-acquire, synchronize with the producer 7717 . . n := t - h 7718 . . n = n - n/2 7719 . . if n == 0 { 7720 . . if stealRunNextG { 7721 . . // Try to steal from pp.runnext. 7722 10ms 10ms if next := pp.runnext; next != 0 { 7723 . . if pp.status == _Prunning { 7724 . . if mp := pp.m.ptr(); mp != nil { 7725 . . if gp := mp.curg; gp == nil || readgstatus(gp)&^_Gscan != _Gsyscall { 7726 . . // Sleep to ensure that pp isn't about to run the g 7727 . . // we are about to steal.
runtime.runqgrab
/usr/lib/go/src/runtime/proc.go
Total: 80ms 210ms (flat, cum) 0.29% 7741 . . // or already is (since we found a waiting or runnable 7742 . . // goroutine hanging off of a running P, suggesting it 7743 . . // either recently transitioned out of running, or will 7744 . . // transition to running shortly). 7745 . . if !osHasLowResTimer { 7746 . 130ms usleep(3) 7747 . . } else { 7748 . . // On some platforms system timer granularity is 7749 . . // 1-15ms, which is way too much for this 7750 . . // optimization. So just yield. 7751 . . osyield() 7752 . . } 7753 . . } 7754 . . } 7755 . . } 7756 10ms 10ms if !pp.runnext.cas(next, 0) { return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) runtime2.go:252 7757 . . continue 7758 . . } 7759 . . batch[batchHead%uint32(len(batch))] = next 7760 . . return 1 7761 . . } 7762 . . } 7763 10ms 10ms return 0 7764 . . } 7765 . . if n > uint32(len(pp.runq)/2) { // read inconsistent h and t 7766 . . continue 7767 . . } 7768 . . for i := uint32(0); i < n; i++ { 7769 . . g := pp.runq[(h+i)%uint32(len(pp.runq))] 7770 30ms 30ms batch[(batchHead+i)%uint32(len(batch))] = g 7771 . . } 7772 30ms 30ms if atomic.CasRel(&pp.runqhead, h, h+n) { // cas-release, commits consume 7773 . . return n 7774 . . } 7775 . . } 7776 . . }
runtime.runqsteal
/usr/lib/go/src/runtime/proc.go
Total: 20ms 370ms (flat, cum) 0.52% 7777 . . 7778 . . // Steal half of elements from local runnable queue of p2 7779 . . // and put onto local runnable queue of p. 7780 . . // Returns one of the stolen elements (or nil if failed). 7781 20ms 20ms func runqsteal(pp, p2 *p, stealRunNextG bool) *g { 7782 . . t := pp.runqtail 7783 . 350ms n := runqgrab(p2, &pp.runq, t, stealRunNextG) 7784 . . if n == 0 { 7785 . . return nil 7786 . . } 7787 . . n-- 7788 . . gp := pp.runq[(t+n)%uint32(len(pp.runq))].ptr()
runtime.(*gList).empty
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 7876 . . size int32 7877 . . } 7878 . . 7879 . . // empty reports whether l is empty. 7880 . . func (l *gList) empty() bool { 7881 10ms 10ms return l.head == 0 7882 . . } 7883 . . 7884 . . // push adds gp to the head of l. 7885 . . func (l *gList) push(gp *g) { 7886 . . gp.schedlink = l.head
runtime.(*gList).pop
/usr/lib/go/src/runtime/proc.go
Total: 360ms 360ms (flat, cum) 0.5% 7899 . . 7900 . . // pop removes and returns the head of l. If l is empty, it returns nil. 7901 . . func (l *gList) pop() *g { 7902 . . gp := l.head.ptr() 7903 . . if gp != nil { 7904 360ms 360ms l.head = gp.schedlink 7905 . . l.size-- 7906 . . } 7907 . . return gp 7908 . . } 7909 . .
runtime.(*randomEnum).done
/usr/lib/go/src/runtime/proc.go
Total: 10ms 10ms (flat, cum) 0.014% 8076 . . inc: ord.coprimes[i/ord.count%uint32(len(ord.coprimes))], 8077 . . } 8078 . . } 8079 . . 8080 . . func (enum *randomEnum) done() bool { 8081 10ms 10ms return enum.i == enum.count 8082 . . } 8083 . .
runtime.(*randomEnum).next
/usr/lib/go/src/runtime/proc.go
Total: 30ms 30ms (flat, cum) 0.042% 8084 . . func (enum *randomEnum) next() { 8085 . . enum.i++ 8086 30ms 30ms enum.pos = (enum.pos + enum.inc) % enum.count 8087 . . }
runtime.(*randomEnum).position
/usr/lib/go/src/runtime/proc.go
Total: 20ms 20ms (flat, cum) 0.028% 8089 . . func (enum *randomEnum) position() uint32 { 8090 20ms 20ms return enum.pos 8091 . . } 8092 . . 8093 . . func gcd(a, b uint32) uint32 { 8094 . . for b != 0 { 8095 . . a, b = b, a%b
runtime.nextFreeFast
/usr/lib/go/src/runtime/malloc.go
Total: 800ms 800ms (flat, cum) 1.12% 965 . . var zerobase uintptr 966 . . 967 . . // nextFreeFast returns the next free object if one is quickly available. 968 . . // Otherwise it returns 0. 969 . . func nextFreeFast(s *mspan) gclinkptr { 970 630ms 630ms theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? 971 . . if theBit < 64 { 972 40ms 40ms result := s.freeindex + uint16(theBit) 973 . . if result < s.nelems { 974 20ms 20ms freeidx := result + 1 975 . . if freeidx%64 == 0 && freeidx != s.nelems { 976 . . return 0 977 . . } 978 10ms 10ms s.allocCache >>= uint(theBit + 1) 979 . . s.freeindex = freeidx 980 50ms 50ms s.allocCount++ 981 50ms 50ms return gclinkptr(uintptr(result)*s.elemsize + s.base()) 982 . . } 983 . . } 984 . . return 0 985 . . } 986 . .
runtime.(*mcache).nextFree
/usr/lib/go/src/runtime/malloc.go
Total: 70ms 3.65s (flat, cum) 5.10% 991 . . // determine whether a new GC cycle needs to be started or if the GC is active 992 . . // whether this goroutine needs to assist the GC. 993 . . // 994 . . // Must run in a non-preemptible context since otherwise the owner of 995 . . // c could change. 996 40ms 120ms func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger bool) { 997 10ms 10ms s = c.alloc[spc] 998 . . checkGCTrigger = false 999 . 390ms freeIndex := s.nextFreeIndex() 1000 20ms 20ms if freeIndex == s.nelems { 1001 . . // The span is full. 1002 . . if s.allocCount != s.nelems { 1003 . . println("runtime: s.allocCount=", s.allocCount, "s.nelems=", s.nelems) 1004 . . throw("s.allocCount != s.nelems && freeIndex == s.nelems") 1005 . . } 1006 . 3.11s c.refill(spc) 1007 . . checkGCTrigger = true 1008 . . s = c.alloc[spc] 1009 . . 1010 . . freeIndex = s.nextFreeIndex() 1011 . . }
runtime.mallocgc
/usr/lib/go/src/runtime/malloc.go
Total: 530ms 12.11s (flat, cum) 16.93% 1062 . . // 1063 . . // Do not remove or change the type signature. 1064 . . // See go.dev/issue/67401. 1065 . . // 1066 . . //go:linkname mallocgc 1067 140ms 4.16s func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 1068 . . if doubleCheckMalloc { 1069 . . if gcphase == _GCmarktermination { 1070 . . throw("mallocgc called with gcphase == _GCmarktermination") 1071 . . } 1072 . . } 1073 . . 1074 . . // Short-circuit zero-sized allocation requests. 1075 50ms 50ms if size == 0 { 1076 10ms 10ms return unsafe.Pointer(&zerobase) 1077 . . } 1078 . . 1079 . . if sizeSpecializedMallocEnabled && size < uintptr(len(mallocNoScanTable)) { 1080 20ms 20ms if typ == nil || !typ.Pointers() { 1081 . . if size >= maxTinySize { 1082 60ms 400ms return mallocNoScanTable[size](size, typ, needzero) 1083 . . } 1084 . 310ms return mallocgcTinySC2(size, typ, needzero) 1085 . . } else { 1086 . . if !needzero { 1087 . . throw("objects with pointers must be zeroed") 1088 . . } 1089 170ms 2.31s return mallocScanTable[size](size, typ, needzero) 1090 . . } 1091 . . } 1092 . . 1093 . . // It's possible for any malloc to trigger sweeping, which may in 1094 . . // turn queue finalizers. Record this dynamic lock edge. 1095 . . // N.B. Compiled away if lockrank experiment is not enabled. 1096 . . lockRankMayQueueFinalizer() 1097 . . 1098 . . // Pre-malloc debug hooks. 1099 30ms 30ms if debug.malloc { 1100 . . if x := preMallocgcDebug(size, typ); x != nil { 1101 . . return x 1102 . . } 1103 . . } 1104 . . 1105 . . // For ASAN, we allocate extra memory around each allocation called the "redzone." 1106 . . // These "redzones" are marked as unaddressable. 1107 . . var asanRZ uintptr 1108 . . if asanenabled { 1109 . . asanRZ = redZoneSize(size) 1110 . . size += asanRZ 1111 . . } 1112 . . 1113 . . // Assist the GC if needed. (On the reuse path, we currently compensate for this; 1114 . . // changes here might require changes there.) 1115 20ms 20ms if gcBlackenEnabled != 0 { 1116 . . deductAssistCredit(size) 1117 . . } 1118 . . 1119 . . // Actually do the allocation. 1120 . . var x unsafe.Pointer 1121 . . var elemsize uintptr 1122 . . if sizeSpecializedMallocEnabled { 1123 . . if size <= maxSmallSize-gc.MallocHeaderSize { 1124 10ms 10ms if typ == nil || !typ.Pointers() { 1125 . 690ms x, elemsize = mallocgcSmallNoscan(size, typ, needzero) 1126 . . } else { 1127 . . if !needzero { 1128 . . throw("objects with pointers must be zeroed") 1129 . . } 1130 10ms 10ms if heapBitsInSpan(size) { 1131 10ms 3.18s x, elemsize = mallocgcSmallScanNoHeader(size, typ) 1132 . . } else { 1133 . 900ms x, elemsize = mallocgcSmallScanHeader(size, typ) 1134 . . } 1135 . . } 1136 . . } else { 1137 . 10ms x, elemsize = mallocgcLarge(size, typ, needzero) 1138 . . } 1139 . . } else { 1140 . . if size <= maxSmallSize-gc.MallocHeaderSize { 1141 . . if typ == nil || !typ.Pointers() { 1142 . . // tiny allocations might be kept alive by other co-located values.
runtime.mallocgc
/usr/lib/go/src/runtime/malloc.go
Total: 30ms 30ms (flat, cum) 0.042% 1190 . . if valgrindenabled { 1191 . . valgrindMalloc(x, size-asanRZ) 1192 . . } 1193 . . 1194 . . // Adjust our GC assist debt to account for internal fragmentation. 1195 10ms 10ms if gcBlackenEnabled != 0 && elemsize != 0 { 1196 . . if assistG := getg().m.curg; assistG != nil { 1197 . . assistG.gcAssistBytes -= int64(elemsize - size) 1198 . . } 1199 . . } 1200 . . 1201 . . // Post-malloc debug hooks. 1202 10ms 10ms if debug.malloc { 1203 . . postMallocgcDebug(x, elemsize, typ) 1204 . . } 1205 10ms 10ms return x 1206 . . } 1207 . . 1208 . . func mallocgcTiny(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { 1209 . . // Set mp.mallocing to keep from being preempted by GC. 1210 . . mp := acquirem()
runtime.mallocgcSmallNoscan
/usr/lib/go/src/runtime/malloc.go
Total: 30ms 30ms (flat, cum) 0.042% 1359 . . x = add(x, span.elemsize-size) 1360 . . } 1361 . . return x, span.elemsize 1362 . . } 1363 . . 1364 30ms 30ms func mallocgcSmallNoscan(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { 1365 . . // Set mp.mallocing to keep from being preempted by GC. 1366 . . mp := acquirem() 1367 . . if doubleCheckMalloc { 1368 . . if mp.mallocing != 0 { 1369 . . throw("malloc deadlock")
runtime.mallocgcSmallNoscan
/usr/lib/go/src/runtime/malloc.go
Total: 240ms 630ms (flat, cum) 0.88% 1396 . . mp.mallocing = 0 1397 . . releasem(mp) 1398 . . return x, size 1399 . . } 1400 . . 1401 240ms 240ms v := nextFreeFast(span) theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 ⋮ return gclinkptr(uintptr(result)*s.elemsize + s.base()) malloc.go:981 ⋮ theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 1402 . . if v == 0 { 1403 . 190ms v, span, checkGCTrigger = c.nextFree(spc) 1404 . . } 1405 . . x := unsafe.Pointer(v) 1406 . . if needzero && span.needzero != 0 { 1407 . 200ms memclrNoHeapPointers(x, size) 1408 . . } 1409 . . 1410 . . // Ensure that the stores above that initialize x to 1411 . . // type-safe memory and set the heap bits occur before 1412 . . // the caller can make x observable to the garbage
runtime.mallocgcSmallNoscan
/usr/lib/go/src/runtime/malloc.go
Total: 10ms 30ms (flat, cum) 0.042% 1428 . . // that are both free and recently-allocated. It's safe to do that 1429 . . // because we allocate-black if the GC is enabled. The conservative 1430 . . // scanner produces pointers out of thin air, so without additional 1431 . . // synchronization it might otherwise observe a partially-initialized 1432 . . // object, which could crash the program. 1433 10ms 10ms span.freeIndexForScan = span.freeindex 1434 . . } 1435 . . 1436 . . // Note cache c only valid while m acquired; see #47302 1437 . . // 1438 . . // N.B. Use the full size because that matches how the GC 1439 . . // will update the mem profile on the "free" side. 1440 . . // 1441 . . // TODO(mknyszek): We should really count the header as part 1442 . . // of gc_sys or something. The code below just pretends it is 1443 . . // internal fragmentation and matches the GC's accounting by 1444 . . // using the whole allocation slot. 1445 . . c.nextSample -= int64(size) 1446 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1447 . 10ms profilealloc(mp, x, size) 1448 . . } 1449 . . mp.mallocing = 0 1450 . . releasem(mp) 1451 . . 1452 . . if checkGCTrigger { 1453 . 10ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1454 . . gcStart(t) 1455 . . } 1456 . . } 1457 . . return x, size 1458 . . }
runtime.mallocgcSmallScanNoHeader
/usr/lib/go/src/runtime/malloc.go
Total: 1s 3.17s (flat, cum) 4.43% 1504 . . gcmarknewobject(span, uintptr(x)) 1505 . . } 1506 . . return x 1507 . . } 1508 . . 1509 40ms 40ms func mallocgcSmallScanNoHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { 1510 . . // Set mp.mallocing to keep from being preempted by GC. 1511 30ms 30ms mp := acquirem() gp.m.locks++ runtime1.go:614 1512 . . if doubleCheckMalloc { 1513 . . if mp.mallocing != 0 { 1514 . . throw("malloc deadlock") 1515 . . } 1516 . . if mp.gsignal == getg() { 1517 . . throw("malloc during signal") 1518 . . } 1519 . . if typ == nil || !typ.Pointers() { 1520 . . throw("noscan allocated in scan-only path") 1521 . . } 1522 . . if !heapBitsInSpan(size) { 1523 . . throw("heap bits in not in span for non-header-only path") 1524 . . } 1525 . . } 1526 . . mp.mallocing = 1 1527 . . 1528 . . checkGCTrigger := false 1529 70ms 70ms c := getMCache(mp) ⋮ if pp == nil { mcache.go:143 ⋮ pp := mp.p.ptr() mcache.go:141 ⋮ c = pp.mcache mcache.go:150 ⋮ if pp == nil { mcache.go:143 1530 30ms 30ms sizeclass := gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)] 1531 40ms 40ms spc := makeSpanClass(sizeclass, false) ⋮ return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) mheap.go:581 ⋮ return int(*(*uint8)(unsafe.Pointer(&x))) stubs.go:409 1532 40ms 40ms span := c.alloc[spc] 1533 420ms 420ms v := nextFreeFast(span) return gclinkptr(uintptr(result)*s.elemsize + s.base()) malloc.go:981 ⋮ s.allocCount++ malloc.go:980 ⋮ result := s.freeindex + uint16(theBit) malloc.go:972 ⋮ theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 ⋮ result := s.freeindex + uint16(theBit) malloc.go:972 ⋮ return gclinkptr(uintptr(result)*s.elemsize + s.base()) malloc.go:981 ⋮ theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 ⋮ s.allocCache >>= uint(theBit + 1) malloc.go:978 ⋮ freeidx := result + 1 malloc.go:974 1534 . . if v == 0 { 1535 . 1.26s v, span, checkGCTrigger = c.nextFree(spc) 1536 . . } 1537 40ms 40ms x := unsafe.Pointer(v) 1538 10ms 10ms if span.needzero != 0 { 1539 . 240ms memclrNoHeapPointers(x, size) 1540 . . } 1541 10ms 10ms if goarch.PtrSize == 8 && sizeclass == 1 { 1542 . . // initHeapBits already set the pointer bits for the 8-byte sizeclass 1543 . . // on 64-bit platforms. 1544 10ms 10ms c.scanAlloc += 8 1545 . . } else { 1546 90ms 590ms c.scanAlloc += heapSetTypeNoHeader(uintptr(x), size, typ, span) ⋮ scanSize := span.writeHeapBitsSmall(x, dataSize, typ) mbitmap.go:711 ⋮ ⋮ 1547 . . } 1548 20ms 20ms size = uintptr(gc.SizeClassToSize[sizeclass]) 1549 . . 1550 . . // Ensure that the stores above that initialize x to 1551 . . // type-safe memory and set the heap bits occur before 1552 . . // the caller can make x observable to the garbage 1553 . . // collector. Otherwise, on weakly ordered machines, 1554 . . // the garbage collector could follow a pointer to x, 1555 . . // but see uninitialized memory or stale heap bits. 1556 10ms 10ms publicationBarrier() 1557 . . 1558 . . if writeBarrier.enabled { 1559 . . // Allocate black during GC. 1560 . . // All slots hold nil so no scanning is needed. 1561 . . // This may be racing with GC so do it atomically if there can be 1562 . . // a race marking the bit. 1563 . . gcmarknewobject(span, uintptr(x)) 1564 . . } else { 1565 . . // Track the last free index before the mark phase. This field 1566 . . // is only used by the garbage collector. During the mark phase 1567 . . // this is used by the conservative scanner to filter out objects 1568 . . // that are both free and recently-allocated. It's safe to do that 1569 . . // because we allocate-black if the GC is enabled. The conservative 1570 . . // scanner produces pointers out of thin air, so without additional 1571 . . // synchronization it might otherwise observe a partially-initialized 1572 . . // object, which could crash the program. 1573 20ms 20ms span.freeIndexForScan = span.freeindex 1574 . . } 1575 . . 1576 . . // Note cache c only valid while m acquired; see #47302 1577 . . // 1578 . . // N.B. Use the full size because that matches how the GC 1579 . . // will update the mem profile on the "free" side. 1580 . . // 1581 . . // TODO(mknyszek): We should really count the header as part 1582 . . // of gc_sys or something. The code below just pretends it is 1583 . . // internal fragmentation and matches the GC's accounting by 1584 . . // using the whole allocation slot. 1585 . . c.nextSample -= int64(size) 1586 20ms 20ms if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1587 30ms 150ms profilealloc(mp, x, size) 1588 . . } 1589 20ms 20ms mp.mallocing = 0 1590 10ms 10ms releasem(mp) 1591 . . 1592 40ms 40ms if checkGCTrigger { 1593 . 50ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1594 . . gcStart(t) 1595 . . } 1596 . . }
runtime.mallocgcSmallScanHeader
/usr/lib/go/src/runtime/malloc.go
Total: 20ms 20ms (flat, cum) 0.028% 1597 . . return x, size 1598 . . } 1599 . . 1600 20ms 20ms func mallocgcSmallScanHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { 1601 . . // Set mp.mallocing to keep from being preempted by GC. 1602 . . mp := acquirem() 1603 . . if doubleCheckMalloc { 1604 . . if mp.mallocing != 0 { 1605 . . throw("malloc deadlock")
runtime.mallocgcSmallScanHeader
/usr/lib/go/src/runtime/malloc.go
Total: 260ms 850ms (flat, cum) 1.19% 1624 . . sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)] 1625 . . } else { 1626 . . sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)] 1627 . . } 1628 . . size = uintptr(gc.SizeClassToSize[sizeclass]) 1629 10ms 10ms spc := makeSpanClass(sizeclass, false) return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) mheap.go:581 1630 . . span := c.alloc[spc] 1631 140ms 140ms v := nextFreeFast(span) theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 ⋮ freeidx := result + 1 malloc.go:974 ⋮ return gclinkptr(uintptr(result)*s.elemsize + s.base()) malloc.go:981 ⋮ theBit := sys.TrailingZeros64(s.allocCache) // Is there a free object in the allocCache? malloc.go:970 ⋮ s.allocCount++ malloc.go:980 1632 . . if v == 0 { 1633 . 440ms v, span, checkGCTrigger = c.nextFree(spc) 1634 . . } 1635 . . x := unsafe.Pointer(v) 1636 . . if span.needzero != 0 { 1637 . 150ms memclrNoHeapPointers(x, size) 1638 . . } 1639 . . header := (**_type)(x) 1640 10ms 10ms x = add(x, gc.MallocHeaderSize) return unsafe.Pointer(uintptr(p) + x) stubs.go:25 1641 90ms 90ms c.scanAlloc += heapSetTypeSmallHeader(uintptr(x), size-gc.MallocHeaderSize, typ, header, span) *header = typ mbitmap.go:739 ⋮ 1642 . . 1643 . . // Ensure that the stores above that initialize x to 1644 . . // type-safe memory and set the heap bits occur before 1645 . . // the caller can make x observable to the garbage 1646 . . // collector. Otherwise, on weakly ordered machines, 1647 . . // the garbage collector could follow a pointer to x, 1648 . . // but see uninitialized memory or stale heap bits. 1649 10ms 10ms publicationBarrier() 1650 . . 1651 . . if writeBarrier.enabled { 1652 . . // Allocate black during GC. 1653 . . // All slots hold nil so no scanning is needed. 1654 . . // This may be racing with GC so do it atomically if there can be
runtime.mallocgcSmallScanHeader
/usr/lib/go/src/runtime/malloc.go
Total: 10ms 30ms (flat, cum) 0.042% 1675 . . // of gc_sys or something. The code below just pretends it is 1676 . . // internal fragmentation and matches the GC's accounting by 1677 . . // using the whole allocation slot. 1678 . . c.nextSample -= int64(size) 1679 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1680 . 10ms profilealloc(mp, x, size) 1681 . . } 1682 . . mp.mallocing = 0 1683 10ms 10ms releasem(mp) if mp.locks == 0 && gp.preempt { runtime1.go:622 1684 . . 1685 . . if checkGCTrigger { 1686 . 10ms if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 1687 . . gcStart(t) 1688 . . } 1689 . . } 1690 . . return x, size 1691 . . }
runtime.mallocgcLarge
/usr/lib/go/src/runtime/malloc.go
Total: 0 10ms (flat, cum) 0.014% 1749 . . // of gc_sys or something. The code below just pretends it is 1750 . . // internal fragmentation and matches the GC's accounting by 1751 . . // using the whole allocation slot. 1752 . . c.nextSample -= int64(size) 1753 . . if c.nextSample < 0 || MemProfileRate != c.memProfRate { 1754 . 10ms profilealloc(mp, x, size) 1755 . . } 1756 . . mp.mallocing = 0 1757 . . releasem(mp) 1758 . . 1759 . . // Check to see if we need to trigger the GC.
runtime.newobject
/usr/lib/go/src/runtime/malloc.go
Total: 120ms 1.69s (flat, cum) 2.36% 2135 . . } 2136 . . 2137 . . // implementation of new builtin 2138 . . // compiler (both frontend and SSA backend) knows the signature 2139 . . // of this function. 2140 10ms 10ms func newobject(typ *_type) unsafe.Pointer { 2141 110ms 1.68s return mallocgc(typ.Size_, typ, true) 2142 . . } 2143 . . 2144 . . //go:linkname maps_newobject internal/runtime/maps.newobject 2145 . . func maps_newobject(typ *_type) unsafe.Pointer { 2146 . . return newobject(typ)
runtime.newarray
/usr/lib/go/src/runtime/malloc.go
Total: 60ms 1.68s (flat, cum) 2.35% 2180 . . // See go.dev/issue/67401. 2181 . . // 2182 . . //go:linkname newarray 2183 . . func newarray(typ *_type, n int) unsafe.Pointer { 2184 . . if n == 1 { 2185 40ms 1.26s return mallocgc(typ.Size_, typ, true) 2186 . . } 2187 10ms 10ms mem, overflow := math.MulUintptr(typ.Size_, uintptr(n)) 2188 10ms 10ms if overflow || mem > maxAlloc || n < 0 { 2189 . . panic(plainError("runtime: allocation size out of range")) 2190 . . } 2191 . 400ms return mallocgc(mem, typ, true) 2192 . . } 2193 . . 2194 . . // reflect_unsafe_NewArray is meant for package reflect, 2195 . . // but widely used packages access it using linkname. 2196 . . // Notable members of the hall of shame include:
internal/runtime/maps.newarray
/usr/lib/go/src/runtime/malloc.go
Total: 10ms 1.69s (flat, cum) 2.36% 2209 . . func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { 2210 . . return newarray(typ, n) 2211 . . } 2212 . . 2213 . . //go:linkname maps_newarray internal/runtime/maps.newarray 2214 10ms 10ms func maps_newarray(typ *_type, n int) unsafe.Pointer { 2215 . 1.68s return newarray(typ, n) 2216 . . } 2217 . . 2218 . . // profilealloc resets the current mcache's nextSample counter and 2219 . . // records a memory profile sample. 2220 . . //
runtime.profilealloc
/usr/lib/go/src/runtime/malloc.go
Total: 0 220ms (flat, cum) 0.31% 2224 . . if c == nil { 2225 . . throw("profilealloc called without a P or outside bootstrapping") 2226 . . } 2227 . . c.memProfRate = MemProfileRate 2228 . . c.nextSample = nextSample() 2229 . 220ms mProf_Malloc(mp, x, size) 2230 . . } 2231 . . 2232 . . // nextSample returns the next sampling point for heap profiling. The goal is 2233 . . // to sample allocations on average every MemProfileRate bytes, but with a 2234 . . // completely random distribution over the allocation timeline; this
runtime.persistentalloc
/usr/lib/go/src/runtime/malloc.go
Total: 0 10ms (flat, cum) 0.014% 2310 . . // nosplit because it is used during write barriers and must not be preempted. 2311 . . // 2312 . . //go:nosplit 2313 . . func persistentalloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { 2314 . . var p *notInHeap 2315 . 10ms systemstack(func() {
runtime.persistentalloc.func1
/usr/lib/go/src/runtime/malloc.go
Total: 0 10ms (flat, cum) 0.014% 2316 . 10ms p = persistentalloc1(size, align, sysStat) 2317 . . }) 2318 . . return unsafe.Pointer(p) 2319 . . } 2320 . . 2321 . . // Must run on system stack because stack growth can (re)invoke it.
runtime.persistentalloc1
/usr/lib/go/src/runtime/malloc.go
Total: 0 10ms (flat, cum) 0.014% 2353 . . lock(&globalAlloc.mutex) 2354 . . persistent = &globalAlloc.persistentAlloc 2355 . . } 2356 . . persistent.off = alignUp(persistent.off, align) 2357 . . if persistent.off+size > persistentChunkSize || persistent.base == nil { 2358 . 10ms persistent.base = (*notInHeap)(sysAlloc(persistentChunkSize, &memstats.other_sys, "immortal metadata")) 2359 . . if persistent.base == nil { 2360 . . if persistent == &globalAlloc.persistentAlloc { 2361 . . unlock(&globalAlloc.mutex) 2362 . . } 2363 . . throw("runtime: cannot allocate memory")
internal/runtime/maps.(*Map).getWithoutKeySmallFastStr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 190ms 440ms (flat, cum) 0.62% 11 . . "internal/race" 12 . . "internal/runtime/sys" 13 . . "unsafe" 14 . . ) 15 . . 16 10ms 260ms func (m *Map) getWithoutKeySmallFastStr(typ *abi.MapType, key string) unsafe.Pointer { 17 . . g := groupReference{ 18 . . data: m.dirPtr, 19 . . } 20 . . 21 20ms 20ms ctrls := *g.ctrls() 22 20ms 20ms slotKey := g.key(typ, 0) 23 . . var keyStride uintptr 24 . . if goexperiment.MapSplitGroup { 25 . . keyStride = 2 * goarch.PtrSize // keys are contiguous in split layout 26 . . } else { 27 100ms 100ms keyStride = typ.KeyStride // == SlotSize in interleaved layout 28 . . } 29 . . 30 . . // The 64 threshold was chosen based on performance of BenchmarkMapStringKeysEight, 31 . . // where there are 8 keys to check, all of which don't quick-match the lookup key. 32 . . // In that case, we can save hashing the lookup key. That savings is worth this extra code 33 . . // for strings that are long enough that hashing is expensive. 34 40ms 40ms if len(key) > 64 { 35 . . // String hashing and equality might be expensive. Do a quick check first. 36 . . j := abi.MapGroupSlots 37 . . for i := range abi.MapGroupSlots { 38 . . if ctrls&(1<<7) == 0 && longStringQuickEqualityTest(key, *(*string)(slotKey)) { 39 . . if j < abi.MapGroupSlots {
internal/runtime/maps.(*Map).getWithoutKeySmallFastStr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 650ms 1.12s (flat, cum) 1.57% 64 . . 65 . . dohash: 66 . . // This path will cost 1 hash and 1+ε comparisons. 67 . . var hash uintptr 68 . . // See the related comment in runtime_mapaccess2_fast32 69 10ms 10ms if memHashAESImplemented && UseAeshash { 70 . 380ms hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 71 . . } else { 72 . . hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 73 . . } 74 . . h2 := uint8(h2(hash)) 75 90ms 90ms ctrls = *g.ctrls() return (*ctrlGroup)(g.data) group.go:294 ⋮ 76 30ms 30ms slotKey = g.key(typ, 0) offset := typ.KeysOff + i*typ.KeyStride group.go:299 77 . . 78 100ms 100ms for i := range uintptr(abi.MapGroupSlots) { 79 260ms 350ms if uint8(ctrls) == h2 && key == *(*string)(slotKey) { 80 . . if goexperiment.MapSplitGroup { 81 . . return g.elem(typ, i) 82 . . } else { 83 10ms 10ms return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) 84 . . } 85 . . } 86 150ms 150ms slotKey = unsafe.Pointer(uintptr(slotKey) + keyStride) 87 . . ctrls >>= 8 88 . . } 89 . . return nil 90 . . } 91 . .
runtime.mapaccess1_faststr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 40ms 890ms (flat, cum) 1.24% 109 . . } 110 . . return true 111 . . } 112 . . 113 . . //go:linkname runtime_mapaccess1_faststr runtime.mapaccess1_faststr 114 20ms 20ms func runtime_mapaccess1_faststr(typ *abi.MapType, m *Map, key string) unsafe.Pointer { 115 . 850ms p, _ := runtime_mapaccess2_faststr(typ, m, key) 116 20ms 20ms return p 117 . . }
runtime.mapaccess2_faststr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 480ms 2.32s (flat, cum) 3.24% 119 . . //go:linkname runtime_mapaccess2_faststr runtime.mapaccess2_faststr 120 40ms 260ms func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Pointer, bool) { 121 . . if race.Enabled && m != nil { 122 . . callerpc := sys.GetCallerPC() 123 . . pc := abi.FuncPCABIInternal(runtime_mapaccess2_faststr) 124 . . race.ReadPC(unsafe.Pointer(m), callerpc, pc) 125 . . } 126 . . 127 120ms 120ms if m == nil || m.Used() == 0 { 128 . . return unsafe.Pointer(&zeroVal[0]), false 129 . . } 130 . . 131 30ms 30ms if m.writing != 0 { 132 . . fatal("concurrent map read and map write") 133 . . return nil, false 134 . . } 135 . . 136 10ms 10ms if m.dirLen <= 0 { 137 . 1.56s elem := m.getWithoutKeySmallFastStr(typ, key) 138 30ms 30ms if elem == nil { 139 10ms 10ms return unsafe.Pointer(&zeroVal[0]), false 140 . . } 141 20ms 20ms return elem, true 142 . . } 143 . . 144 . . var hash uintptr 145 . . // See the related comment in runtime_mapaccess2_fast32 146 . . if memHashAESImplemented && UseAeshash { 147 10ms 60ms hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 148 . . } else { 149 . . hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 150 . . } 151 . . 152 . . // Select table. 153 30ms 30ms idx := m.directoryIndex(hash) if m.dirLen == 1 { map.go:358 154 40ms 40ms t := m.directoryAt(idx) return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) map.go:365 155 . . 156 . . // Probe table. 157 10ms 10ms seq := makeProbeSeq(h1(hash), t.groups.lengthMask) 158 . . h2Hash := h2(hash) 159 . . for ; ; seq = seq.next() { 160 . . g := t.groups.group(typ, seq.offset) 161 . . 162 30ms 30ms match := g.ctrls().matchH2(h2Hash) return ctrlGroupMatchH2(g, h) group.go:154 ⋮ 163 . . 164 20ms 20ms for match != 0 { 165 20ms 20ms i := match.first() 166 . . 167 40ms 40ms slotKey := g.key(typ, i) offset := typ.KeysOff + i*typ.KeyStride group.go:299 168 . 10ms if key == *(*string)(slotKey) { 169 . . if goexperiment.MapSplitGroup { 170 . . return g.elem(typ, i), true 171 . . } else { 172 . . return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize), true 173 . . } 174 . . } 175 . . match = match.removeFirst() 176 . . } 177 . . 178 10ms 10ms match = g.ctrls().matchEmpty() return ctrlGroupMatchEmpty(g) group.go:180 return bitset((v &^ (v << 6)) & bitsetMSB) group.go:195 179 10ms 10ms if match != 0 { 180 . . // Finding an empty slot means we've reached the end of 181 . . // the probe sequence. 182 . . return unsafe.Pointer(&zeroVal[0]), false
internal/runtime/maps.(*Map).putSlotSmallFastStr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 520ms 530ms (flat, cum) 0.74% 184 . . } 185 . . } 186 . . 187 50ms 50ms func (m *Map) putSlotSmallFastStr(typ *abi.MapType, hash uintptr, key string) unsafe.Pointer { 188 . . g := groupReference{ 189 40ms 40ms data: m.dirPtr, 190 . . } 191 . . 192 80ms 80ms match := g.ctrls().matchH2(h2(hash)) ⋮ return ctrlGroupMatchH2(g, h) group.go:154 return bitset((v&bitsetL7B + bitsetLSB) & (v & bitsetMSB)) group.go:165 ⋮ v = ^v group.go:164 193 . . 194 . . // Look for an existing slot containing this key. 195 40ms 40ms for match != 0 { 196 20ms 20ms i := match.first() 197 . . 198 . . slotKey := g.key(typ, i) 199 . 10ms if key == *(*string)(slotKey) { 200 . . // Key needs update, as the backing storage may differ. 201 10ms 10ms *(*string)(slotKey) = key 202 . . slotElem := g.elem(typ, i) 203 . . return slotElem 204 . . } 205 10ms 10ms match = match.removeFirst() 206 . . } 207 . . 208 . . // There can't be deleted slots, small maps can't have them 209 . . // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit 210 . . // more efficient than matchEmpty. 211 90ms 90ms match = g.ctrls().matchEmptyOrDeleted() 212 . . if match == 0 { 213 . . // No empty slot found. Need to grow the map. 214 . . return nil 215 . . } 216 . . 217 10ms 10ms i := match.first() return bitsetFirst(b) group.go:50 return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 group.go:58 218 . . 219 10ms 10ms slotKey := g.key(typ, i) offset := typ.KeysOff + i*typ.KeyStride group.go:299 220 40ms 40ms *(*string)(slotKey) = key 221 . . 222 50ms 50ms slotElem := g.elem(typ, i) return unsafe.Pointer(uintptr(g.data) + offset) group.go:308 ⋮ offset := typ.ElemsOff + i*typ.ElemStride group.go:306 ⋮ return unsafe.Pointer(uintptr(g.data) + offset) group.go:308 223 . . 224 20ms 20ms g.ctrls().set(i, ctrl(h2(hash))) *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c group.go:143 225 30ms 30ms m.used++ 226 . . 227 20ms 20ms return slotElem 228 . . }
internal/runtime/maps.(*table).uncheckedPutSlotForAssignFastStr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 10ms 10ms (flat, cum) 0.014% 230 . . func (t *table) uncheckedPutSlotForAssignFastStr(typ *abi.MapType, hash uintptr, key string) unsafe.Pointer { 231 10ms 10ms if t.growthLeft == 0 { 232 . . panic("invariant failed: growthLeft is unexpectedly 0") 233 . . } 234 . . 235 . . // Given key and its hash hash(key), to insert it, we construct a 236 . . // probeSeq, and use it to find the first group with an unoccupied (empty
runtime.mapassign_faststr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 300ms 2.31s (flat, cum) 3.23% 256 . . } 257 . . } 258 . . } 259 . . 260 . . //go:linkname runtime_mapassign_faststr runtime.mapassign_faststr 261 10ms 10ms func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Pointer { 262 20ms 20ms if m == nil { 263 . . panic(errNilAssign) 264 . . } 265 . . if race.Enabled { 266 . . callerpc := sys.GetCallerPC() 267 . . pc := abi.FuncPCABIInternal(runtime_mapassign_faststr) 268 . . race.WritePC(unsafe.Pointer(m), callerpc, pc) 269 . . } 270 40ms 40ms if m.writing != 0 { 271 . . fatal("concurrent map writes") 272 . . } 273 . . 274 . . var hash uintptr 275 . . // See the related comment in runtime_mapaccess2_fast32 276 20ms 20ms if memHashAESImplemented && UseAeshash { 277 . 250ms hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 278 . . } else { 279 . . hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) 280 . . } 281 . . 282 . . // Set writing after calling Hasher, since Hasher may panic, in which 283 . . // case we have not actually done a write. 284 90ms 90ms m.writing ^= 1 // toggle, see comment on writing 285 . . 286 . . if m.dirPtr == nil { 287 . 1.09s m.growToSmall(typ) 288 . . } 289 . . 290 10ms 10ms if m.dirLen == 0 { 291 40ms 570ms elem := m.putSlotSmallFastStr(typ, hash, key) 292 . . if elem == nil { 293 . . // Can't fit another entry, grow to full size map. 294 . 130ms tab := m.growToTable(typ) 295 . . 296 . 10ms elem = tab.uncheckedPutSlotForAssignFastStr(typ, hash, key) 297 . . m.used++ 298 . . 299 . . tab.checkInvariants(typ, m) 300 . . } 301 . . 302 20ms 20ms if m.writing == 0 { 303 . . fatal("concurrent map writes") 304 . . } 305 20ms 20ms m.writing ^= 1 306 . . 307 . . return elem 308 . . } 309 . . 310 . . var slotElem unsafe.Pointer 311 . . outer: 312 . . for { 313 . . // Select table. 314 10ms 10ms idx := m.directoryIndex(hash) 315 . . t := m.directoryAt(idx) 316 . . 317 . . seq := makeProbeSeq(h1(hash), t.groups.lengthMask) 318 . . 319 . . // As we look for a match, keep track of the first deleted slot 320 . . // we find, which we'll use to insert the new entry if 321 . . // necessary. 322 . . var firstDeletedGroup groupReference 323 . . var firstDeletedSlot uintptr 324 . . 325 . . h2Hash := h2(hash) 326 . . for ; ; seq = seq.next() { 327 . . g := t.groups.group(typ, seq.offset) 328 20ms 20ms match := g.ctrls().matchH2(h2Hash) 329 . . 330 . . // Look for an existing slot containing this key. 331 . . for match != 0 { 332 . . i := match.first() 333 . .
runtime.mapassign_faststr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 70ms 330ms (flat, cum) 0.46% 344 . . match = match.removeFirst() 345 . . } 346 . . 347 . . // No existing slot for this key in this group. Is this the end 348 . . // of the probe sequence? 349 10ms 10ms match = g.ctrls().matchEmptyOrDeleted() 350 . . if match == 0 { 351 . . continue // nothing but filled slots. Keep probing. 352 . . } 353 10ms 10ms i := match.first() 354 . . if g.ctrls().get(i) == ctrlDeleted { 355 . . // There are some deleted slots. Remember 356 . . // the first one, and keep probing. 357 . . if firstDeletedGroup.data == nil { 358 . . firstDeletedGroup = g 359 . . firstDeletedSlot = i 360 . . } 361 . . continue 362 . . } 363 . . // We've found an empty slot, which means we've reached the end of 364 . . // the probe sequence. 365 . . 366 . . // If we found a deleted slot along the way, we can 367 . . // replace it without consuming growthLeft. 368 40ms 40ms if firstDeletedGroup.data != nil { 369 . . g = firstDeletedGroup 370 . . i = firstDeletedSlot 371 . . t.growthLeft++ // will be decremented below to become a no-op. 372 . . } 373 . . 374 . . // If we have no space left, first try to remove some tombstones. 375 . . if t.growthLeft == 0 { 376 . . t.pruneTombstones(typ, m) 377 . . } 378 . . 379 . . // If there is room left to grow, just insert the new entry. 380 . . if t.growthLeft > 0 { 381 10ms 10ms slotKey := g.key(typ, i) offset := typ.KeysOff + i*typ.KeyStride group.go:299 382 . . *(*string)(slotKey) = key 383 . . 384 . . slotElem = g.elem(typ, i) 385 . . 386 . . g.ctrls().set(i, ctrl(h2Hash)) 387 . . t.growthLeft-- 388 . . t.used++ 389 . . m.used++ 390 . . 391 . . t.checkInvariants(typ, m) 392 . . break outer 393 . . } 394 . . 395 . 260ms t.rehash(typ, m) 396 . . continue outer 397 . . } 398 . . } 399 . . 400 . . if m.writing == 0 {
runtime.mapdelete_faststr
/usr/lib/go/src/internal/runtime/maps/runtime_faststr.go
Total: 0 20ms (flat, cum) 0.028% 415 . . 416 . . if m == nil || m.Used() == 0 { 417 . . return 418 . . } 419 . . 420 . 20ms m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) 421 . . }
runtime.stackpoolalloc
/usr/lib/go/src/runtime/stack.go
Total: 10ms 10ms (flat, cum) 0.014% 189 . . return bits.Len64(uint64(n)) 190 . . } 191 . . 192 . . // Allocates a stack from the free pool. Must be called with 193 . . // stackpool[order].item.mu held. 194 10ms 10ms func stackpoolalloc(order uint8) gclinkptr { 195 . . list := &stackpool[order].item.span 196 . . s := list.first 197 . . lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) 198 . . if s == nil { 199 . . // no free stacks. Allocate another span worth.
runtime.stackpoolalloc
/usr/lib/go/src/runtime/stack.go
Total: 90ms 90ms (flat, cum) 0.13% 222 . . s.manualFreeList = x 223 . . } 224 . . list.insert(s) 225 . . } 226 . . x := s.manualFreeList 227 30ms 30ms if x.ptr() == nil { return (*gclink)(unsafe.Pointer(p)) mcache.go:86 228 . . throw("span has no free stacks") 229 . . } 230 60ms 60ms s.manualFreeList = x.ptr().next 231 . . s.allocCount++ 232 . . if s.manualFreeList.ptr() == nil { 233 . . // all stacks in s are allocated. 234 . . list.remove(s) 235 . . }
runtime.stackpoolfree
/usr/lib/go/src/runtime/stack.go
Total: 140ms 160ms (flat, cum) 0.22% 236 . . return x 237 . . } 238 . . 239 . . // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held. 240 . . func stackpoolfree(x gclinkptr, order uint8) { 241 20ms 20ms s := spanOfUnchecked(uintptr(x)) ai := arenaIndex(p) mheap.go:723 return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) mheap.go:607 ⋮ return mheap_.arenas[ai.l1()][ai.l2()].spans[(p/pageSize)%pagesPerArena] mheap.go:724 242 110ms 110ms if s.state.get() != mSpanManual { return mSpanState(b.s.Load()) mheap.go:419 return Load8(&u.value) types.go:124 ⋮ 243 . . throw("freeing stack not in a stack span") 244 . . } 245 . . if s.manualFreeList.ptr() == nil { 246 . . // s will now have a free stack 247 . . stackpool[order].item.span.insert(s) 248 . . } 249 . . x.ptr().next = s.manualFreeList 250 . . s.manualFreeList = x 251 10ms 10ms s.allocCount-- 252 . . if gcphase == _GCoff && s.allocCount == 0 { 253 . . // Span is completely free. Return it to the heap 254 . . // immediately if we're sweeping. 255 . . // 256 . . // If GC is active, we delay the free until the end of 257 . . // GC to avoid the following type of situation: 258 . . // 259 . . // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer 260 . . // 2) The stack that pointer points to is copied 261 . . // 3) The old stack is freed 262 . . // 4) The containing span is marked free 263 . . // 5) GC attempts to mark the SudoG.elem pointer. The 264 . . // marking fails because the pointer looks like a 265 . . // pointer into a free span. 266 . . // 267 . . // By not freeing, we prevent step #4 until GC is done. 268 . . stackpool[order].item.span.remove(s) 269 . . s.manualFreeList = 0 270 . . osStackFree(s) 271 . 20ms mheap_.freeManual(s, spanAllocStack) 272 . . } 273 . . } 274 . . 275 . . // stackcacherefill/stackcacherelease implement a global pool of stack segments. 276 . . // The pool is required to prevent unlimited growth of per-thread caches.
runtime.stackcacherefill
/usr/lib/go/src/runtime/stack.go
Total: 0 110ms (flat, cum) 0.15% 285 . . // Grab half of the allowed capacity (to prevent thrashing). 286 . . var list gclinkptr 287 . . var size uintptr 288 . . lock(&stackpool[order].item.mu) 289 . . for size < _StackCacheSize/2 { 290 . 100ms x := stackpoolalloc(order) 291 . . x.ptr().next = list 292 . . list = x 293 . . size += fixedStack << order 294 . . } 295 . 10ms unlock(&stackpool[order].item.mu) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 296 . . c.stackcache[order].list = list 297 . . c.stackcache[order].size = size 298 . . } 299 . . 300 . . //go:systemstack
runtime.stackcacherelease
/usr/lib/go/src/runtime/stack.go
Total: 50ms 220ms (flat, cum) 0.31% 302 . . if stackDebug >= 1 { 303 . . print("stackcacherelease order=", order, "\n") 304 . . } 305 . . x := c.stackcache[order].list 306 . . size := c.stackcache[order].size 307 . 20ms lock(&stackpool[order].item.mu) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 308 . . for size > _StackCacheSize/2 { 309 40ms 40ms y := x.ptr().next 310 . 150ms stackpoolfree(x, order) 311 . . x = y 312 . . size -= fixedStack << order 313 . . } 314 10ms 10ms unlock(&stackpool[order].item.mu) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 315 . . c.stackcache[order].list = x 316 . . c.stackcache[order].size = size 317 . . } 318 . . 319 . . //go:systemstack
runtime.stackcache_clear
/usr/lib/go/src/runtime/stack.go
Total: 0 10ms (flat, cum) 0.014% 324 . . for order := uint8(0); order < _NumStackOrders; order++ { 325 . . lock(&stackpool[order].item.mu) 326 . . x := c.stackcache[order].list 327 . . for x.ptr() != nil { 328 . . y := x.ptr().next 329 . 10ms stackpoolfree(x, order) 330 . . x = y 331 . . } 332 . . c.stackcache[order].list = 0 333 . . c.stackcache[order].size = 0 334 . . unlock(&stackpool[order].item.mu)
runtime.stackalloc
/usr/lib/go/src/runtime/stack.go
Total: 40ms 40ms (flat, cum) 0.056% 339 . . // 340 . . // stackalloc must run on the system stack because it uses per-P 341 . . // resources and must not split the stack. 342 . . // 343 . . //go:systemstack 344 30ms 30ms func stackalloc(n uint32) stack { 345 . . // Stackalloc must be called on scheduler stack, so that we 346 . . // never try to grow the stack during the code that stackalloc runs. 347 . . // Doing so would cause a deadlock (issue 1547). 348 . . thisg := getg() 349 . . if thisg != thisg.m.g0 { 350 . . throw("stackalloc not on scheduler stack") 351 . . } 352 10ms 10ms if n&(n-1) != 0 { 353 . . throw("stack size not a power of 2") 354 . . } 355 . . if stackDebug >= 1 { 356 . . print("stackalloc ", n, "\n") 357 . . }
runtime.stackalloc
/usr/lib/go/src/runtime/stack.go
Total: 130ms 240ms (flat, cum) 0.34% 370 . . // a dedicated span. 371 . . var v unsafe.Pointer 372 . . if n < fixedStack<<_NumStackOrders && n < _StackCacheSize { 373 . . order := uint8(0) 374 . . n2 := n 375 10ms 10ms for n2 > fixedStack { 376 . . order++ 377 . . n2 >>= 1 378 . . } 379 . . var x gclinkptr 380 . . if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" { 381 . . // thisg.m.p == 0 can happen in the guts of exitsyscall 382 . . // or procresize. Just get a stack from the global pool. 383 . . // Also don't touch stackcache during gc 384 . . // as it's flushed concurrently. 385 . . lock(&stackpool[order].item.mu) 386 . . x = stackpoolalloc(order) 387 . . unlock(&stackpool[order].item.mu) 388 . . } else { 389 10ms 10ms c := thisg.m.p.ptr().mcache 390 . . x = c.stackcache[order].list 391 10ms 10ms if x.ptr() == nil { return (*gclink)(unsafe.Pointer(p)) mcache.go:86 392 . 110ms stackcacherefill(c, order) 393 . . x = c.stackcache[order].list 394 . . } 395 100ms 100ms c.stackcache[order].list = x.ptr().next ⋮ return (*gclink)(unsafe.Pointer(p)) mcache.go:86 396 . . c.stackcache[order].size -= uintptr(n) 397 . . } 398 . . if valgrindenabled { 399 . . // We're about to allocate the stack region starting at x.ptr(). 400 . . // To prevent valgrind from complaining about overlapping allocations,
runtime.stackalloc
/usr/lib/go/src/runtime/stack.go
Total: 10ms 10ms (flat, cum) 0.014% 449 . . valgrindMalloc(v, uintptr(n)) 450 . . } 451 . . if stackDebug >= 1 { 452 . . print(" allocated ", v, "\n") 453 . . } 454 10ms 10ms return stack{uintptr(v), uintptr(v) + uintptr(n)} 455 . . } 456 . . 457 . . // stackfree frees an n byte stack allocation at stk. 458 . . //
runtime.stackfree
/usr/lib/go/src/runtime/stack.go
Total: 180ms 400ms (flat, cum) 0.56% 459 . . // stackfree must run on the system stack because it uses per-P 460 . . // resources and must not split the stack. 461 . . // 462 . . //go:systemstack 463 10ms 10ms func stackfree(stk stack) { 464 . . gp := getg() 465 . . v := unsafe.Pointer(stk.lo) 466 . . n := stk.hi - stk.lo 467 . . if n&(n-1) != 0 { 468 . . throw("stack not a power of 2") 469 . . } 470 . . if stk.lo+n < stk.hi { 471 . . throw("bad stack size") 472 . . } 473 . . if stackDebug >= 1 { 474 . . println("stackfree", v, n) 475 . . memclrNoHeapPointers(v, n) // for testing, clobber stack data 476 . . } 477 10ms 10ms if debug.efence != 0 || stackFromSystem != 0 { 478 . . if debug.efence != 0 || stackFaultOnFree != 0 { 479 . . sysFault(v, n) 480 . . } else { 481 . . sysFree(v, n, &memstats.stacks_sys) 482 . . } 483 . . return 484 . . } 485 10ms 10ms if traceAllocFreeEnabled() { 486 . . trace := traceAcquire() 487 . . if trace.ok() { 488 . . trace.GoroutineStackFree(uintptr(v)) 489 . . traceRelease(trace) 490 . . } 491 . . } 492 . . if msanenabled { 493 . . msanfree(v, n) 494 . . } 495 . . if asanenabled { 496 . . asanpoison(v, n) 497 . . } 498 . . if valgrindenabled { 499 . . valgrindFree(v) 500 . . } 501 . . if n < fixedStack<<_NumStackOrders && n < _StackCacheSize { 502 . . order := uint8(0) 503 . . n2 := n 504 . . for n2 > fixedStack { 505 10ms 10ms order++ 506 10ms 10ms n2 >>= 1 507 . . } 508 . . x := gclinkptr(v) 509 20ms 20ms if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" { 510 . . lock(&stackpool[order].item.mu) 511 . . if valgrindenabled { 512 . . // x.ptr() is the head of the list of free stacks, and will be used 513 . . // when allocating a new stack, so it has to be marked allocated. 514 . . valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr())) 515 . . } 516 . . stackpoolfree(x, order) 517 . . unlock(&stackpool[order].item.mu) 518 . . } else { 519 . . c := gp.m.p.ptr().mcache 520 90ms 90ms if c.stackcache[order].size >= _StackCacheSize { 521 . 220ms stackcacherelease(c, order) 522 . . } 523 . . if valgrindenabled { 524 . . // x.ptr() is the head of the list of free stacks, and will 525 . . // be used when allocating a new stack, so it has to be 526 . . // marked allocated. 527 . . valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr())) 528 . . } 529 10ms 10ms x.ptr().next = c.stackcache[order].list 530 . . c.stackcache[order].list = x 531 10ms 10ms c.stackcache[order].size += n 532 . . } 533 . . } else { 534 . . s := spanOfUnchecked(uintptr(v)) 535 . . if s.state.get() != mSpanManual { 536 . . println(hex(s.base()), v)
runtime.stackfree
/usr/lib/go/src/runtime/stack.go
Total: 30ms 30ms (flat, cum) 0.042% 551 . . lock(&stackLarge.lock) 552 . . stackLarge.free[log2npage].insert(s) 553 . . unlock(&stackLarge.lock) 554 . . } 555 . . } 556 30ms 30ms } 557 . . 558 . . var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real 559 . . 560 . . var maxstackceiling = maxstacksize 561 . .
runtime.adjustpointer
/usr/lib/go/src/runtime/stack.go
Total: 100ms 100ms (flat, cum) 0.14% 607 . . 608 . . // adjustpointer checks whether *vpp is in the old stack described by adjinfo. 609 . . // If so, it rewrites *vpp to point into the new stack. 610 . . func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) { 611 . . pp := (*uintptr)(vpp) 612 70ms 70ms p := *pp 613 . . if stackDebug >= 4 { 614 . . print(" ", pp, ":", hex(p), "\n") 615 . . } 616 . . if valgrindenabled { 617 . . // p is a pointer on a stack, it is inherently initialized, as 618 . . // everything on the stack is, but valgrind for _some unknown reason_ 619 . . // sometimes thinks it's uninitialized, and flags operations on p below 620 . . // as uninitialized. We just initialize it if valgrind thinks its 621 . . // uninitialized. 622 . . // 623 . . // See go.dev/issues/73801. 624 . . valgrindMakeMemDefined(unsafe.Pointer(&p), unsafe.Sizeof(&p)) 625 . . } 626 30ms 30ms if adjinfo.old.lo <= p && p < adjinfo.old.hi { 627 . . *pp = p + adjinfo.delta 628 . . if stackDebug >= 3 { 629 . . print(" adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n") 630 . . } 631 . . }
runtime.adjustpointers
/usr/lib/go/src/runtime/stack.go
Total: 470ms 470ms (flat, cum) 0.66% 647 . . return (b >> (i % 8)) & 1 648 . . } 649 . . 650 . . // bv describes the memory starting at address scanp. 651 . . // Adjust any pointers contained therein. 652 40ms 40ms func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) { 653 . . minp := adjinfo.old.lo 654 . . maxp := adjinfo.old.hi 655 . . delta := adjinfo.delta 656 40ms 40ms num := uintptr(bv.n) 657 . . // If this frame might contain channel receive slots, use CAS 658 . . // to adjust pointers. If the slot hasn't been received into 659 . . // yet, it may contain stack pointers and a concurrent send 660 . . // could race with adjusting those pointers. (The sent value 661 . . // itself can never contain stack pointers.) 662 10ms 10ms useCAS := uintptr(scanp) < adjinfo.sghi 663 70ms 70ms for i := uintptr(0); i < num; i += 8 { 664 . . if stackDebug >= 4 { 665 . . for j := uintptr(0); j < 8; j++ { 666 . . print(" ", add(scanp, (i+j)*goarch.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*goarch.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n") 667 . . } 668 . . } 669 30ms 30ms b := *(addb(bv.bytedata, i/8)) return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) mbitmap.go:1034 ⋮ ⋮ 670 80ms 80ms for b != 0 { 671 60ms 60ms j := uintptr(sys.TrailingZeros8(b)) 672 . . b &= b - 1 673 10ms 10ms pp := (*uintptr)(add(scanp, (i+j)*goarch.PtrSize)) return unsafe.Pointer(uintptr(p) + x) stubs.go:25 674 . . retry: 675 . . p := *pp 676 40ms 40ms if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 { 677 . . // Looks like a junk value in a pointer slot. 678 . . // Live analysis wrong? 679 . . getg().m.traceback = 2 680 . . print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n") 681 . . throw("invalid pointer found on stack") 682 . . } 683 70ms 70ms if minp <= p && p < maxp { 684 . . if stackDebug >= 3 { 685 . . print("adjust ptr ", hex(p), " ", funcname(f), "\n") 686 . . } 687 10ms 10ms if useCAS { 688 . . ppu := (*unsafe.Pointer)(unsafe.Pointer(pp)) 689 . . if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) { 690 . . goto retry 691 . . } 692 . . } else { 693 . . *pp = p + delta 694 . . } 695 . . } 696 . . } 697 . . } 698 10ms 10ms } 699 . .
runtime.adjustframe
/usr/lib/go/src/runtime/stack.go
Total: 120ms 120ms (flat, cum) 0.17% 700 . . // Note: the argument/return area is adjusted by the callee. 701 40ms 40ms func adjustframe(frame *stkframe, adjinfo *adjustinfo) { 702 . . // Adjust saved frame pointer if there is one. 703 . . if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize { 704 . . if stackDebug >= 3 { 705 . . print(" saved bp\n") 706 . . } 707 . . if debugCheckBP { 708 . . // Frame pointers should always point to the next higher frame on 709 . . // the Go stack (or be nil, for the top frame on the stack). 710 . . bp := *(*uintptr)(unsafe.Pointer(frame.varp)) 711 . . if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) { 712 . . println("runtime: found invalid frame pointer") 713 . . print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") 714 . . throw("bad frame pointer") 715 . . } 716 . . } 717 . . // On AMD64, this is the caller's frame pointer saved in the current 718 . . // frame. 719 . . // On ARM64, this is the frame pointer of the caller's caller saved 720 . . // by the caller in its frame (one word below its SP). 721 30ms 30ms adjustpointer(adjinfo, unsafe.Pointer(frame.varp)) if adjinfo.old.lo <= p && p < adjinfo.old.hi { stack.go:626 ⋮ p := *pp stack.go:612 722 . . } 723 50ms 50ms if goarch.ArchFamily == goarch.ARM64 && isInjectedCall(frame.fn.funcID) { ⋮ return id == abi.FuncID_sigpanic || id == abi.FuncID_asyncPreempt || id == abi.FuncID_debugCallV2 traceback.go:442 724 . . // If this is an injected call on arm64, then we need to adjust 725 . . // the frame pointer saved by the original function into which 726 . . // the call was injected. Normally this would be handled when 727 . . // adjusting the callee's frame or in adjustctxt. But when a 728 . . // call is injected, the frame is placed 16 bytes below the
runtime.adjustframe
/usr/lib/go/src/runtime/stack.go
Total: 470ms 4.44s (flat, cum) 6.21% 746 . . // | injected call | 747 . . // V frame below... V 748 . . adjustpointer(adjinfo, unsafe.Pointer(frame.fp+goarch.PtrSize)) 749 . . } 750 . . 751 30ms 30ms if frame.continpc == 0 { 752 . . // Frame is dead. The program might still see the frame pointer 753 . . // saved in the frame, adjusted above, but we don't need to 754 . . // adjust the rest of the frame. 755 . . return 756 . . } 757 . . f := frame.fn 758 . . if stackDebug >= 2 { 759 . . print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") 760 . . } 761 . . 762 10ms 3.51s locals, args, objs := frame.getStackMap(true) 763 . . 764 . . // Adjust local variables if stack frame has been allocated. 765 . . if locals.n > 0 { 766 . . size := uintptr(locals.n) * goarch.PtrSize 767 . 210ms adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f) 768 . . } 769 . . 770 . . // Adjust arguments. 771 20ms 20ms if args.n > 0 { 772 . . if stackDebug >= 3 { 773 . . print(" args\n") 774 . . } 775 30ms 290ms adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{}) 776 . . } 777 . . 778 . . // Adjust pointers in all stack objects (whether they are live or not). 779 . . // See comments in mgcmark.go:scanframeworker. 780 30ms 30ms if frame.varp != 0 { 781 40ms 40ms for i := range objs { 782 30ms 30ms obj := &objs[i] 783 20ms 20ms off := obj.off 784 . . base := frame.varp // locals base pointer 785 10ms 10ms if off >= 0 { 786 . . base = frame.argp // arguments and return values base pointer 787 . . } 788 10ms 10ms p := base + uintptr(off) 789 . . if p < frame.sp { 790 . . // Object hasn't been allocated in the frame yet. 791 . . // (Happens when the stack bounds check fails and 792 . . // we call into morestack.) 793 . . continue 794 . . } 795 90ms 90ms ptrBytes, gcData := obj.gcdata() res := mod.rodata + uintptr(r.gcdataoff) stack.go:1403 ⋮ for datap := &firstmoduledata; datap != nil; datap = datap.next { stack.go:1388 ⋮ if datap.gofunc <= ptr && ptr < datap.epclntab { stack.go:1390 ⋮ ⋮ res := mod.rodata + uintptr(r.gcdataoff) stack.go:1403 796 40ms 40ms for i := uintptr(0); i < ptrBytes; i += goarch.PtrSize { 797 70ms 70ms if *addb(gcData, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 { 798 30ms 30ms adjustpointer(adjinfo, unsafe.Pointer(p+i)) p := *pp stack.go:612 799 . . } 800 . . } 801 . . } 802 . . } 803 10ms 10ms } 804 . . 805 . . func adjustctxt(gp *g, adjinfo *adjustinfo) { 806 . . adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt)) 807 . . if !framepointer_enabled { 808 . . return
runtime.adjustctxt
/usr/lib/go/src/runtime/stack.go
Total: 40ms 40ms (flat, cum) 0.056% 814 . . print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n") 815 . . throw("bad top frame pointer") 816 . . } 817 . . } 818 . . oldfp := gp.sched.bp 819 10ms 10ms adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp)) if adjinfo.old.lo <= p && p < adjinfo.old.hi { stack.go:626 820 . . if GOARCH == "arm64" { 821 . . // On ARM64, the frame pointer is saved one word *below* the SP, 822 . . // which is not copied or adjusted in any frame. Do it explicitly 823 . . // here. 824 10ms 10ms if oldfp == gp.sched.sp-goarch.PtrSize { 825 10ms 10ms memmove(unsafe.Pointer(gp.sched.bp), unsafe.Pointer(oldfp), goarch.PtrSize) 826 10ms 10ms adjustpointer(adjinfo, unsafe.Pointer(gp.sched.bp)) p := *pp stack.go:612 827 . . } 828 . . } 829 . . } 830 . . 831 . . func adjustdefers(gp *g, adjinfo *adjustinfo) {
runtime.adjustpanics
/usr/lib/go/src/runtime/stack.go
Total: 20ms 20ms (flat, cum) 0.028% 841 . . } 842 . . 843 . . func adjustpanics(gp *g, adjinfo *adjustinfo) { 844 . . // Panics are on stack and already adjusted. 845 . . // Update pointer to head of list in G. 846 20ms 20ms adjustpointer(adjinfo, unsafe.Pointer(&gp._panic)) p := *pp stack.go:612 ⋮ if adjinfo.old.lo <= p && p < adjinfo.old.hi { stack.go:626 847 . . } 848 . . 849 . . func adjustsudogs(gp *g, adjinfo *adjustinfo) { 850 . . // the data elements pointed to by a SudoG structure 851 . . // might be in the stack.
runtime.copystack
/usr/lib/go/src/runtime/stack.go
Total: 50ms 190ms (flat, cum) 0.27% 924 . . return sgsize 925 . . } 926 . . 927 . . // Copies gp's stack to a new stack of a different size. 928 . . // Caller must have changed gp status to Gcopystack. 929 10ms 10ms func copystack(gp *g, newsize uintptr) { 930 . . if gp.syscallsp != 0 { 931 . . throw("stack growth not allowed in system call") 932 . . } 933 . . old := gp.stack 934 . . if old.lo == 0 { 935 . . throw("nil stackbase") 936 . . } 937 . . used := old.hi - gp.sched.sp 938 . . // Add just the difference to gcController.addScannableStack. 939 . . // g0 stacks never move, so this will never account for them. 940 . . // It's also fine if we have no P, addScannableStack can deal with 941 . . // that case. 942 10ms 10ms gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo)) pp.maxStackScanDelta += amount mgcpacer.go:979 943 . . 944 . . // allocate new stack 945 . 140ms new := stackalloc(uint32(newsize)) 946 . . if stackPoisonCopy != 0 { 947 . . fillstack(new, 0xfd) 948 . . } 949 . . if stackDebug >= 1 { 950 . . print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n") 951 . . } 952 . . 953 . . // Compute adjustment. 954 . . var adjinfo adjustinfo 955 10ms 10ms adjinfo.old = old 956 . . adjinfo.delta = new.hi - old.hi 957 . . 958 . . // Adjust sudogs, synchronizing with channel ops if necessary. 959 . . ncopy := used 960 . . if !gp.activeStackChans { 961 20ms 20ms if newsize < old.hi-old.lo && gp.parkingOnChan.Load() { 962 . . // It's not safe for someone to shrink this stack while we're actively 963 . . // parking on a channel, but it is safe to grow since we do that 964 . . // ourselves and explicitly don't want to synchronize with channels 965 . . // since we could self-deadlock. 966 . . throw("racy sudog adjustment due to parking on channel")
runtime.copystack
/usr/lib/go/src/runtime/stack.go
Total: 50ms 8.98s (flat, cum) 12.55% 980 . . // the stack they may interact with. 981 . . ncopy -= syncadjustsudogs(gp, used, &adjinfo) 982 . . } 983 . . 984 . . // Copy the stack (or the rest of it) to the new location 985 . 120ms memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy) 986 . . 987 . . // Adjust remaining structures that have pointers into stacks. 988 . . // We have to do most of these before we traceback the new 989 . . // stack because gentraceback uses them. 990 . 40ms adjustctxt(gp, &adjinfo) 991 . . adjustdefers(gp, &adjinfo) 992 20ms 20ms adjustpanics(gp, &adjinfo) adjustpointer(adjinfo, unsafe.Pointer(&gp._panic)) stack.go:846 p := *pp stack.go:612 ⋮ if adjinfo.old.lo <= p && p < adjinfo.old.hi { stack.go:626 993 . . if adjinfo.sghi != 0 { 994 . . adjinfo.sghi += adjinfo.delta 995 . . } 996 . . 997 . . // Swap out old stack for new one 998 . . gp.stack = new 999 . . gp.stackguard0 = new.lo + stackGuard // NOTE: might clobber a preempt request 1000 . . gp.sched.sp = new.hi - used 1001 . . gp.stktopsp += adjinfo.delta 1002 . . 1003 . . // Adjust pointers in the new stack. 1004 10ms 10ms var u unwinder 1005 20ms 4.23s for u.init(gp, 0); u.valid(); u.next() { ⋮ return u.frame.pc != 0 traceback.go:229 ⋮ ⋮ u.initAt(^uintptr(0), ^uintptr(0), ^uintptr(0), gp, flags) traceback.go:130 1006 . 4.56s adjustframe(&u.frame, &adjinfo) 1007 . . } 1008 . . 1009 . . if valgrindenabled { 1010 . . if gp.valgrindStackID == 0 { 1011 . . gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(new.lo), unsafe.Pointer(new.hi))
runtime.copystack
/usr/lib/go/src/runtime/stack.go
Total: 20ms 360ms (flat, cum) 0.5% 1023 . . // is fast and optimized, might not be worth it. 1024 . . memclrNoHeapPointers(unsafe.Pointer(old.lo), old.hi-old.lo) 1025 . . // The memmove call above might put secrets from the stack into registers. 1026 . . secretEraseRegisters() 1027 . . } 1028 10ms 10ms if stackPoisonCopy != 0 { 1029 . . fillstack(old, 0xfc) 1030 . . } 1031 . 340ms stackfree(old) 1032 10ms 10ms } 1033 . . 1034 . . // round x up to a power of 2. 1035 . . func round2(x int32) int32 { 1036 . . s := uint(0) 1037 . . for 1<<s < x {
runtime.newstack
/usr/lib/go/src/runtime/stack.go
Total: 10ms 10ms (flat, cum) 0.014% 1050 . . // This must be nowritebarrierrec because it can be called as part of 1051 . . // stack growth from other nowritebarrierrec functions, but the 1052 . . // compiler doesn't check this. 1053 . . // 1054 . . //go:nowritebarrierrec 1055 10ms 10ms func newstack() { 1056 . . thisg := getg() 1057 . . // TODO: double check all gp. shouldn't be getg(). 1058 . . if thisg.m.morebuf.g.ptr().stackguard0 == stackFork { 1059 . . throw("stack growth after fork") 1060 . . }
runtime.newstack
/usr/lib/go/src/runtime/stack.go
Total: 10ms 10ms (flat, cum) 0.014% 1073 . . // is not in secret mode, and it will not be careful 1074 . . // about clearing its registers. 1075 . . secretEraseRegisters() 1076 . . } 1077 . . 1078 10ms 10ms if thisg.m.curg.throwsplit { 1079 . . // Update syscallsp, syscallpc in case traceback uses them. 1080 . . morebuf := thisg.m.morebuf 1081 . . gp.syscallsp = morebuf.sp 1082 . . gp.syscallpc = morebuf.pc 1083 . . pcname, pcoff := "(unknown)", uintptr(0)
runtime.newstack
/usr/lib/go/src/runtime/stack.go
Total: 20ms 20ms (flat, cum) 0.028% 1094 . . thisg.m.traceback = 2 // Include runtime frames 1095 . . traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp) 1096 . . throw("runtime: stack split at bad time") 1097 . . } 1098 . . 1099 10ms 10ms morebuf := thisg.m.morebuf 1100 . . thisg.m.morebuf.pc = 0 1101 . . thisg.m.morebuf.lr = 0 1102 . . thisg.m.morebuf.sp = 0 1103 . . thisg.m.morebuf.g = 0 1104 . . 1105 . . // NOTE: stackguard0 may change underfoot, if another thread 1106 . . // is about to try to preempt gp. Read it just once and use that same 1107 . . // value now and below. 1108 10ms 10ms stackguard0 := atomic.Loaduintptr(&gp.stackguard0) 1109 . . 1110 . . // Be conservative about where we preempt. 1111 . . // We are interested in preempting user Go code, not runtime code. 1112 . . // If we're holding locks, mallocing, or preemption is disabled, don't 1113 . . // preempt.
runtime.newstack
/usr/lib/go/src/runtime/stack.go
Total: 10ms 10ms (flat, cum) 0.014% 1127 . . gp.stackguard0 = gp.stack.lo + stackGuard 1128 . . gogo(&gp.sched) // never return 1129 . . } 1130 . . } 1131 . . 1132 10ms 10ms if gp.stack.lo == 0 { 1133 . . throw("missing stack in newstack") 1134 . . } 1135 . . sp := gp.sched.sp 1136 . . if goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.I386 || goarch.ArchFamily == goarch.WASM { 1137 . . // The call to morestack cost a word.
runtime.newstack
/usr/lib/go/src/runtime/stack.go
Total: 40ms 10.41s (flat, cum) 14.55% 1174 . . gopreempt_m(gp) // never return 1175 . . } 1176 . . 1177 . . // Allocate a bigger segment and move the stack. 1178 . . oldsize := gp.stack.hi - gp.stack.lo 1179 10ms 10ms newsize := oldsize * 2 1180 . . 1181 . . // Make sure we grow at least as much as needed to fit the new frame. 1182 . . // (This is just an optimization - the caller of morestack will 1183 . . // recheck the bounds on return.) 1184 . 210ms if f := findfunc(gp.sched.pc); f.valid() { 1185 . 420ms max := uintptr(funcMaxSPDelta(f)) 1186 . . needed := max + stackGuard 1187 10ms 10ms used := gp.stack.hi - gp.sched.sp 1188 10ms 10ms for newsize-used < needed { 1189 . . newsize *= 2 1190 . . } 1191 . . } 1192 . . 1193 . . if stackguard0 == stackForceMove { 1194 . . // Forced stack movement used for debugging. 1195 . . // Don't double the stack (or we may quickly run out 1196 . . // if this is done repeatedly). 1197 . . newsize = oldsize 1198 . . } 1199 . . 1200 10ms 10ms if newsize > maxstacksize || newsize > maxstackceiling { 1201 . . if maxstacksize < maxstackceiling { 1202 . . print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n") 1203 . . } else { 1204 . . print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n") 1205 . . } 1206 . . print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n") 1207 . . throw("stack overflow") 1208 . . } 1209 . . 1210 . . // The goroutine must be executing in order to call newstack, 1211 . . // so it must be Grunning (or Gscanrunning). 1212 . 40ms casgstatus(gp, _Grunning, _Gcopystack) 1213 . . 1214 . . // The concurrent GC will not scan the stack while we are doing the copy since 1215 . . // the gp is in a Gcopystack status. 1216 . 9.53s copystack(gp, newsize) 1217 . . if stackDebug >= 1 { 1218 . . print("stack grow done\n") 1219 . . } 1220 . 170ms casgstatus(gp, _Gcopystack, _Grunning) 1221 . . gogo(&gp.sched) 1222 . . } 1223 . . 1224 . . //go:nosplit 1225 . . func nilfunc() {
runtime.gostartcallfn
/usr/lib/go/src/runtime/stack.go
Total: 40ms 40ms (flat, cum) 0.056% 1233 . . if fv != nil { 1234 . . fn = unsafe.Pointer(fv.fn) 1235 . . } else { 1236 . . fn = unsafe.Pointer(abi.FuncPCABIInternal(nilfunc)) 1237 . . } 1238 40ms 40ms gostartcall(gobuf, fn, unsafe.Pointer(fv)) if buf.lr != 0 { sys_arm64.go:12 1239 . . } 1240 . . 1241 . . // isShrinkStackSafe returns whether it's safe to attempt to shrink 1242 . . // gp's stack. Shrinking the stack is only safe when we have precise 1243 . . // pointer maps for all frames on the stack. The caller must hold the
runtime.(*stackObjectRecord).gcdata
/usr/lib/go/src/runtime/stack.go
Total: 90ms 90ms (flat, cum) 0.13% 1383 . . // a ptr/nonptr bitmask covering those bytes. 1384 . . // Note that this bitmask might be larger than internal/abi.MaxPtrmaskBytes. 1385 . . func (r *stackObjectRecord) gcdata() (uintptr, *byte) { 1386 . . ptr := uintptr(unsafe.Pointer(r)) 1387 . . var mod *moduledata 1388 10ms 10ms for datap := &firstmoduledata; datap != nil; datap = datap.next { 1389 . . // The normal case: stackObjectRecord is in funcdata. 1390 60ms 60ms if datap.gofunc <= ptr && ptr < datap.epclntab { 1391 . . mod = datap 1392 . . break 1393 . . } 1394 . . // A special case: methodValueCallFrameObjs. 1395 . . if datap.noptrbss <= ptr && ptr < datap.enoptrbss { 1396 . . mod = datap 1397 . . break 1398 . . } 1399 . . } 1400 . . // If you get a panic here due to a nil mod, 1401 . . // you may have made a copy of a stackObjectRecord. 1402 . . // You must use the original pointer. 1403 20ms 20ms res := mod.rodata + uintptr(r.gcdataoff) 1404 . . return uintptr(r.ptrBytes), (*byte)(unsafe.Pointer(res)) 1405 . . } 1406 . . 1407 . . // This is exported as ABI0 via linkname so obj can call it. 1408 . . //
internal/runtime/atomic.(*Int32).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 10ms 10ms (flat, cum) 0.014% 16 . . 17 . . // Load accesses and returns the value atomically. 18 . . // 19 . . //go:nosplit 20 . . func (i *Int32) Load() int32 { 21 10ms 10ms return Loadint32(&i.value) 22 . . } 23 . . 24 . . // Store updates the value atomically. 25 . . // 26 . . //go:nosplit
internal/runtime/atomic.(*Int32).CompareAndSwap
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 30ms 30ms (flat, cum) 0.042% 32 . . // and if they're equal, swaps i's value with new. 33 . . // It reports whether the swap ran. 34 . . // 35 . . //go:nosplit 36 . . func (i *Int32) CompareAndSwap(old, new int32) bool { 37 30ms 30ms return Casint32(&i.value, old, new) 38 . . } 39 . . 40 . . // Swap replaces i's value with new, returning 41 . . // i's value before the replacement. 42 . . //
internal/runtime/atomic.(*Int32).Add
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 100ms 100ms (flat, cum) 0.14% 51 . . // This operation wraps around in the usual 52 . . // two's-complement way. 53 . . // 54 . . //go:nosplit 55 . . func (i *Int32) Add(delta int32) int32 { 56 100ms 100ms return Xaddint32(&i.value, delta) 57 . . } 58 . . 59 . . // Int64 is an atomically accessed int64 value. 60 . . // 61 . . // 8-byte aligned on all platforms, unlike a regular int64.
internal/runtime/atomic.(*Int64).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 100ms 100ms (flat, cum) 0.14% 69 . . 70 . . // Load accesses and returns the value atomically. 71 . . // 72 . . //go:nosplit 73 . . func (i *Int64) Load() int64 { 74 100ms 100ms return Loadint64(&i.value) 75 . . } 76 . . 77 . . // Store updates the value atomically. 78 . . // 79 . . //go:nosplit
internal/runtime/atomic.(*Int64).Add
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 30ms 30ms (flat, cum) 0.042% 104 . . // This operation wraps around in the usual 105 . . // two's-complement way. 106 . . // 107 . . //go:nosplit 108 . . func (i *Int64) Add(delta int64) int64 { 109 30ms 30ms return Xaddint64(&i.value, delta) 110 . . } 111 . . 112 . . // Uint8 is an atomically accessed uint8 value. 113 . . // 114 . . // A Uint8 must not be copied.
internal/runtime/atomic.(*Uint8).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 90ms 90ms (flat, cum) 0.13% 119 . . 120 . . // Load accesses and returns the value atomically. 121 . . // 122 . . //go:nosplit 123 . . func (u *Uint8) Load() uint8 { 124 90ms 90ms return Load8(&u.value) 125 . . } 126 . . 127 . . // Store updates the value atomically. 128 . . // 129 . . //go:nosplit
internal/runtime/atomic.(*Bool).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 60ms 60ms (flat, cum) 0.084% 163 . . 164 . . // Load accesses and returns the value atomically. 165 . . // 166 . . //go:nosplit 167 . . func (b *Bool) Load() bool { 168 60ms 60ms return b.u.Load() != 0 169 . . } 170 . . 171 . . // Store updates the value atomically. 172 . . // 173 . . //go:nosplit
internal/runtime/atomic.(*Uint32).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 30ms 30ms (flat, cum) 0.042% 189 . . 190 . . // Load accesses and returns the value atomically. 191 . . // 192 . . //go:nosplit 193 . . func (u *Uint32) Load() uint32 { 194 30ms 30ms return Load(&u.value) 195 . . } 196 . . 197 . . // LoadAcquire is a partially unsynchronized version 198 . . // of Load that relaxes ordering constraints. Other threads 199 . . // may observe operations that precede this operation to
internal/runtime/atomic.(*Uint32).CompareAndSwap
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 490ms 490ms (flat, cum) 0.69% 231 . . // and if they're equal, swaps u's value with new. 232 . . // It reports whether the swap ran. 233 . . // 234 . . //go:nosplit 235 . . func (u *Uint32) CompareAndSwap(old, new uint32) bool { 236 490ms 490ms return Cas(&u.value, old, new) 237 . . } 238 . . 239 . . // CompareAndSwapRelease is a partially unsynchronized version 240 . . // of Cas that relaxes ordering constraints. Other threads 241 . . // may observe operations that occur after this operation to
internal/runtime/atomic.(*Uint32).Add
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 140ms 140ms (flat, cum) 0.2% 286 . . // This operation wraps around in the usual 287 . . // two's-complement way. 288 . . // 289 . . //go:nosplit 290 . . func (u *Uint32) Add(delta int32) uint32 { 291 140ms 140ms return Xadd(&u.value, delta) 292 . . } 293 . . 294 . . // Uint64 is an atomically accessed uint64 value. 295 . . // 296 . . // 8-byte aligned on all platforms, unlike a regular uint64.
internal/runtime/atomic.(*Uint64).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 30ms 30ms (flat, cum) 0.042% 304 . . 305 . . // Load accesses and returns the value atomically. 306 . . // 307 . . //go:nosplit 308 . . func (u *Uint64) Load() uint64 { 309 30ms 30ms return Load64(&u.value) 310 . . } 311 . . 312 . . // Store updates the value atomically. 313 . . // 314 . . //go:nosplit
internal/runtime/atomic.(*Uint64).CompareAndSwap
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 80ms 80ms (flat, cum) 0.11% 320 . . // and if they're equal, swaps u's value with new. 321 . . // It reports whether the swap ran. 322 . . // 323 . . //go:nosplit 324 . . func (u *Uint64) CompareAndSwap(old, new uint64) bool { 325 80ms 80ms return Cas64(&u.value, old, new) 326 . . } 327 . . 328 . . // Swap replaces u's value with new, returning 329 . . // u's value before the replacement. 330 . . //
internal/runtime/atomic.(*Uint64).Add
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 360ms 360ms (flat, cum) 0.5% 339 . . // This operation wraps around in the usual 340 . . // two's-complement way. 341 . . // 342 . . //go:nosplit 343 . . func (u *Uint64) Add(delta int64) uint64 { 344 360ms 360ms return Xadd64(&u.value, delta) 345 . . } 346 . . 347 . . // Uintptr is an atomically accessed uintptr value. 348 . . // 349 . . // A Uintptr must not be copied.
internal/runtime/atomic.(*Uintptr).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 10ms 10ms (flat, cum) 0.014% 354 . . 355 . . // Load accesses and returns the value atomically. 356 . . // 357 . . //go:nosplit 358 . . func (u *Uintptr) Load() uintptr { 359 10ms 10ms return Loaduintptr(&u.value) 360 . . } 361 . . 362 . . // LoadAcquire is a partially unsynchronized version 363 . . // of Load that relaxes ordering constraints. Other threads 364 . . // may observe operations that precede this operation to
internal/runtime/atomic.(*Uintptr).Add
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 30ms 30ms (flat, cum) 0.042% 415 . . // This operation wraps around in the usual 416 . . // two's-complement way. 417 . . // 418 . . //go:nosplit 419 . . func (u *Uintptr) Add(delta uintptr) uintptr { 420 30ms 30ms return Xadduintptr(&u.value, delta) 421 . . } 422 . . 423 . . // Float64 is an atomically accessed float64 value. 424 . . // 425 . . // 8-byte aligned on all platforms, unlike a regular float64.
internal/runtime/atomic.(*UnsafePointer).Load
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 20ms 20ms (flat, cum) 0.028% 461 . . 462 . . // Load accesses and returns the value atomically. 463 . . // 464 . . //go:nosplit 465 . . func (u *UnsafePointer) Load() unsafe.Pointer { 466 20ms 20ms return Loadp(unsafe.Pointer(&u.value)) 467 . . } 468 . . 469 . . // StoreNoWB updates the value atomically. 470 . . // 471 . . // WARNING: As the name implies this operation does *not*
internal/runtime/atomic.(*UnsafePointer).StoreNoWB
/usr/lib/go/src/internal/runtime/atomic/types.go
Total: 20ms 20ms (flat, cum) 0.028% 474 . . // It is safe to use with values not found in the Go heap. 475 . . // Prefer Store instead. 476 . . // 477 . . //go:nosplit 478 . . func (u *UnsafePointer) StoreNoWB(value unsafe.Pointer) { 479 20ms 20ms StorepNoWB(unsafe.Pointer(&u.value), value) 480 . . } 481 . . 482 . . // Store updates the value atomically. 483 . . func (u *UnsafePointer) Store(value unsafe.Pointer) { 484 . . storePointer(&u.value, value)
runtime.mutexPreferLowLatency
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 10ms (flat, cum) 0.014% 142 . . //go:nosplit 143 . . func mutexPreferLowLatency(l *mutex) bool { 144 . . switch l { 145 . . default: 146 . . return false 147 10ms 10ms case &sched.lock: 148 . . // We often expect sched.lock to pass quickly between Ms in a way that 149 . . // each M has unique work to do: for instance when we stop-the-world 150 . . // (bringing each P to idle) or add new netpoller-triggered work to the 151 . . // global run queue. 152 . . return true
runtime.lock
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 1.24s (flat, cum) 1.73% 156 . . func mutexContended(l *mutex) bool { 157 . . return atomic.Loaduintptr(&l.key)&^mutexMMask != 0 158 . . } 159 . . 160 . . func lock(l *mutex) { 161 10ms 1.24s lockWithRank(l, getLockRank(l)) lock2(l) lockrank_off.go:24 ⋮ ⋮ ⋮ lock2(l) lockrank_off.go:24 ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ 162 . . } 163 . .
runtime.lock2
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 780ms 780ms (flat, cum) 1.09% 164 . . func lock2(l *mutex) { 165 . . gp := getg() 166 10ms 10ms if gp.m.locks < 0 { 167 . . throw("runtime·lock: lock count") 168 . . } 169 . . gp.m.locks += mutexMLocksDelta 170 . . 171 . . k8 := key8(&l.key) 172 . . 173 . . // Speculative grab for lock. 174 . . v8 := atomic.Xchg8(k8, mutexLocked) 175 760ms 760ms if v8&mutexLocked == 0 { 176 . . if v8&mutexSleeping != 0 { 177 . . atomic.Or8(k8, mutexSleeping) 178 . . } 179 10ms 10ms return 180 . . } 181 . . semacreate(gp.m) 182 . . 183 . . var startTime int64 184 . . // On uniprocessors, no point spinning.
runtime.lock2
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 180ms 480ms (flat, cum) 0.67% 198 . . if next&^mutexMMask == 0 { 199 . . // The fast-path Xchg8 may have cleared mutexSleeping. Fix 200 . . // the hint so unlock2 knows when to use its slow path. 201 . . next = next &^ mutexSleeping 202 . . } 203 40ms 40ms if atomic.Casuintptr(&l.key, v, next) { 204 . . gp.m.mLockProfile.end(startTime) 205 . . return 206 . . } 207 . . } else { 208 . . prev8 := atomic.Xchg8(k8, mutexLocked|mutexSleeping) 209 . . if prev8&mutexLocked == 0 { 210 . . gp.m.mLockProfile.end(startTime) 211 . . return 212 . . } 213 . . } 214 . . v = atomic.Loaduintptr(&l.key) 215 . . continue tryAcquire 216 . . } 217 . . 218 10ms 10ms if !weSpin && v&mutexSpinning == 0 && atomic.Casuintptr(&l.key, v, v|mutexSpinning) { 219 . . v |= mutexSpinning 220 . . weSpin = true 221 . . } 222 . . 223 10ms 10ms if weSpin || atTail || mutexPreferLowLatency(l) { case &sched.lock: lock_spinbit.go:147 224 40ms 40ms if i < spin { 225 . . procyield(mutexActiveSpinSize) 226 . . v = atomic.Loaduintptr(&l.key) 227 . . continue tryAcquire 228 . . } else if i < spin+mutexPassiveSpinCount { 229 . 200ms osyield() // TODO: Consider removing this step. See https://go.dev/issue/69268. 230 10ms 10ms v = atomic.Loaduintptr(&l.key) 231 . . continue tryAcquire 232 . . } 233 . . } 234 . . 235 . . // Go to sleep 236 . . if v&mutexLocked == 0 { 237 . . throw("runtime·lock: sleeping while lock is available") 238 . . } 239 . . 240 . . // Collect times for mutex profile (seen in unlock2 only via mWaitList), 241 . . // and for "/sync/mutex/wait/total:seconds" metric (to match). 242 . . if !haveTimers { 243 20ms 20ms gp.m.mWaitList.startTicks = cputicks() return nanotime() os_linux_arm64.go:21 return nanotime1() time_nofake.go:33 244 . 10ms startTime = gp.m.mLockProfile.start() 245 . . haveTimers = true 246 . . } 247 . . // Store the current head of the list of sleeping Ms in our gp.m.mWaitList.next field 248 . . gp.m.mWaitList.next = mutexWaitListHead(v) 249 . . 250 . . // Pack a (partial) pointer to this M with the current lock state bits 251 . . next := (uintptr(unsafe.Pointer(gp.m)) &^ mutexMMask) | v&mutexMMask | mutexSleeping 252 . . if weSpin { // If we were spinning, prepare to retire 253 . . next = next &^ mutexSpinning 254 . . } 255 . . 256 30ms 30ms if atomic.Casuintptr(&l.key, v, next) { 257 . . weSpin = false 258 . . // We've pushed ourselves onto the stack of waiters. Wait. 259 . 90ms semasleep(-1) 260 10ms 10ms atTail = gp.m.mWaitList.next == 0 // we were at risk of starving 261 . . i = 0 262 . . } 263 . . 264 10ms 10ms gp.m.mWaitList.next = 0 265 . . v = atomic.Loaduintptr(&l.key) 266 . . }
runtime.unlock
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 850ms (flat, cum) 1.19% 268 . . 269 . . func unlock(l *mutex) { 270 10ms 850ms unlockWithRank(l) unlock2(l) lockrank_off.go:36 ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ 271 . . } 272 . .
runtime.unlock2
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 40ms 40ms (flat, cum) 0.056% 274 . . // 275 . . //go:nowritebarrier 276 20ms 20ms func unlock2(l *mutex) { 277 . . gp := getg() 278 . . 279 . . var prev8 uint8 280 . . var haveStackLock bool 281 . . var endTicks int64 282 20ms 20ms if !mutexSampleContention() { return rate > 0 && cheaprandu64()%rate == 0 lock_spinbit.go:340 283 . . // Not collecting a sample for the contention profile, do the quick release 284 . . prev8 = atomic.Xchg8(key8(&l.key), 0) 285 . . } else { 286 . . // If there's contention, we'll sample it. Don't allow another 287 . . // lock2/unlock2 pair to finish before us and take our blame. Prevent
runtime.unlock2
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 540ms 800ms (flat, cum) 1.12% 313 . . break 314 . . } 315 . . v = atomic.Loaduintptr(&l.key) 316 . . } 317 . . } 318 480ms 480ms if prev8&mutexLocked == 0 { 319 . . throw("unlock of unlocked lock") 320 . . } 321 . . 322 . . if prev8&mutexSleeping != 0 { 323 . 260ms unlock2Wake(l, haveStackLock, endTicks) 324 . . } 325 . . 326 50ms 50ms gp.m.mLockProfile.store() ⋮ if gp := getg(); gp.m.locks/mutexMLocksDelta == 1 && gp.m.mLockProfile.haveStack { mprof.go:748 ⋮ ⋮ if gp := getg(); gp.m.locks/mutexMLocksDelta == 1 && gp.m.mLockProfile.haveStack { mprof.go:748 327 . . gp.m.locks -= mutexMLocksDelta 328 . . if gp.m.locks < 0 { 329 . . throw("runtime·unlock: lock count") 330 . . } 331 10ms 10ms if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack 332 . . gp.stackguard0 = stackPreempt 333 . . } 334 . . } 335 . .
runtime.mutexSampleContention
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 20ms 20ms (flat, cum) 0.028% 336 . . // mutexSampleContention returns whether the current mutex operation should 337 . . // report any contention it discovers. 338 . . func mutexSampleContention() bool { 339 . . rate := atomic.Load64(&mutexprofilerate) 340 20ms 20ms return rate > 0 && cheaprandu64()%rate == 0 341 . . } 342 . . 343 . . // unlock2Wake updates the list of Ms waiting on l, waking an M if necessary. 344 . . // 345 . . //go:nowritebarrier
runtime.unlock2Wake
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 10ms (flat, cum) 0.014% 346 . . func unlock2Wake(l *mutex, haveStackLock bool, endTicks int64) { 347 . . v := atomic.Loaduintptr(&l.key) 348 . . 349 . . // On occasion, seek out and wake the M at the bottom of the stack so it 350 . . // doesn't starve. 351 10ms 10ms antiStarve := cheaprandn(mutexTailWakePeriod) == 0 return uint32((uint64(cheaprand()) * uint64(n)) >> 32) rand.go:321 hi, lo := bits.Mul32(mp.cheaprand, mp.cheaprand^0x74743c1b) rand.go:238 tmp := uint64(x) * uint64(y) bits.go:460 352 . . 353 . . if haveStackLock { 354 . . goto useStackLock 355 . . } 356 . .
runtime.unlock2Wake
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 10ms (flat, cum) 0.014% 422 . . 423 . . mp := mutexWaitListHead(v).ptr() 424 . . wakem := committed 425 . . if committed == nil { 426 . . if v&mutexSpinning == 0 || mutexPreferLowLatency(l) { 427 10ms 10ms wakem = mp 428 . . } 429 . . if antiStarve { 430 . . // Wake the M at the bottom of the stack of waiters. (This is 431 . . // O(N) with the number of waiters.) 432 . . wakem = mp
runtime.unlock2Wake
/usr/lib/go/src/runtime/lock_spinbit.go
Total: 10ms 240ms (flat, cum) 0.34% 454 . . if wakem == mp { 455 . . headM = uintptr(mp.mWaitList.next) &^ mutexMMask 456 . . } 457 . . 458 . . next := headM | flags 459 10ms 10ms if atomic.Casuintptr(&l.key, v, next) { 460 . . if wakem != nil { 461 . . // Claimed an M. Wake it. 462 . 230ms semawakeup(wakem) futexwakeup(&mp.waitsema, 1) lock_futex.go:161 463 . . } 464 . . return 465 . . } 466 . . 467 . . v = atomic.Loaduintptr(&l.key)
git.urbach.dev/cli/q/src/ssa.NewBlock
/home/user/q/src/ssa/Block.go
Total: 10ms 480ms (flat, cum) 0.67% 15 . . Predecessors []*Block 16 . . } 17 . . 18 . . // NewBlock creates a new basic block. 19 . . func NewBlock(label string) *Block { 20 . 220ms return &Block{ 21 10ms 260ms Instructions: make([]Value, 0, 8), 22 . . Label: label, 23 . . } 24 . . } 25 . . 26 . . // AddSuccessor adds the given block as a successor.
git.urbach.dev/cli/q/src/ssa.(*Block).AddSuccessor
/home/user/q/src/ssa/Block.go
Total: 10ms 3.66s (flat, cum) 5.12% 27 . . func (b *Block) AddSuccessor(successor *Block) { 28 . . if slices.Contains(successor.Predecessors, b) { 29 . . return 30 . . } 31 . . 32 10ms 170ms successor.Predecessors = append(successor.Predecessors, b) 33 . 30ms b.copyProtected(successor) maps.Copy(successor.Protected, b.Protected) Block.go:186 dst[k] = v maps.go:64 ⋮ successor.Protected = make(map[Value][]Value, len(b.Protected)) Block.go:183 34 . 3.46s mergeIdentifiers(b, successor) 35 . . } 36 . .
git.urbach.dev/cli/q/src/ssa.(*Block).Append
/home/user/q/src/ssa/Block.go
Total: 80ms 220ms (flat, cum) 0.31% 37 . . // Append adds a new value to the block. 38 . . func (b *Block) Append(value Value) { 39 80ms 220ms b.Instructions = append(b.Instructions, value) 40 . . } 41 . .
git.urbach.dev/cli/q/src/ssa.(*Block).CanReachPredecessor
/home/user/q/src/ssa/Block.go
Total: 30ms 240ms (flat, cum) 0.34% 42 . . // CanReachPredecessor checks if the `other` block appears as a predecessor or is the block itself. 43 . . func (b *Block) CanReachPredecessor(other *Block) bool { 44 30ms 240ms return b.canReachPredecessor(other, make(map[*Block]bool)) 45 . . } 46 . . 47 . . // Contains checks if the value exists within the block. 48 . . func (b *Block) Contains(value Value) bool {
git.urbach.dev/cli/q/src/ssa.(*Block).FindExisting
/home/user/q/src/ssa/Block.go
Total: 840ms 1.29s (flat, cum) 1.80% 49 . . return b.Index(value) != -1 50 . . } 51 . . 52 . . // FindExisting returns an equal instruction that's already appended or `nil` if none could be found. 53 10ms 10ms func (b *Block) FindExisting(instr Value) Value { 54 210ms 420ms if !instr.IsPure() { 55 . . return nil 56 . . } 57 . . 58 620ms 860ms for _, existing := range slices.Backward(b.Instructions) { ⋮ if !yield(i, s[i]) { iter.go:29 if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ ⋮ ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 case *Call, *CallExtern, *CallPointer, *Store, *Syscall, *Cas: Block.go:67 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ switch existing.(type) { Block.go:66 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 for _, existing := range slices.Backward(b.Instructions) { Block.go:58 ⋮ switch existing.(type) { Block.go:66 ⋮ ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ switch existing.(type) { Block.go:66 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ switch existing.(type) { Block.go:66
git.urbach.dev/cli/q/src/ssa.(*Block).FindExisting-range1
/home/user/q/src/ssa/Block.go
Total: 290ms 530ms (flat, cum) 0.74% 59 130ms 370ms if existing.IsPure() && instr.Equals(existing) { 60 . . return existing 61 . . } 62 . . 63 . . // If we encounter an instruction with side effects, 64 . . // we can't be sure that the value is still the same. 65 . . // TODO: This is a bit too conservative. We could check if the instruction affects the value. 66 150ms 150ms switch existing.(type) { 67 10ms 10ms case *Call, *CallExtern, *CallPointer, *Store, *Syscall, *Cas: 68 . . return nil
git.urbach.dev/cli/q/src/ssa.(*Block).FindExisting
/home/user/q/src/ssa/Block.go
Total: 30ms 30ms (flat, cum) 0.042% 69 . . } 70 20ms 20ms } 71 . . 72 10ms 10ms return nil 73 . . } 74 . .
git.urbach.dev/cli/q/src/ssa.(*Block).Index
/home/user/q/src/ssa/Block.go
Total: 430ms 720ms (flat, cum) 1.01% 75 . . // Index returns the position of the value or -1 if it doesn't exist within the block. 76 . . func (b *Block) Index(search Value) int { 77 340ms 340ms for i, value := range b.Instructions { 78 90ms 380ms if value == search { 79 . . return i 80 . . } 81 . . } 82 . .
git.urbach.dev/cli/q/src/ssa.(*Block).InsertAt
/home/user/q/src/ssa/Block.go
Total: 10ms 60ms (flat, cum) 0.084% 84 . . } 85 . . 86 . . // InsertAt inserts the `value` at the given `index`. 87 . . func (b *Block) InsertAt(index int, values ...Value) { 88 10ms 60ms b.Instructions = slices.Insert(b.Instructions, index, values...) 89 . . } 90 . .
git.urbach.dev/cli/q/src/ssa.(*Block).Last
/home/user/q/src/ssa/Block.go
Total: 20ms 20ms (flat, cum) 0.028% 91 . . // Last returns the last value. 92 . . func (b *Block) Last() Value { 93 20ms 20ms if len(b.Instructions) == 0 { 94 . . return nil 95 . . } 96 . . 97 . . return b.Instructions[len(b.Instructions)-1] 98 . . }
git.urbach.dev/cli/q/src/ssa.(*Block).Phis
/home/user/q/src/ssa/Block.go
Total: 100ms 180ms (flat, cum) 0.25% 100 . . // Phis is an iterator for all phis at the top of the block. 101 . . func (b *Block) Phis(yield func(*Phi) bool) { 102 . . for _, instr := range b.Instructions { 103 . . phi, isPhi := instr.(*Phi) 104 . . 105 100ms 180ms if !isPhi || !yield(phi) { oldValue := phi.Arguments[0] compileLoop.go:132 ⋮ for i := 1; i < len(phi.Arguments); i++ { compileLoop.go:163 ⋮ for _, instr := range block.Instructions { compileLoop.go:138 ⋮ ⋮ instr.Replace(oldValue, phi) compileLoop.go:159 ⋮ 106 . . return 107 . . } 108 . . } 109 . . } 110 . .
git.urbach.dev/cli/q/src/ssa.(*Block).Protect
/home/user/q/src/ssa/Block.go
Total: 0 20ms (flat, cum) 0.028% 112 . . func (b *Block) Protect(err Value, protected []Value) { 113 . . if b.Protected == nil { 114 . . b.Protected = make(map[Value][]Value) 115 . . } 116 . . 117 . 20ms b.Protected[err] = protected 118 . . } 119 . . 120 . . // RemoveAt sets the value at the given index to nil.
git.urbach.dev/cli/q/src/ssa.(*Block).RemoveAt
/home/user/q/src/ssa/Block.go
Total: 10ms 40ms (flat, cum) 0.056% 121 . . func (b *Block) RemoveAt(index int) { 122 . . value := b.Instructions[index] 123 . . 124 10ms 10ms for _, input := range value.Inputs() { 125 . 30ms input.RemoveUser(value) 126 . . } 127 . . 128 . . b.Instructions[index] = nil
git.urbach.dev/cli/q/src/ssa.(*Block).RemoveNilValues
/home/user/q/src/ssa/Block.go
Total: 0 160ms (flat, cum) 0.22% 130 . . 131 . . // RemoveNilValues removes all nil values from the block. 132 . . func (b *Block) RemoveNilValues() { 133 . 160ms b.Instructions = slices.DeleteFunc(b.Instructions, func(value Value) bool { 134 . . return value == nil 135 . . }) 136 . . } 137 . . 138 . . // ReplaceAllUses replaces all uses of `old` with `new`.
git.urbach.dev/cli/q/src/ssa.(*Block).canReachPredecessor
/home/user/q/src/ssa/Block.go
Total: 80ms 390ms (flat, cum) 0.55% 151 . . func (b *Block) Unprotect(err Value) { 152 . . delete(b.Protected, err) 153 . . } 154 . . 155 . . // canReachPredecessor checks if the `other` block appears as a predecessor or is the block itself. 156 30ms 30ms func (b *Block) canReachPredecessor(other *Block, traversed map[*Block]bool) bool { 157 . . if other == b { 158 20ms 20ms return true 159 . . } 160 . . 161 10ms 50ms if traversed[b] { 162 . . return false 163 . . } 164 . . 165 . 70ms traversed[b] = true 166 . . 167 10ms 10ms for _, pre := range b.Predecessors { 168 10ms 210ms if pre.canReachPredecessor(other, traversed) { 169 . . return true 170 . . } 171 . . } 172 . . 173 . . return false
git.urbach.dev/cli/q/src/ssa.(*Block).copyProtected
/home/user/q/src/ssa/Block.go
Total: 0 30ms (flat, cum) 0.042% 178 . . if len(b.Protected) == 0 { 179 . . return 180 . . } 181 . . 182 . . if successor.Protected == nil { 183 . 10ms successor.Protected = make(map[Value][]Value, len(b.Protected)) 184 . . } 185 . . 186 . 20ms maps.Copy(successor.Protected, b.Protected) dst[k] = v maps.go:64 187 . . }
runtime.(*stkframe).argMapInternal
/usr/lib/go/src/runtime/stkframe.go
Total: 130ms 130ms (flat, cum) 0.18% 89 . . // this if non-nil, and otherwise fetch the argument map using the 90 . . // current PC. 91 . . // 92 . . // hasReflectStackObj indicates that this frame also has a reflect 93 . . // function stack object, which the caller must synthesize. 94 90ms 90ms func (frame *stkframe) argMapInternal() (argMap bitvector, hasReflectStackObj bool) { 95 . . f := frame.fn 96 30ms 30ms if f.args != abi.ArgsSizeUnknown { 97 . . argMap.n = f.args / goarch.PtrSize 98 10ms 10ms return 99 . . } 100 . . // Extract argument bitmaps for reflect stubs from the calls they made to reflect. 101 . . switch funcname(f) { 102 . . case "reflect.makeFuncStub", "reflect.methodValueCall": 103 . . // These take a *reflect.methodValue as their
runtime.(*stkframe).getStackMap
/usr/lib/go/src/runtime/stkframe.go
Total: 1.22s 3.50s (flat, cum) 4.89% 152 . . return 153 . . } 154 . . 155 . . // getStackMap returns the locals and arguments live pointer maps, and 156 . . // stack object list for frame. 157 20ms 20ms func (frame *stkframe) getStackMap(debug bool) (locals, args bitvector, objs []stackObjectRecord) { 158 . . targetpc := frame.continpc 159 . . if targetpc == 0 { 160 . . // Frame is dead. Return empty bitvectors. 161 . . return 162 . . } 163 . . 164 20ms 20ms f := frame.fn 165 . . pcdata := int32(-1) 166 30ms 50ms if targetpc != f.entry() { return f.datap.textAddr(f.entryOff) symtab.go:901 ⋮ ⋮ 167 . . // Back up to the CALL. If we're at the function entry 168 . . // point, we want to use the entry map (-1), even if 169 . . // the first instruction of the function changes the 170 . . // stack map. 171 10ms 10ms targetpc-- 172 . 2.13s pcdata = pcdatavalue(f, abi.PCDATA_StackMapIndex, targetpc) 173 . . } 174 . . if pcdata == -1 { 175 . . // We do not have a valid pcdata value but there might be a 176 . . // stackmap for this function. It is likely that we are looking 177 . . // at the function prologue, assume so and hope for the best. 178 . . pcdata = 0 179 . . } 180 . . 181 . . // Local variables. 182 40ms 40ms size := frame.varp - frame.sp 183 . . var minsize uintptr 184 . . switch goarch.ArchFamily { 185 . . case goarch.ARM64: 186 . . minsize = sys.StackAlign 187 . . default: 188 . . minsize = sys.MinFrameSize 189 . . } 190 . . if size > minsize { 191 . . stackid := pcdata 192 110ms 110ms stkmap := (*stackmap)(funcdata(f, abi.FUNCDATA_LocalsPointerMaps)) ⋮ raw := base + uintptr(off) symtab.go:1279 ⋮ if i < 0 || i >= f.nfuncdata { symtab.go:1266 193 450ms 450ms if stkmap == nil || stkmap.n <= 0 { 194 . . print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n") 195 . . throw("missing stackmap") 196 . . } 197 . . // If nbit == 0, there's no work to do. 198 10ms 10ms if stkmap.nbit > 0 { 199 10ms 10ms if stackid < 0 || stackid >= stkmap.n { 200 . . // don't know where we are 201 . . print("runtime: pcdata is ", stackid, " and ", stkmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", hex(targetpc), ")\n") 202 . . throw("bad symbol table") 203 . . } 204 20ms 20ms locals = stackmapdata(stkmap, stackid) 205 . . if stackDebug >= 3 && debug { 206 . . print(" locals ", stackid, "/", stkmap.n, " ", locals.n, " words ", locals.bytedata, "\n") 207 . . } 208 . . } else if stackDebug >= 3 && debug { 209 . . print(" no locals to adjust\n") 210 . . } 211 . . } 212 . . 213 . . // Arguments. First fetch frame size and special-case argument maps. 214 50ms 50ms var isReflect bool 215 . 130ms args, isReflect = frame.argMapInternal() 216 10ms 10ms if args.n > 0 && args.bytedata == nil { 217 . . // Non-empty argument frame, but not a special map. 218 . . // Fetch the argument map at pcdata. 219 70ms 70ms stackmap := (*stackmap)(funcdata(f, abi.FUNCDATA_ArgsPointerMaps)) raw := base + uintptr(off) symtab.go:1279 ⋮ if i < 0 || i >= f.nfuncdata { symtab.go:1266 ⋮ p := uintptr(unsafe.Pointer(&f.nfuncdata)) + unsafe.Sizeof(f.nfuncdata) + uintptr(f.npcdata)*4 + uintptr(i)*4 symtab.go:1270 ⋮ 220 170ms 170ms if stackmap == nil || stackmap.n <= 0 { 221 . . print("runtime: frame ", funcname(f), " untyped args ", hex(frame.argp), "+", hex(args.n*goarch.PtrSize), "\n") 222 . . throw("missing stackmap") 223 . . } 224 . . if pcdata < 0 || pcdata >= stackmap.n { 225 . . // don't know where we are 226 . . print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", hex(targetpc), ")\n") 227 . . throw("bad symbol table") 228 . . } 229 10ms 10ms if stackmap.nbit == 0 { 230 . . args.n = 0 231 . . } else { 232 10ms 10ms args = stackmapdata(stackmap, pcdata) return bitvector{stkmap.nbit, addb(&stkmap.bytedata[0], uintptr(n*((stkmap.nbit+7)>>3)))} symtab.go:1337 233 . . } 234 . . } 235 . . 236 . . // stack objects. 237 . . if (GOARCH == "amd64" || GOARCH == "arm64" || GOARCH == "loong64" || GOARCH == "ppc64" || GOARCH == "ppc64le" || GOARCH == "riscv64" || GOARCH == "s390x") && 238 . . unsafe.Sizeof(abi.RegArgs{}) > 0 && isReflect { 239 . . // For reflect.makeFuncStub and reflect.methodValueCall, 240 . . // we need to fake the stack object record. 241 . . // These frames contain an internal/abi.RegArgs at a hard-coded offset. 242 . . // This offset matches the assembly code on amd64 and arm64. 243 . . objs = methodValueCallFrameObjs[:] 244 . . } else { 245 60ms 60ms p := funcdata(f, abi.FUNCDATA_StackObjects) if i < 0 || i >= f.nfuncdata { symtab.go:1266 ⋮ ⋮ ⋮ p := uintptr(unsafe.Pointer(&f.nfuncdata)) + unsafe.Sizeof(f.nfuncdata) + uintptr(f.npcdata)*4 + uintptr(i)*4 symtab.go:1270 ⋮ raw := base + uintptr(off) symtab.go:1279 246 . . if p != nil { 247 . . n := *(*uintptr)(p) 248 10ms 10ms p = add(p, goarch.PtrSize) return unsafe.Pointer(uintptr(p) + x) stubs.go:25 249 . . r0 := (*stackObjectRecord)(noescape(p)) 250 100ms 100ms objs = unsafe.Slice(r0, int(n)) 251 . . // Note: the noescape above is needed to keep 252 . . // getStackMap from "leaking param content: 253 . . // frame". That leak propagates up to getgcmask, then 254 . . // GCMask, then verifyGCInfo, which converts the stack 255 . . // gcinfo tests into heap gcinfo tests :( 256 . . } 257 . . } 258 . . 259 10ms 10ms return 260 . . } 261 . . 262 . . var methodValueCallFrameObjs [1]stackObjectRecord // initialized in stkobjinit 263 . . 264 . . func stkobjinit() {
git.urbach.dev/cli/q/src/codegen.(*Function).findFreeRegister
/home/user/q/src/codegen/findFreeRegister.go
Total: 20ms 20ms (flat, cum) 0.028% 11 . . 12 . . // findFreeRegister finds a free register for the given value. 13 . . func (f *Function) findFreeRegister(step *Step) cpu.Register { 14 . . usedRegisters := bitSet(0) 15 . . 16 10ms 10ms if f.needsFramePointer { 17 . . usedRegisters.Set(f.CPU.FramePointer) 18 . . } 19 . . 20 . . binaryOp, isBinaryOp := step.Value.(*ssa.BinaryOp) 21 . . 22 . . if isBinaryOp && !binaryOp.Op.IsComparison() { 23 10ms 10ms switch f.build.Arch { 24 . . case config.ARM: 25 . . if binaryOp.Op == token.Mod { 26 . . left := f.ValueToStep[binaryOp.Left] 27 . . right := f.ValueToStep[binaryOp.Right] 28 . .
git.urbach.dev/cli/q/src/codegen.(*Function).findFreeRegister
/home/user/q/src/codegen/findFreeRegister.go
Total: 1.30s 1.50s (flat, cum) 2.10% 41 . . usedRegisters.Set(right.Register) 42 . . } 43 . . } 44 . . } 45 . . 46 90ms 90ms for _, current := range f.Steps { 47 . . // These checks need to happen regardless of whether the value is alive after execution. 48 . . // If it is used as an operand, the operand restrictions of the architecture apply. 49 320ms 320ms binaryOp, isBinaryOp := current.Value.(*ssa.BinaryOp) 50 . . 51 50ms 50ms if isBinaryOp && !binaryOp.Op.IsComparison() { ⋮ return k > ___COMPARISONS___ && k < ___END_COMPARISONS___ Kind.go:113 52 . . switch f.build.Arch { 53 . . case config.ARM: 54 . . if current.Register != -1 && binaryOp.Op == token.Mod { 55 . . if binaryOp.Left == step.Value { 56 . . usedRegisters.Set(current.Register) 57 . . } 58 . . 59 . . if binaryOp.Right == step.Value { 60 . . usedRegisters.Set(current.Register) 61 . . } 62 . . } 63 . . case config.X86: 64 . . if current.Register != -1 && binaryOp.Right == step.Value { 65 . . usedRegisters.Set(current.Register) 66 . . } 67 . . } 68 . . 69 10ms 10ms switch binaryOp.Op { 70 . . case token.Div, token.Mod: 71 . . if binaryOp.Right == step.Value { 72 . . for _, reg := range f.CPU.DivisorRestricted { 73 . . usedRegisters.Set(reg) 74 . . } 75 . . } 76 . . case token.Shl, token.Shr: 77 . . if current == step { 78 . . for _, reg := range f.CPU.ShiftRestricted { 79 . . usedRegisters.Set(reg) 80 . . } 81 . . } 82 . . } 83 . . } 84 . . 85 . . // If it's not alive in this step, ignore it. 86 340ms 340ms if !slices.Contains(current.Live, step) { return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ ⋮ return Index(s, v) >= 0 slices.go:118 for i := range s { slices.go:97 ⋮ ⋮ return Index(s, v) >= 0 slices.go:118 ⋮ for i := range s { slices.go:97 ⋮ if v == s[i] { slices.go:98 ⋮ ⋮ 87 . . continue 88 . . } 89 . . 90 . . // Mark all the neighbor registers that are alive 91 . . // at the same time as used. 92 80ms 80ms for _, live := range current.Live { 93 90ms 90ms if live.Register == -1 { 94 . . continue 95 . . } 96 . . 97 190ms 190ms switch instr := live.Value.(type) { 98 . . case *ssa.Field: 99 10ms 10ms _, isFieldFromCall := instr.Tuple.(*ssa.Call) 100 . . 101 . . if isFieldFromCall && live.Index > current.Index { 102 . . usedRegisters.Set(f.CPU.Call.Out[instr.Index]) 103 . . } 104 30ms 30ms case *ssa.Parameter: 105 10ms 10ms if live.Index > current.Index { 106 . . usedRegisters.Set(f.CPU.Call.In[instr.Index]) 107 . . } 108 . . } 109 . . 110 20ms 20ms usedRegisters.Set(live.Register) *b |= 1 << (reg & 63) bitset.go:10 111 . . } 112 . . 113 . . // Ignore the definition itself. 114 . . if current == step { 115 . . continue 116 . . } 117 . . 118 . . // Find all the registers that this instruction 119 . . // would clobber and mark them as used. 120 10ms 210ms for _, reg := range f.clobberedRegisters(current) { 121 . . usedRegisters.Set(reg) 122 . . } 123 . . } 124 . . 125 . . // Pick one of the register hints if possible. 126 10ms 10ms for _, reg := range step.Hints { 127 10ms 10ms if !usedRegisters.Has(reg) { return *b&(1<<(reg&63)) != 0 bitset.go:15 128 . . return reg 129 . . } 130 . . } 131 . . 132 . . // Pick a general purpose register that's not used yet. 133 30ms 30ms for _, reg := range f.CPU.General { 134 . . if !usedRegisters.Has(reg) { 135 . . return reg 136 . . } 137 . . } 138 . .
internal/runtime/maps.newTable
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 30ms 520ms (flat, cum) 0.73% 69 . . // (consider uint8 key, uint64 element). Consider placing all keys 70 . . // together in these cases to save space. 71 . . groups groupsReference 72 . . } 73 . . 74 30ms 30ms func newTable(typ *abi.MapType, capacity uint64, index int, localDepth uint8) *table { 75 . . if capacity < abi.MapGroupSlots { 76 . . capacity = abi.MapGroupSlots 77 . . } 78 . . 79 . 10ms t := &table{ 80 . . index: index, 81 . . localDepth: localDepth, 82 . . } 83 . . 84 . . if capacity > maxTableCapacity { 85 . . panic("initial table capacity too large") 86 . . } 87 . . 88 . . // N.B. group count must be a power of two for probeSeq to visit every 89 . . // group. 90 . . capacity, overflow := alignUpPow2(capacity) 91 . . if overflow { 92 . . panic("rounded-up capacity overflows uint64") 93 . . } 94 . . 95 . 480ms t.reset(typ, uint16(capacity)) 96 . . 97 . . return t 98 . . }
internal/runtime/maps.(*table).reset
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 60ms 480ms (flat, cum) 0.67% 99 . . 100 . . // reset resets the table with new, empty groups with the specified new total 101 . . // capacity. 102 10ms 10ms func (t *table) reset(typ *abi.MapType, capacity uint16) { 103 . . groupCount := uint64(capacity) / abi.MapGroupSlots 104 . 420ms t.groups = newGroups(typ, groupCount) data: newarray(typ.Group, int(length)), group.go:330 105 . . t.capacity = capacity 106 . . t.growthLeft = t.maxGrowthLeft() 107 . . 108 . . for i := uint64(0); i <= t.groups.lengthMask; i++ { 109 . . g := t.groups.group(typ, i) 110 50ms 50ms g.ctrls().setEmpty() *g = ctrlGroup(bitsetEmpty) group.go:148 111 . . } 112 . . } 113 . . 114 . . // maxGrowthLeft is the number of inserts we can do before 115 . . // resizing, starting from an empty table.
internal/runtime/maps.(*table).uncheckedPutSlot
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 50ms 90ms (flat, cum) 0.13% 379 . . // put directly into the map, they do not need to be copied. This 380 . . // requires the caller to ensure that the referenced memory never 381 . . // changes (by sourcing those pointers from another indirect key/elem 382 . . // map). 383 . . func (t *table) uncheckedPutSlot(typ *abi.MapType, hash uintptr, key, elem unsafe.Pointer) { 384 10ms 10ms if t.growthLeft == 0 { 385 . . panic("invariant failed: growthLeft is unexpectedly 0") 386 . . } 387 . . 388 . . // Given key and its hash hash(key), to insert it, we construct a 389 . . // probeSeq, and use it to find the first group with an unoccupied (empty 390 . . // or deleted) slot. We place the key/value into the first such slot in 391 . . // the group and mark it as full with key's H2. 392 . . seq := makeProbeSeq(h1(hash), t.groups.lengthMask) 393 . . for ; ; seq = seq.next() { 394 . . g := t.groups.group(typ, seq.offset) 395 . . 396 20ms 20ms match := g.ctrls().matchEmptyOrDeleted() return ctrlGroupMatchEmptyOrDeleted(g) group.go:201 397 . . if match != 0 { 398 20ms 20ms i := match.first() return bitsetFirst(b) group.go:50 return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 group.go:58 ⋮ 399 . . 400 . . slotKey := g.key(typ, i) 401 . . if typ.IndirectKey() { 402 . . *(*unsafe.Pointer)(slotKey) = key 403 . . } else { 404 . 10ms typedmemmove(typ.Key, slotKey, key) 405 . . } 406 . . 407 . . slotElem := g.elem(typ, i) 408 . . if typ.IndirectElem() { 409 . . *(*unsafe.Pointer)(slotElem) = elem 410 . . } else { 411 . 30ms typedmemmove(typ.Elem, slotElem, elem) 412 . . } 413 . . 414 . . t.growthLeft-- 415 . . t.used++ 416 . . g.ctrls().set(i, ctrl(h2(hash)))
internal/runtime/maps.(*table).uncheckedPutSlotForAssign
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 0 10ms (flat, cum) 0.014% 455 . . if typ.IndirectKey() { 456 . . kmem := newobject(typ.Key) 457 . . *(*unsafe.Pointer)(slotKey) = kmem 458 . . slotKey = kmem 459 . . } 460 . 10ms typedmemmove(typ.Key, slotKey, key) 461 . . 462 . . slotElem := g.elem(typ, i) 463 . . if typ.IndirectElem() { 464 . . emem := newobject(typ.Elem) 465 . . *(*unsafe.Pointer)(slotElem) = emem
internal/runtime/maps.(*Iter).Init
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 160ms 340ms (flat, cum) 0.48% 736 . . // are the group index. 737 . . entryIdx uint64 738 . . } 739 . . 740 . . // Init initializes Iter for iteration. 741 80ms 80ms func (it *Iter) Init(typ *abi.MapType, m *Map) { 742 10ms 10ms it.typ = typ 743 . . 744 20ms 20ms if m == nil || m.used == 0 { 745 . . return 746 . . } 747 . . 748 . . dirIdx := 0 749 . . var groupSmall groupReference 750 . . if m.dirLen <= 0 { 751 . . // Use dirIdx == -1 as sentinel for small maps. 752 . . dirIdx = -1 753 . . groupSmall.data = m.dirPtr 754 . . } 755 . . 756 10ms 10ms it.m = m 757 . 130ms it.entryOffset = rand() 758 10ms 60ms it.dirOffset = rand() 759 30ms 30ms it.globalDepth = m.globalDepth 760 . . it.dirIdx = dirIdx 761 . . it.group = groupSmall 762 . . it.clearSeq = m.clearSeq 763 . . } 764 . .
internal/runtime/maps.(*Iter).Next
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 890ms 890ms (flat, cum) 1.24% 868 . . // 869 . . // The table can be mutated during iteration, though there is no guarantee that 870 . . // the mutations will be visible to the iteration. 871 . . // 872 . . // Init must be called prior to Next. 873 60ms 60ms func (it *Iter) Next() { 874 10ms 10ms if it.m == nil { 875 . . // Map was empty at Iter.Init. 876 10ms 10ms it.key = nil 877 . . it.elem = nil 878 . . return 879 . . } 880 . . 881 60ms 60ms if it.m.writing != 0 { 882 . . fatal("concurrent map iteration and map write") 883 . . return 884 . . } 885 . . 886 10ms 10ms if it.dirIdx < 0 { 887 . . // Map was small at Init. 888 150ms 150ms for ; it.entryIdx < abi.MapGroupSlots; it.entryIdx++ { 889 50ms 50ms k := uintptr(it.entryIdx+it.entryOffset) % abi.MapGroupSlots 890 . . 891 200ms 200ms if (it.group.ctrls().get(k) & ctrlEmpty) == ctrlEmpty { 892 . . // Empty or deleted. 893 . . continue 894 . . } 895 . . 896 110ms 110ms key := it.group.key(it.typ, k) offset := typ.KeysOff + i*typ.KeyStride group.go:299 ⋮ 897 . . if it.typ.IndirectKey() { 898 . . key = *((*unsafe.Pointer)(key)) 899 . . } 900 . . 901 . . // As below, if we have grown to a full map since Init, 902 . . // we continue to use the old group to decide the keys 903 . . // to return, but must look them up again in the new 904 . . // tables. 905 40ms 40ms grown := it.m.dirLen > 0 906 . . var elem unsafe.Pointer 907 . . if grown { 908 . . var ok bool 909 . . newKey, newElem, ok := it.m.getWithKey(it.typ, key) 910 . . if !ok { 911 . . // See comment below. 912 . . if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { 913 . . elem = it.group.elem(it.typ, k) 914 . . if it.typ.IndirectElem() { 915 . . elem = *((*unsafe.Pointer)(elem)) 916 . . } 917 . . } else { 918 . . continue 919 . . } 920 . . } else { 921 . . key = newKey 922 . . elem = newElem 923 . . } 924 . . } else { 925 10ms 10ms elem = it.group.elem(it.typ, k) offset := typ.ElemsOff + i*typ.ElemStride group.go:306 926 . . if it.typ.IndirectElem() { 927 . . elem = *((*unsafe.Pointer)(elem)) 928 . . } 929 . . } 930 . . 931 . . it.entryIdx++ 932 40ms 40ms it.key = key 933 . . it.elem = elem 934 30ms 30ms return 935 . . } 936 100ms 100ms it.key = nil 937 . . it.elem = nil 938 10ms 10ms return 939 . . } 940 . . 941 . . if it.globalDepth != it.m.globalDepth { 942 . . // Directory has grown since the last call to Next. Adjust our 943 . . // directory index.
internal/runtime/maps.(*Iter).Next
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 30ms 30ms (flat, cum) 0.042% 975 . . 976 . . it.globalDepth = it.m.globalDepth 977 . . } 978 . . 979 . . // Continue iteration until we find a full slot. 980 10ms 10ms for ; it.dirIdx < it.m.dirLen; it.nextDirIdx() { 981 . . // Resolve the table. 982 10ms 10ms if it.tab == nil { 983 . . dirIdx := int((uint64(it.dirIdx) + it.dirOffset) & uint64(it.m.dirLen-1)) 984 10ms 10ms newTab := it.m.directoryAt(uintptr(dirIdx)) 985 . . if newTab.index != dirIdx { 986 . . // Normally we skip past all duplicates of the 987 . . // same entry in the table (see updates to 988 . . // it.dirIdx at the end of the loop below), so 989 . . // this case wouldn't occur.
internal/runtime/maps.(*Iter).Next
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 20ms 20ms (flat, cum) 0.028% 1031 . . // not be zero due to entryOffset). 1032 . . groupIdx := entryIdx >> abi.MapGroupSlotsBits 1033 . . it.group = it.tab.groups.group(it.typ, groupIdx) 1034 . . } 1035 . . 1036 10ms 10ms if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == 0 { 1037 . . // Slot full. 1038 . . 1039 10ms 10ms key := it.group.key(it.typ, slotIdx) offset := typ.KeysOff + i*typ.KeyStride group.go:299 1040 . . if it.typ.IndirectKey() { 1041 . . key = *((*unsafe.Pointer)(key)) 1042 . . } 1043 . . 1044 . . grown := it.tab.index == -1
internal/runtime/maps.(*Iter).Next
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 30ms 30ms (flat, cum) 0.042% 1059 . . if it.typ.IndirectElem() { 1060 . . elem = *((*unsafe.Pointer)(elem)) 1061 . . } 1062 . . } 1063 . . 1064 10ms 10ms it.entryIdx++ 1065 10ms 10ms it.key = key 1066 . . it.elem = elem 1067 10ms 10ms return 1068 . . } 1069 . . 1070 . . next: 1071 . . it.entryIdx++ 1072 . .
internal/runtime/maps.(*Iter).Next
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 10ms 10ms (flat, cum) 0.014% 1183 . . // this group. Continue 1184 . . // to next group. 1185 . . it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) 1186 . . } else { 1187 . . // Next full slot. 1188 10ms 10ms i := groupMatch.first() return bitsetFirst(b) group.go:50 return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 group.go:58 1189 . . it.entryIdx += uint64(i - slotIdx) 1190 . . } 1191 . . 1192 . . it.key = key 1193 . . it.elem = elem
internal/runtime/maps.(*table).rehash
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 0 290ms (flat, cum) 0.41% 1221 . . // new allocation, so the existing grow support in iteration would 1222 . . // continue to work. 1223 . . 1224 . . newCapacity := 2 * t.capacity 1225 . . if newCapacity <= maxTableCapacity { 1226 . 290ms t.grow(typ, m, newCapacity) 1227 . . return 1228 . . } 1229 . . 1230 . . t.split(typ, m) 1231 . . }
internal/runtime/maps.(*table).grow
/usr/lib/go/src/internal/runtime/maps/table.go
Total: 20ms 290ms (flat, cum) 0.41% 1286 . . // grow the capacity of the table by allocating a new table with a bigger array 1287 . . // and uncheckedPutting each element of the table into the new table (we know 1288 . . // that no insertion here will Put an already-present value), and discard the 1289 . . // old table. 1290 . . func (t *table) grow(typ *abi.MapType, m *Map, newCapacity uint16) { 1291 . 130ms newTable := newTable(typ, uint64(newCapacity), t.index, t.localDepth) 1292 . . 1293 . . if t.capacity > 0 { 1294 . . for i := uint64(0); i <= t.groups.lengthMask; i++ { 1295 . . g := t.groups.group(typ, i) 1296 10ms 10ms for j := uintptr(0); j < abi.MapGroupSlots; j++ { 1297 . . if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { 1298 . . // Empty or deleted 1299 . . continue 1300 . . } 1301 . . 1302 . . key := g.key(typ, j) 1303 . . if typ.IndirectKey() { 1304 . . key = *((*unsafe.Pointer)(key)) 1305 . . } 1306 . . 1307 . . elem := g.elem(typ, j) 1308 . . if typ.IndirectElem() { 1309 . . elem = *((*unsafe.Pointer)(elem)) 1310 . . } 1311 . . 1312 10ms 80ms hash := typ.Hasher(key, m.seed) 1313 . . 1314 . 70ms newTable.uncheckedPutSlot(typ, hash, key, elem) 1315 . . } 1316 . . } 1317 . . } 1318 . . 1319 . . newTable.checkInvariants(typ, m)
runtime.makeslicecopy
/usr/lib/go/src/runtime/slice.go
Total: 30ms 2.28s (flat, cum) 3.19% 34 . . panic(errorString("makeslice: cap out of range")) 35 . . } 36 . . 37 . . // makeslicecopy allocates a slice of "tolen" elements of type "et", 38 . . // then copies "fromlen" elements of type "et" into that new allocation from "from". 39 10ms 100ms func makeslicecopy(et *_type, tolen int, fromlen int, from unsafe.Pointer) unsafe.Pointer { 40 . . var tomem, copymem uintptr 41 . . if uintptr(tolen) > uintptr(fromlen) { 42 . . var overflow bool 43 . . tomem, overflow = math.MulUintptr(et.Size_, uintptr(tolen)) 44 . . if overflow || tomem > maxAlloc || tolen < 0 { 45 . . panicmakeslicelen() 46 . . } 47 . . copymem = et.Size_ * uintptr(fromlen) 48 . . } else { 49 . . // fromlen is a known good length providing and equal or greater than tolen, 50 . . // thereby making tolen a good slice length too as from and to slices have the 51 . . // same element width. 52 . . tomem = et.Size_ * uintptr(tolen) 53 . . copymem = tomem 54 . . } 55 . . 56 . . var to unsafe.Pointer 57 . . if !et.Pointers() { 58 . 2.14s to = mallocgc(tomem, nil, false) 59 20ms 20ms if copymem < tomem { 60 . 20ms memclrNoHeapPointers(add(to, copymem), tomem-copymem) 61 . . } 62 . . } else { 63 . . // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. 64 . . to = mallocgc(tomem, et, true) 65 . . if copymem > 0 && writeBarrier.enabled {
runtime.makeslicecopy
/usr/lib/go/src/runtime/slice.go
Total: 0 20ms (flat, cum) 0.028% 83 . . } 84 . . if asanenabled { 85 . . asanread(from, copymem) 86 . . } 87 . . 88 . 20ms memmove(to, from, copymem) 89 . . 90 . . return to 91 . . } 92 . . 93 . . // makeslice should be an internal detail,
runtime.makeslice
/usr/lib/go/src/runtime/slice.go
Total: 370ms 4s (flat, cum) 5.59% 97 . . // 98 . . // Do not remove or change the type signature. 99 . . // See go.dev/issue/67401. 100 . . // 101 . . //go:linkname makeslice 102 50ms 370ms func makeslice(et *_type, len, cap int) unsafe.Pointer { 103 100ms 100ms mem, overflow := math.MulUintptr(et.Size_, uintptr(cap)) 104 180ms 180ms if overflow || mem > maxAlloc || len < 0 || len > cap { 105 . . // NOTE: Produce a 'len out of range' error instead of a 106 . . // 'cap out of range' error when someone does make([]T, bignumber). 107 . . // 'cap out of range' is true too, but since the cap is only being 108 . . // supplied implicitly, saying len is clearer. 109 . . // See golang.org/issue/4085. 110 . . mem, overflow := math.MulUintptr(et.Size_, uintptr(len)) 111 . . if overflow || mem > maxAlloc || len < 0 { 112 . . panicmakeslicelen() 113 . . } 114 . . panicmakeslicecap() 115 . . } 116 . . 117 40ms 3.35s return mallocgc(mem, et, true) 118 . . } 119 . . 120 . . func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer { 121 . . len := int(len64) 122 . . if int64(len) != len64 {
runtime.growslice
/usr/lib/go/src/runtime/slice.go
Total: 610ms 610ms (flat, cum) 0.85% 173 . . // 174 . . // Do not remove or change the type signature. 175 . . // See go.dev/issue/67401. 176 . . // 177 . . //go:linkname growslice 178 60ms 60ms func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type) slice { 179 . . oldLen := newLen - num 180 . . if raceenabled { 181 . . callerpc := sys.GetCallerPC() 182 . . racereadrangepc(oldPtr, uintptr(oldLen*int(et.Size_)), callerpc, abi.FuncPCABIInternal(growslice)) 183 . . } 184 . . if msanenabled { 185 . . msanread(oldPtr, uintptr(oldLen*int(et.Size_))) 186 . . } 187 . . if asanenabled { 188 . . asanread(oldPtr, uintptr(oldLen*int(et.Size_))) 189 . . } 190 . . 191 70ms 70ms if newLen < 0 { 192 . . panic(errorString("growslice: len out of range")) 193 . . } 194 . . 195 60ms 60ms if et.Size_ == 0 { 196 . . // append should not create a slice with nil pointer but non-zero len. 197 . . // We assume that append doesn't need to preserve oldPtr in this case. 198 . . return slice{unsafe.Pointer(&zerobase), newLen, newLen} 199 . . } 200 . . 201 20ms 20ms newcap := nextslicecap(newLen, oldCap) if newLen > doublecap { slice.go:329 ⋮ 202 . . 203 . . var overflow bool 204 . . var lenmem, newlenmem, capmem uintptr 205 . . // Specialize for common values of et.Size. 206 . . // For 1 we don't need any division/multiplication. 207 . . // For goarch.PtrSize, compiler will optimize division/multiplication into a shift by a constant. 208 . . // For powers of 2, use a variable shift. 209 . . noscan := !et.Pointers() 210 . . switch { 211 50ms 50ms case et.Size_ == 1: 212 . . lenmem = uintptr(oldLen) 213 . . newlenmem = uintptr(newLen) 214 40ms 40ms capmem = roundupsize(uintptr(newcap), noscan) return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 return (n + a - 1) / a stubs.go:400 ⋮ ⋮ 215 10ms 10ms overflow = uintptr(newcap) > maxAlloc 216 . . newcap = int(capmem) 217 30ms 30ms case et.Size_ == goarch.PtrSize: 218 . . lenmem = uintptr(oldLen) * goarch.PtrSize 219 . . newlenmem = uintptr(newLen) * goarch.PtrSize 220 70ms 70ms capmem = roundupsize(uintptr(newcap)*goarch.PtrSize, noscan) if reqSize <= gc.SmallSizeMax-8 { msize.go:25 ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 ⋮ return (n + a - 1) / a stubs.go:400 ⋮ if !noscan && reqSize > gc.MinSizeForMallocHeader { // !noscan && !heapBitsInSpan(reqSize) msize.go:20 ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 221 10ms 10ms overflow = uintptr(newcap) > maxAlloc/goarch.PtrSize 222 . . newcap = int(capmem / goarch.PtrSize) 223 10ms 10ms case isPowerOfTwo(et.Size_): return x&(x-1) == 0 slice.go:388 224 . . var shift uintptr 225 . . if goarch.PtrSize == 8 { 226 . . // Mask shift for better code generation. 227 20ms 20ms shift = uintptr(sys.TrailingZeros64(uint64(et.Size_))) & 63 228 . . } else { 229 . . shift = uintptr(sys.TrailingZeros32(uint32(et.Size_))) & 31 230 . . } 231 . . lenmem = uintptr(oldLen) << shift 232 . . newlenmem = uintptr(newLen) << shift 233 160ms 160ms capmem = roundupsize(uintptr(newcap)<<shift, noscan) return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass128[divRoundUp(reqSize-gc.SmallSizeMax, gc.LargeSizeDiv)]]) - (reqSize - size) msize.go:28 ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 return (n + a - 1) / a stubs.go:400 ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass128[divRoundUp(reqSize-gc.SmallSizeMax, gc.LargeSizeDiv)]]) - (reqSize - size) msize.go:28 return (n + a - 1) / a stubs.go:400 ⋮ ⋮ if !noscan && reqSize > gc.MinSizeForMallocHeader { // !noscan && !heapBitsInSpan(reqSize) msize.go:20 ⋮ if reqSize <= gc.SmallSizeMax-8 { msize.go:25 ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 ⋮ ⋮ 234 . . overflow = uintptr(newcap) > (maxAlloc >> shift) 235 . . newcap = int(capmem >> shift) 236 . . capmem = uintptr(newcap) << shift 237 . . default: 238 . . lenmem = uintptr(oldLen) * et.Size_
runtime.growslice
/usr/lib/go/src/runtime/slice.go
Total: 160ms 1.73s (flat, cum) 2.42% 254 . . // 255 . . // func main() { 256 . . // s = append(s, d, d, d, d) 257 . . // print(len(s), "\n") 258 . . // } 259 10ms 10ms if overflow || capmem > maxAlloc { 260 . . panic(errorString("growslice: len out of range")) 261 . . } 262 . . 263 . . var p unsafe.Pointer 264 30ms 30ms if !et.Pointers() { 265 . 130ms p = mallocgc(capmem, nil, false) 266 . . // The append() that calls growslice is going to overwrite from oldLen to newLen. 267 . . // Only clear the part that will not be overwritten. 268 . . // The reflect_growslice() that calls growslice will manually clear 269 . . // the region not cleared here. 270 10ms 20ms memclrNoHeapPointers(add(p, newlenmem), capmem-newlenmem) 271 . . } else { 272 . . // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. 273 20ms 1.33s p = mallocgc(capmem, et, true) 274 30ms 30ms if lenmem > 0 && writeBarrier.enabled { 275 . . // Only shade the pointers in oldPtr since we know the destination slice p 276 . . // only contains nil pointers because it has been cleared during alloc. 277 . . // 278 . . // It's safe to pass a type to this function as an optimization because 279 . . // from and to only ever refer to memory representing whole values of 280 . . // type et. See the comment on bulkBarrierPreWrite. 281 . . bulkBarrierPreWriteSrcOnly(uintptr(p), uintptr(oldPtr), lenmem-et.Size_+et.PtrBytes, et) 282 . . } 283 . . } 284 40ms 160ms memmove(p, oldPtr, lenmem) 285 . . 286 20ms 20ms return slice{p, newLen, newcap} 287 . . } 288 . . 289 . . // growsliceNoAlias is like growslice but only for the case where 290 . . // we know that oldPtr is not aliased. 291 . . //
runtime.nextslicecap
/usr/lib/go/src/runtime/slice.go
Total: 10ms 10ms (flat, cum) 0.014% 324 . . 325 . . // nextslicecap computes the next appropriate slice length. 326 . . func nextslicecap(newLen, oldCap int) int { 327 . . newcap := oldCap 328 . . doublecap := newcap + newcap 329 10ms 10ms if newLen > doublecap { 330 . . return newLen 331 . . } 332 . . 333 . . const threshold = 256 334 . . if oldCap < threshold {
runtime.isPowerOfTwo
/usr/lib/go/src/runtime/slice.go
Total: 10ms 10ms (flat, cum) 0.014% 383 . . new.len = old.len // preserve the old length 384 . . return new 385 . . } 386 . . 387 . . func isPowerOfTwo(x uintptr) bool { 388 10ms 10ms return x&(x-1) == 0 389 . . } 390 . . 391 . . // slicecopy is used to copy from a string or slice of pointerless elements into a slice. 392 . . func slicecopy(toPtr unsafe.Pointer, toLen int, fromPtr unsafe.Pointer, fromLen int, width uintptr) int { 393 . . if fromLen == 0 || toLen == 0 {
internal/bytealg.MakeNoZero
/usr/lib/go/src/runtime/slice.go
Total: 60ms 240ms (flat, cum) 0.34% 431 . . //go:linkname bytealg_MakeNoZero internal/bytealg.MakeNoZero 432 . . func bytealg_MakeNoZero(len int) []byte { 433 . . if uintptr(len) > maxAlloc { 434 . . panicmakeslicelen() 435 . . } 436 40ms 40ms cap := roundupsize(uintptr(len), true) if reqSize <= maxSmallSize-gc.MallocHeaderSize { msize.go:18 ⋮ ⋮ return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) msize.go:26 ⋮ 437 20ms 200ms return unsafe.Slice((*byte)(mallocgc(cap, nil, false)), cap)[:len] 438 . . } 439 . . 440 . . // moveSlice copies the input slice to the heap and returns it. 441 . . // et is the element type of the slice. 442 . . func moveSlice(et *_type, old unsafe.Pointer, len, cap int) (unsafe.Pointer, int, int) {
runtime.moveSliceNoCapNoScan
/usr/lib/go/src/runtime/slice.go
Total: 60ms 190ms (flat, cum) 0.27% 486 . . memmove(new, old, lenmem) 487 . . return new, len, int(cap) 488 . . } 489 . . 490 . . // moveSliceNoCapNoScan is a combination of moveSliceNoScan and moveSliceNoCap. 491 60ms 160ms func moveSliceNoCapNoScan(elemSize uintptr, old unsafe.Pointer, len int) (unsafe.Pointer, int, int) { 492 . . if len == 0 { 493 . . if old != nil { 494 . . old = unsafe.Pointer(&zerobase) 495 . . } 496 . . return old, 0, 0 497 . . } 498 . . lenmem := uintptr(len) * elemSize 499 . . capmem := roundupsize(lenmem, true) 500 . 20ms new := mallocgc(capmem, nil, false) 501 . 10ms memmove(new, old, lenmem) 502 . . if capmem > lenmem { 503 . . memclrNoHeapPointers(add(new, lenmem), capmem-lenmem) 504 . . } 505 . . return new, len, int(capmem / elemSize) 506 . . }
slices.Backward[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }].func1
/usr/lib/go/src/slices/iter.go
Total: 120ms 120ms (flat, cum) 0.17% 23 . . 24 . . // Backward returns an iterator over index-value pairs in the slice, 25 . . // traversing it backward with descending indices. 26 . . func Backward[Slice ~[]E, E any](s Slice) iter.Seq2[int, E] { 27 . . return func(yield func(int, E) bool) { 28 120ms 120ms for i := len(s) - 1; i >= 0; i-- {
slices.Backward[go.shape.[]*git.urbach.dev/cli/q/src/codegen.Step,go.shape.*uint8].func1
/usr/lib/go/src/slices/iter.go
Total: 1.24s 10.10s (flat, cum) 14.12% 29 1.24s 10.10s if !yield(i, s[i]) { if step.Register == -1 && f.needsRegister(step) { CompileToAssembly.go:24 ⋮ switch instr.(type) { Reorder.go:14 ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ end := block.Index(firstUser) Reorder.go:27 if value == search { Block.go:78 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ } removeDeadCode.go:23 ⋮ ⋮ ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ f.createLiveRanges(step) CompileToAssembly.go:20 ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 ⋮ if len(errors) > 0 { removeDeadCode.go:25 ⋮ ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ switch instr.(type) { Reorder.go:14 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ set.BringToBack(block.Instructions[start:end], 0) Reorder.go:37 slice[i] = slice[i+1] BringToBack.go:12 ⋮ } removeDeadCode.go:23 ⋮ ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ end := block.Index(firstUser) Reorder.go:27 for i, value := range b.Instructions { Block.go:77 ⋮ ⋮ for end > start && slices.Contains(block.Instructions[end-1].Users(), firstUser) { Reorder.go:33 ⋮ if live.Value == current.Value { markAlive.go:49 ⋮ ⋮ case *ssa.Bytes, *ssa.Global, *ssa.Int: Reorder.go:15 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ case *Call, *CallExtern, *CallPointer, *Store, *Syscall, *Cas: Block.go:67 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ switch instr.(type) { Reorder.go:14 ⋮ end := block.Index(firstUser) Reorder.go:27 for i, value := range b.Instructions { Block.go:77 ⋮ switch existing.(type) { Block.go:66 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ f.hintABI(step) CompileToAssembly.go:19 ⋮ } removeDeadCode.go:23 ⋮ f.hintDestination(step) CompileToAssembly.go:28 ⋮ ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ switch instr.(type) { Reorder.go:14 ⋮ set.BringToBack(block.Instructions[start:end], 0) Reorder.go:37 ⋮ value, err := f.evaluateRight(expr) evaluateAll.go:15 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ for _, existing := range slices.Backward(b.Instructions) { Block.go:58 ⋮ switch existing.(type) { Block.go:66 ⋮ block.RemoveNilValues() removeDeadCode.go:29 ⋮ switch existing.(type) { Block.go:66 ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 ⋮ return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ end := block.Index(firstUser) Reorder.go:27 for i, value := range b.Instructions { Block.go:77 ⋮ ⋮ if step.Register == -1 && f.needsRegister(step) { CompileToAssembly.go:24 ⋮ ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ end := block.Index(firstUser) Reorder.go:27 for i, value := range b.Instructions { Block.go:77 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ switch existing.(type) { Block.go:66 ⋮ } markAlive.go:58 ⋮ if existing.IsPure() && instr.Equals(existing) { Block.go:59 ⋮ for _, block := range slices.Backward(f.Blocks) { removeDeadCode.go:14 ⋮ switch existing.(type) { Block.go:66 ⋮ ⋮ ⋮ f.assignFreeRegister(step) CompileToAssembly.go:25 step.Register = f.findFreeRegister(step) assignFreeRegister.go:6 ⋮ if live.Value == current.Value { markAlive.go:49 ⋮ for _, current := range slices.Backward(steps) { markAlive.go:38 ⋮ if live.Value == current.Value { markAlive.go:49 30 . . return 31 . . } 32 . . } 33 . . } 34 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).AddBlock
/home/user/q/src/ssa/IR.go
Total: 20ms 110ms (flat, cum) 0.15% 5 . . Blocks []*Block 6 . . } 7 . . 8 . . // AddBlock adds a new block to the function. 9 . . func (ir *IR) AddBlock(block *Block) { 10 20ms 110ms ir.Blocks = append(ir.Blocks, block) 11 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).Append
/home/user/q/src/ssa/IR.go
Total: 120ms 1.47s (flat, cum) 2.06% 13 . . // Append adds a new value to the last block. 14 20ms 20ms func (ir *IR) Append(instr Value) Value { 15 20ms 1.24s existing := ir.Block().FindExisting(instr) return ir.Blocks[len(ir.Blocks)-1] IR.go:27 ⋮ ⋮ 16 . . 17 . . if existing != nil { 18 . . return existing 19 . . } 20 . . 21 80ms 210ms ir.Block().Append(instr) b.Instructions = append(b.Instructions, value) Block.go:39 ⋮ ⋮ ⋮ ⋮ ⋮ 22 . . return instr 23 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).Block
/home/user/q/src/ssa/IR.go
Total: 60ms 60ms (flat, cum) 0.084% 25 . . // Block returns the last block. 26 . . func (ir *IR) Block() *Block { 27 60ms 60ms return ir.Blocks[len(ir.Blocks)-1] 28 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).ComputeUsers
/home/user/q/src/ssa/IR.go
Total: 600ms 1.69s (flat, cum) 2.36% 30 . . // ComputeUsers creates the list of users for each value. 31 10ms 10ms func (ir *IR) ComputeUsers() { 32 20ms 20ms for _, block := range ir.Blocks { 33 60ms 60ms for _, value := range block.Instructions { 34 110ms 470ms for _, input := range value.Inputs() { 35 390ms 1.12s input.AddUser(value) 36 . . } 37 . . } 38 . . } 39 10ms 10ms } 40 . . 41 . . // CountValues returns the total number of values.
git.urbach.dev/cli/q/src/ssa.(*IR).CountValues
/home/user/q/src/ssa/IR.go
Total: 40ms 40ms (flat, cum) 0.056% 43 . . count := 0 44 . . 45 10ms 10ms for _, block := range ir.Blocks { 46 30ms 30ms count += len(block.Instructions) 47 . . } 48 . . 49 . . return count 50 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).ExitBlocks
/home/user/q/src/ssa/IR.go
Total: 50ms 190ms (flat, cum) 0.27% 51 . . 52 . . // ExitBlocks is an iterator for all exit blocks. 53 . . func (ir *IR) ExitBlocks(yield func(*Block) bool) { 54 . . for _, block := range ir.Blocks { 55 20ms 20ms _, returns := block.Last().(*Return) if len(b.Instructions) == 0 { Block.go:93 56 . . 57 . . if !returns { 58 . . continue 59 . . } 60 . . 61 30ms 170ms if !yield(block) { resource, isResource := value.Type().(*types.Resource) verifyDeallocation.go:23 ⋮ ⋮ for _, value := range exitBlock.Identifiers.After { verifyDeallocation.go:12 ⋮ ⋮ if value == nil { verifyDeallocation.go:13 ⋮ for _, value := range exitBlock.Identifiers.After { verifyDeallocation.go:12 62 . . return 63 . . } 64 . . }
git.urbach.dev/cli/q/src/ssa.(*IR).IsIdentified
/home/user/q/src/ssa/IR.go
Total: 220ms 540ms (flat, cum) 0.75% 66 . . 67 . . // IsIdentified returns true if the value can be obtained from one of the identifiers. 68 . . func (ir *IR) IsIdentified(value Value) bool { 69 30ms 30ms for _, block := range ir.Blocks { 70 190ms 510ms if block.IsIdentified(value) { for _, existing := range i.After { Identifiers.go:52 ⋮ ⋮ if existing == value { Identifiers.go:53 ⋮ ⋮ ⋮ ⋮ for _, existing := range i.After { Identifiers.go:52 ⋮ ⋮ ⋮ 71 . . return true 72 . . } 73 . . } 74 . .
git.urbach.dev/cli/q/src/ssa.(*IR).ReplaceAll
/home/user/q/src/ssa/IR.go
Total: 80ms 120ms (flat, cum) 0.17% 76 . . } 77 . . 78 . . // ReplaceAll replaces all occurrences of the given `old` value with the `new` value. 79 . . func (ir *IR) ReplaceAll(old Value, new Value) { 80 10ms 10ms for _, block := range ir.Blocks { 81 10ms 10ms for _, value := range block.Instructions { 82 60ms 100ms value.Replace(old, new) 83 . . } 84 . . } 85 . . }
runtime.memclrNoHeapPointers
/usr/lib/go/src/runtime/memclr_arm64.s
Total: 180ms 180ms (flat, cum) 0.25% 7 . . // See memclrNoHeapPointers Go doc for important implementation constraints. 8 . . 9 . . // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 10 . . // Also called from assembly in sys_windows_arm64.s without g (but using Go stack convention). 11 . . TEXT runtime·memclrNoHeapPointers<ABIInternal>(SB),NOSPLIT,$0-16 12 10ms 10ms CMP $16, R1 13 . . // If n is equal to 16 bytes, use zero_exact_16 to zero 14 . . BEQ zero_exact_16 15 . . 16 . . // If n is greater than 16 bytes, use zero_by_16 to zero 17 . . BHI zero_by_16 18 . . 19 . . // n is less than 16 bytes 20 10ms 10ms ADD R1, R0, R7 21 10ms 10ms TBZ $3, R1, less_than_8 22 . . MOVD ZR, (R0) 23 . . MOVD ZR, -8(R7) 24 . . RET 25 . . 26 . . less_than_8: 27 . . TBZ $2, R1, less_than_4 28 . . MOVW ZR, (R0) 29 . . MOVW ZR, -4(R7) 30 . . RET 31 . . 32 . . less_than_4: 33 . . CBZ R1, ending 34 10ms 10ms MOVB ZR, (R0) 35 10ms 10ms TBZ $1, R1, ending 36 . . MOVH ZR, -2(R7) 37 . . 38 . . ending: 39 . . RET 40 . . 41 . . zero_exact_16: 42 . . // n is exactly 16 bytes 43 20ms 20ms STP (ZR, ZR), (R0) 44 . . RET 45 . . 46 . . zero_by_16: 47 . . // n greater than 16 bytes, check if the start address is aligned 48 . . NEG R0, R4 49 . . ANDS $15, R4, R4 50 . . // Try zeroing using zva if the start address is aligned with 16 51 . . BEQ try_zva 52 . . 53 . . // Non-aligned store 54 . . STP (ZR, ZR), (R0) 55 . . // Make the destination aligned 56 . . SUB R4, R1, R1 57 . . ADD R4, R0, R0 58 . . B try_zva 59 . . 60 . . tail_maybe_long: 61 . . CMP $64, R1 62 10ms 10ms BHS no_zva 63 . . 64 . . tail63: 65 . . ANDS $48, R1, R3 66 . . BEQ last16 67 10ms 10ms CMPW $32, R3 68 . . BEQ last48 69 . . BLT last32 70 . . STP.P (ZR, ZR), 16(R0) 71 . . last48: 72 . . STP.P (ZR, ZR), 16(R0) 73 . . last32: 74 20ms 20ms STP.P (ZR, ZR), 16(R0) 75 . . // The last store length is at most 16, so it is safe to use 76 . . // stp to write last 16 bytes 77 . . last16: 78 . . ANDS $15, R1, R1 79 . . CBZ R1, last_end 80 . . ADD R1, R0, R0 81 . . STP (ZR, ZR), -16(R0) 82 . . last_end: 83 . . RET 84 . . 85 . . PCALIGN $16 86 . . no_zva: 87 . . SUB $16, R0, R0 88 . . SUB $64, R1, R1 89 . . 90 . . loop_64: 91 . . STP (ZR, ZR), 16(R0) 92 30ms 30ms STP (ZR, ZR), 32(R0) 93 . . STP (ZR, ZR), 48(R0) 94 . . STP.W (ZR, ZR), 64(R0) 95 . . SUBS $64, R1, R1 96 . . BGE loop_64 97 . . ANDS $63, R1, ZR 98 . . ADD $16, R0, R0 99 . . BNE tail63 100 . . RET 101 . . 102 . . PCALIGN $16 103 . . try_zva: 104 . . // Try using the ZVA feature to zero entire cache lines 105 . . // It is not meaningful to use ZVA if the block size is less than 64, 106 . . // so make sure that n is greater than or equal to 64 107 20ms 20ms CMP $63, R1 108 . . BLE tail63 109 . . 110 . . CMP $128, R1 111 . . // Ensure n is at least 128 bytes, so that there is enough to copy after 112 . . // alignment. 113 . . BLT no_zva 114 . . // Check if ZVA is allowed from user code, and if so get the block size 115 . . MOVW block_size<>(SB), R5 116 10ms 10ms TBNZ $31, R5, no_zva 117 10ms 10ms CBNZ R5, zero_by_line 118 . . // DCZID_EL0 bit assignments 119 . . // [63:5] Reserved 120 . . // [4] DZP, if bit set DC ZVA instruction is prohibited, else permitted 121 . . // [3:0] log2 of the block size in words, eg. if it returns 0x4 then block size is 16 words 122 . . MRS DCZID_EL0, R3
runtime.memclrNoHeapPointers
/usr/lib/go/src/runtime/memclr_arm64.s
Total: 1s 1s (flat, cum) 1.40% 160 . . // We now have at least 64 bytes to zero, update n 161 . . MOVD R3, R1 162 . . 163 . . loop_zva_prolog: 164 . . STP (ZR, ZR), (R0) 165 100ms 100ms STP (ZR, ZR), 16(R0) 166 . . STP (ZR, ZR), 32(R0) 167 . . SUBS $64, R4, R4 168 . . STP (ZR, ZR), 48(R0) 169 . . ADD $64, R0, R0 170 . . BGE loop_zva_prolog 171 . . 172 . . ADD R4, R0, R0 173 . . 174 . . aligned: 175 10ms 10ms SUB R5, R1, R1 176 . . 177 . . PCALIGN $16 178 . . loop_zva: 179 470ms 470ms DC ZVA, R0 180 400ms 400ms ADD R5, R0, R0 181 . . SUBS R5, R1, R1 182 . . BHS loop_zva 183 . . ANDS R6, R1, R1 184 10ms 10ms BNE tail_maybe_long 185 10ms 10ms RET 186 . . 187 . . GLOBL block_size<>(SB), NOPTR, $8
runtime.(*mSpanStateBox).get
/usr/lib/go/src/runtime/mheap.go
Total: 90ms 90ms (flat, cum) 0.13% 414 . . // It is nosplit because it's called indirectly by typedmemclr, 415 . . // which must not be preempted. 416 . . 417 . . //go:nosplit 418 . . func (b *mSpanStateBox) get() mSpanState { 419 90ms 90ms return mSpanState(b.s.Load()) return Load8(&u.value) types.go:124 420 . . } 421 . . 422 . . type mspan struct { 423 . . _ sys.NotInHeap 424 . . next *mspan // next span in list, or nil if none
runtime.(*mspan).base
/usr/lib/go/src/runtime/mheap.go
Total: 90ms 90ms (flat, cum) 0.13% 514 . . userArenaChunkFree addrRange // interval for managing chunk allocation 515 . . largeType *_type // malloc header for large objects. 516 . . } 517 . . 518 . . func (s *mspan) base() uintptr { 519 90ms 90ms return s.startAddr 520 . . } 521 . . 522 . . // recordspan adds a newly allocated span to h.allspans. 523 . . // 524 . . // This only happens the first time a span is allocated from
runtime.makeSpanClass
/usr/lib/go/src/runtime/mheap.go
Total: 30ms 30ms (flat, cum) 0.042% 576 . . numSpanClasses = gc.NumSizeClasses << 1 577 . . tinySpanClass = spanClass(tinySizeClass<<1 | 1) 578 . . ) 579 . . 580 . . func makeSpanClass(sizeclass uint8, noscan bool) spanClass { 581 30ms 30ms return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) ⋮ ⋮ return int(*(*uint8)(unsafe.Pointer(&x))) stubs.go:409 582 . . } 583 . . 584 . . //go:nosplit 585 . . func (sc spanClass) sizeclass() int8 { 586 . . return int8(sc >> 1)
runtime.arenaIndex
/usr/lib/go/src/runtime/mheap.go
Total: 40ms 40ms (flat, cum) 0.056% 602 . . // It is nosplit because it's called by spanOf and several other 603 . . // nosplit functions. 604 . . // 605 . . //go:nosplit 606 . . func arenaIndex(p uintptr) arenaIdx { 607 40ms 40ms return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) 608 . . } 609 . . 610 . . // arenaBase returns the low address of the region covered by heap 611 . . // arena i. 612 . . func arenaBase(i arenaIdx) uintptr {
runtime.spanOf
/usr/lib/go/src/runtime/mheap.go
Total: 20ms 20ms (flat, cum) 0.028% 705 . . l2 := mheap_.arenas[ri.l1()] 706 . . if arenaL1Bits != 0 && l2 == nil { // Should never happen if there's no L1. 707 . . return nil 708 . . } 709 . . ha := l2[ri.l2()] 710 20ms 20ms if ha == nil { 711 . . return nil 712 . . } 713 . . return ha.spans[(p/pageSize)%pagesPerArena] 714 . . } 715 . .
runtime.spanOfUnchecked
/usr/lib/go/src/runtime/mheap.go
Total: 20ms 20ms (flat, cum) 0.028% 718 . . // 719 . . // Must be nosplit because it has callers that are nosplit. 720 . . // 721 . . //go:nosplit 722 . . func spanOfUnchecked(p uintptr) *mspan { 723 10ms 10ms ai := arenaIndex(p) return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) mheap.go:607 724 10ms 10ms return mheap_.arenas[ai.l1()][ai.l2()].spans[(p/pageSize)%pagesPerArena] 725 . . } 726 . . 727 . . // spanOfHeap is like spanOf, but returns nil if p does not point to a 728 . . // heap object.
runtime.spanOfHeap
/usr/lib/go/src/runtime/mheap.go
Total: 20ms 20ms (flat, cum) 0.028% 729 . . // 730 . . // Must be nosplit because it has callers that are nosplit. 731 . . // 732 . . //go:nosplit 733 10ms 10ms func spanOfHeap(p uintptr) *mspan { 734 10ms 10ms s := spanOf(p) 735 . . // s is nil if it's never been allocated. Otherwise, we check 736 . . // its state first because we don't trust this pointer, so we 737 . . // have to synchronize with span initialization. Then, it's 738 . . // still possible we picked up a stale span pointer, so we 739 . . // have to check the span's bounds.
runtime.pageIndexOf
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 10ms (flat, cum) 0.014% 744 . . } 745 . . 746 . . // pageIndexOf returns the arena, page index, and page mask for pointer p. 747 . . // The caller must ensure p is in the heap. 748 . . func pageIndexOf(p uintptr) (arena *heapArena, pageIdx uintptr, pageMask uint8) { 749 10ms 10ms ai := arenaIndex(p) return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) mheap.go:607 750 . . arena = mheap_.arenas[ai.l1()][ai.l2()] 751 . . pageIdx = ((p / pageSize) / 8) % uintptr(len(arena.pageInUse)) 752 . . pageMask = byte(1 << ((p / pageSize) % 8)) 753 . . return 754 . . }
runtime.(*mheap).alloc
/usr/lib/go/src/runtime/mheap.go
Total: 40ms 1.04s (flat, cum) 1.45% 992 . . // 993 . . // spanclass indicates the span's size class and scannability. 994 . . // 995 . . // Returns a span that has been fully initialized. span.needzero indicates 996 . . // whether the span has been zeroed. Note that it may not be. 997 . 10ms func (h *mheap) alloc(npages uintptr, spanclass spanClass) *mspan { 998 . . // Don't do any operations that lock the heap on the G stack. 999 . . // It might trigger stack growth, and the stack growth code needs 1000 . . // to be able to allocate heap. 1001 . . var s *mspan 1002 40ms 1.03s systemstack(func() { 1003 . . // To prevent excessive heap growth, before allocating n pages
runtime.(*mheap).alloc.func1
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 930ms (flat, cum) 1.30% 1004 . . // we need to sweep and reclaim at least n pages. 1005 10ms 10ms if !isSweepDone() { 1006 . . h.reclaim(npages) 1007 . . } 1008 . 920ms s = h.allocSpan(npages, spanAllocHeap, spanclass) 1009 . . }) 1010 . . return s 1011 . . } 1012 . . 1013 . . // allocManual allocates a manually-managed span of npage pages.
runtime.(*mheap).allocNeedsZero
/usr/lib/go/src/runtime/mheap.go
Total: 160ms 160ms (flat, cum) 0.22% 1058 . . // allocator can otherwise prove the memory it's allocating is already zero because 1059 . . // they're fresh from the operating system. It updates heapArena metadata that is 1060 . . // critical for future page allocations. 1061 . . // 1062 . . // There are no locking constraints on this method. 1063 10ms 10ms func (h *mheap) allocNeedsZero(base, npage uintptr) (needZero bool) { 1064 10ms 10ms for npage > 0 { 1065 20ms 20ms ai := arenaIndex(base) return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) mheap.go:607 1066 . . ha := h.arenas[ai.l1()][ai.l2()] 1067 . . 1068 30ms 30ms zeroedBase := atomic.Loaduintptr(&ha.zeroedBase) 1069 90ms 90ms arenaBase := base % heapArenaBytes 1070 . . if arenaBase < zeroedBase { 1071 . . // We extended into the non-zeroed part of the 1072 . . // arena, so this region needs to be zeroed before use. 1073 . . // 1074 . . // zeroedBase is monotonically increasing, so if we see this now then
runtime.(*mheap).tryAllocMSpan
/usr/lib/go/src/runtime/mheap.go
Total: 30ms 30ms (flat, cum) 0.042% 1127 . . //go:systemstack 1128 . . func (h *mheap) tryAllocMSpan() *mspan { 1129 . . pp := getg().m.p.ptr() 1130 . . // If we don't have a p or the cache is empty, we can't do 1131 . . // anything here. 1132 10ms 10ms if pp == nil || pp.mspancache.len == 0 { 1133 . . return nil 1134 . . } 1135 . . // Pull off the last entry in the cache. 1136 . . s := pp.mspancache.buf[pp.mspancache.len-1] 1137 20ms 20ms pp.mspancache.len-- 1138 . . return s 1139 . . } 1140 . . 1141 . . // allocMSpanLocked allocates an mspan object. 1142 . . //
runtime.(*mheap).allocMSpanLocked
/usr/lib/go/src/runtime/mheap.go
Total: 0 190ms (flat, cum) 0.27% 1158 . . } 1159 . . // Refill the cache if necessary. 1160 . . if pp.mspancache.len == 0 { 1161 . . const refillCount = len(pp.mspancache.buf) / 2 1162 . . for i := 0; i < refillCount; i++ { 1163 . 190ms pp.mspancache.buf[i] = (*mspan)(h.spanalloc.alloc()) 1164 . . } 1165 . . pp.mspancache.len = refillCount 1166 . . } 1167 . . // Pull off the last entry in the cache. 1168 . . s := pp.mspancache.buf[pp.mspancache.len-1]
runtime.(*mheap).freeMSpanLocked
/usr/lib/go/src/runtime/mheap.go
Total: 30ms 30ms (flat, cum) 0.042% 1183 . . func (h *mheap) freeMSpanLocked(s *mspan) { 1184 . . assertLockHeld(&h.lock) 1185 . . 1186 . . pp := getg().m.p.ptr() 1187 . . // First try to free the mspan directly to the cache. 1188 10ms 10ms if pp != nil && pp.mspancache.len < len(pp.mspancache.buf) { 1189 . . pp.mspancache.buf[pp.mspancache.len] = s 1190 . . pp.mspancache.len++ 1191 . . return 1192 . . } 1193 . . // Failing that (or if we don't have a p), just free it to 1194 . . // the heap. 1195 20ms 20ms h.spanalloc.free(unsafe.Pointer(s)) f.inuse -= f.size mfixalloc.go:105 1196 . . } 1197 . . 1198 . . // allocSpan allocates an mspan which owns npages worth of memory. 1199 . . // 1200 . . // If typ.manual() == false, allocSpan allocates a heap span of class spanclass
runtime.(*mheap).allocSpan
/usr/lib/go/src/runtime/mheap.go
Total: 40ms 100ms (flat, cum) 0.14% 1225 . . 1226 . . // If the allocation is small enough, try the page cache! 1227 . . // The page cache does not support aligned allocations, so we cannot use 1228 . . // it if we need to provide a physical page aligned stack allocation. 1229 . . pp := gp.m.p.ptr() 1230 10ms 10ms if !needPhysPageAlign && pp != nil && npages < pageCachePages/4 { 1231 . . c := &pp.pcache 1232 . . 1233 . . // If the cache is empty, refill it. 1234 . . if c.empty() { 1235 . . lock(&h.lock) 1236 . 30ms *c = h.pages.allocToCache() 1237 . . unlock(&h.lock) 1238 . . } 1239 . . 1240 . . // Try to allocate from the cache. 1241 . 30ms base, scav = c.alloc(npages) 1242 . . if base != 0 { 1243 30ms 30ms s = h.tryAllocMSpan() pp.mspancache.len-- mheap.go:1137 ⋮ if pp == nil || pp.mspancache.len == 0 { mheap.go:1132 1244 . . if s != nil { 1245 . . goto HaveSpan 1246 . . } 1247 . . // We have a base but no mspan, so we need 1248 . . // to lock the heap.
runtime.(*mheap).allocSpan
/usr/lib/go/src/runtime/mheap.go
Total: 30ms 240ms (flat, cum) 0.34% 1285 . . if base == 0 { 1286 . . // Try to acquire a base address. 1287 . . base, scav = h.pages.alloc(npages) 1288 . . if base == 0 { 1289 . . var ok bool 1290 . 10ms growth, ok = h.grow(npages) 1291 . . if !ok { 1292 . . unlock(&h.lock) 1293 . . return nil 1294 . . } 1295 . . base, scav = h.pages.alloc(npages) 1296 . . if base == 0 { 1297 . . throw("grew heap, but no adequate free space found") 1298 . . } 1299 . . } 1300 . . } 1301 . . if s == nil { 1302 . . // We failed to get an mspan earlier, so grab 1303 . . // one now that we have the heap lock. 1304 . 190ms s = h.allocMSpanLocked() 1305 . . } 1306 . 10ms unlock(&h.lock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 1307 . . 1308 . . HaveSpan: 1309 . . // Decide if we need to scavenge in response to what we just allocated. 1310 . . // Specifically, we track the maximum amount of memory to scavenge of all 1311 . . // the alternatives below, assuming that the maximum satisfies *all* 1312 . . // conditions we check (e.g. if we need to scavenge X to satisfy the 1313 . . // memory limit and Y to satisfy heap-growth scavenging, and Y > X, then 1314 . . // it's fine to pick Y, because the memory limit is still satisfied). 1315 . . // 1316 . . // It's fine to do this after allocating because we expect any scavenged 1317 . . // pages not to get touched until we return. Simultaneously, it's important 1318 . . // to do this before calling sysUsed because that may commit address space. 1319 . . bytesToScavenge := uintptr(0) 1320 . . forceScavenge := false 1321 10ms 10ms if limit := gcController.memoryLimit.Load(); !gcCPULimiter.limiting() { return l.enabled.Load() mgclimit.go:91 return b.u.Load() != 0 types.go:168 1322 . . // Assist with scavenging to maintain the memory limit by the amount 1323 . . // that we expect to page in. 1324 . . inuse := gcController.mappedReady.Load() 1325 . . // Be careful about overflow, especially with uintptrs. Even on 32-bit platforms 1326 . . // someone can set a really big memory limit that isn't math.MaxInt64. 1327 20ms 20ms if uint64(scav)+inuse > uint64(limit) { 1328 . . bytesToScavenge = uintptr(uint64(scav) + inuse - uint64(limit)) 1329 . . forceScavenge = true 1330 . . } 1331 . . } 1332 . . if goal := scavenge.gcPercentGoal.Load(); goal != ^uint64(0) && growth > 0 {
runtime.(*mheap).allocSpan
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 580ms (flat, cum) 0.81% 1379 . . } 1380 . . scavenge.assistTime.Add(now - start) 1381 . . } 1382 . . 1383 . . // Initialize the span. 1384 . 530ms h.initSpan(s, typ, spanclass, base, npages, scav) 1385 . . 1386 . . if valgrindenabled { 1387 . . valgrindMempoolMalloc(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base), npages*pageSize) 1388 . . } 1389 . . 1390 . . // Commit and account for any scavenged memory that the span now owns. 1391 . . nbytes := npages * pageSize 1392 . . if scav != 0 { 1393 . . // sysUsed all the pages that are actually available 1394 . . // in the span since some of them might be scavenged. 1395 . . sysUsed(unsafe.Pointer(base), nbytes, scav) 1396 . . gcController.heapReleased.add(-int64(scav)) 1397 . . } 1398 . . // Update stats. 1399 . 10ms gcController.heapFree.add(-int64(nbytes - scav)) 1400 . . if typ == spanAllocHeap { 1401 . 10ms gcController.heapInUse.add(int64(nbytes)) 1402 . . } 1403 . . // Update consistent stats. 1404 . 10ms stats := memstats.heapStats.acquire() 1405 10ms 10ms atomic.Xaddint64(&stats.committed, int64(scav)) 1406 . . atomic.Xaddint64(&stats.released, -int64(scav)) 1407 . . switch typ { 1408 . . case spanAllocHeap: 1409 . . atomic.Xaddint64(&stats.inHeap, int64(nbytes)) 1410 . . case spanAllocStack: 1411 . . atomic.Xaddint64(&stats.inStacks, int64(nbytes)) 1412 . . case spanAllocWorkBuf: 1413 . . atomic.Xaddint64(&stats.inWorkBufs, int64(nbytes)) 1414 . . } 1415 . 10ms memstats.heapStats.release() 1416 . . 1417 . . // Trace the span alloc. 1418 . . if traceAllocFreeEnabled() { 1419 . . trace := traceAcquire() 1420 . . if trace.ok() {
runtime.(*mheap).initSpan
/usr/lib/go/src/runtime/mheap.go
Total: 0 320ms (flat, cum) 0.45% 1428 . . // initSpan initializes a blank span s which will represent the range 1429 . . // [base, base+npages*pageSize). typ is the type of span being allocated. 1430 . . func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, npages, scav uintptr) { 1431 . . // At this point, both s != nil and base != 0, and the heap 1432 . . // lock is no longer held. Initialize the span. 1433 . 160ms s.init(base, npages) 1434 . . // Always call allocNeedsZero to update the arena's zeroedBase watermark 1435 . . // and determine if the memory is considered dirty. 1436 . 160ms needZero := h.allocNeedsZero(base, npages) 1437 . . // If these pages were scavenged (returned to the OS), the kernel guarantees 1438 . . // they will be zero-filled on next use (fault-in), so we can treat them as 1439 . . // already zeroed and skip explicit clearing. 1440 . . if (needZeroAfterSysUnused() || scav != npages*pageSize) && needZero { 1441 . . s.needzero = 1
runtime.(*mheap).initSpan
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 150ms (flat, cum) 0.21% 1461 . . // Reserve space for the inline mark bits. 1462 . . reserve += unsafe.Sizeof(spanInlineMarkBits{}) 1463 . . } 1464 . . if heapBitsInSpan(s.elemsize) && !s.spanclass.noscan() { 1465 . . // Reserve space for the pointer/scan bitmap at the end. 1466 10ms 10ms reserve += nbytes / goarch.PtrSize / 8 1467 . . } 1468 . . s.nelems = uint16((nbytes - reserve) / s.elemsize) 1469 . . } else { 1470 . . if !s.spanclass.noscan() && heapBitsInSpan(s.elemsize) { 1471 . . // Reserve space for the pointer/scan bitmap at the end. 1472 . . s.nelems = uint16((nbytes - (nbytes / goarch.PtrSize / 8)) / s.elemsize) 1473 . . } else { 1474 . . s.nelems = uint16(nbytes / s.elemsize) 1475 . . } 1476 . . } 1477 . . s.divMul = gc.SizeClassToDivMagic[sizeclass] 1478 . . } 1479 . . 1480 . . // Initialize mark and allocation structures. 1481 . . s.freeindex = 0 1482 . . s.freeIndexForScan = 0 1483 . . s.allocCache = ^uint64(0) // all 1s indicating all free. 1484 . 130ms s.gcmarkBits = newMarkBits(uintptr(s.nelems)) 1485 . 10ms s.allocBits = newAllocBits(uintptr(s.nelems)) return newMarkBits(nelems) mheap.go:2968 1486 . . 1487 . . // Adjust s.limit down to the object-containing part of the span. 1488 . . s.limit = s.base() + s.elemsize*uintptr(s.nelems) 1489 . . 1490 . . // It's safe to access h.sweepgen without the heap lock because it's
runtime.(*mheap).initSpan
/usr/lib/go/src/runtime/mheap.go
Total: 60ms 60ms (flat, cum) 0.084% 1522 . . // prior to this line. 1523 . . arena, pageIdx, pageMask := pageIndexOf(s.base()) 1524 . . atomic.Or8(&arena.pageInUse[pageIdx], pageMask) 1525 . . 1526 . . // Mark packed span. 1527 40ms 40ms if gcUsesSpanInlineMarkBits(s.elemsize) { 1528 . . atomic.Or8(&arena.pageUseSpanInlineMarkBits[pageIdx], pageMask) 1529 . . } 1530 . . 1531 . . // Update related page sweeper stats. 1532 20ms 20ms h.pagesInUse.Add(npages) 1533 . . } 1534 . . 1535 . . // Make sure the newly allocated span will be observed 1536 . . // by the GC before pointers into the span are published. 1537 . . publicationBarrier()
runtime.(*mheap).grow
/usr/lib/go/src/runtime/mheap.go
Total: 0 10ms (flat, cum) 0.014% 1630 . . atomic.Xaddint64(&stats.released, int64(nBase-v)) 1631 . . memstats.heapStats.release() 1632 . . 1633 . . // Update the page allocator's structures to make this 1634 . . // space ready for allocation. 1635 . 10ms h.pages.grow(v, nBase-v) 1636 . . totalGrowth += nBase - v 1637 . . 1638 . . if firstGrow && randomizeHeapBase { 1639 . . // The top heapAddrBits-log2(pallocChunkBytes) bits are now randomized, 1640 . . // we finally want to randomize the next
runtime.(*mheap).freeSpan
/usr/lib/go/src/runtime/mheap.go
Total: 0 520ms (flat, cum) 0.73% 1653 . . return totalGrowth, true 1654 . . } 1655 . . 1656 . . // Free the span back into the heap. 1657 . . func (h *mheap) freeSpan(s *mspan) { 1658 . 520ms systemstack(func() { 1659 . . // Trace the span free. 1660 . . if traceAllocFreeEnabled() { 1661 . . trace := traceAcquire() 1662 . . if trace.ok() {
runtime.(*mheap).freeSpan.func1
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 520ms (flat, cum) 0.73% 1664 . . traceRelease(trace) 1665 . . } 1666 . . } 1667 . . 1668 10ms 180ms lock(&h.lock) 1669 . . if msanenabled { 1670 . . // Tell msan that this entire span is no longer in use. 1671 . . base := unsafe.Pointer(s.base()) 1672 . . bytes := s.npages << gc.PageShift 1673 . . msanfree(base, bytes) 1674 . . } 1675 . . if asanenabled { 1676 . . // Tell asan that this entire span is no longer in use. 1677 . . base := unsafe.Pointer(s.base()) 1678 . . bytes := s.npages << gc.PageShift 1679 . . asanpoison(base, bytes) 1680 . . } 1681 . . if valgrindenabled { 1682 . . base := s.base() 1683 . . valgrindMempoolFree(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base)) 1684 . . } 1685 . 290ms h.freeSpanLocked(s, spanAllocHeap) 1686 . 50ms unlock(&h.lock) 1687 . . }) 1688 . . } 1689 . . 1690 . . // freeManual frees a manually-managed span returned by allocManual. 1691 . . // typ must be the same as the spanAllocType passed to the allocManual that
runtime.(*mheap).freeManual
/usr/lib/go/src/runtime/mheap.go
Total: 0 20ms (flat, cum) 0.028% 1712 . . lock(&h.lock) 1713 . . if valgrindenabled { 1714 . . base := s.base() 1715 . . valgrindMempoolFree(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base)) 1716 . . } 1717 . 20ms h.freeSpanLocked(s, typ) 1718 . . unlock(&h.lock) 1719 . . } 1720 . . 1721 . . func (h *mheap) freeSpanLocked(s *mspan, typ spanAllocType) { 1722 . . assertLockHeld(&h.lock)
runtime.(*mheap).freeSpanLocked
/usr/lib/go/src/runtime/mheap.go
Total: 120ms 310ms (flat, cum) 0.43% 1732 . . } 1733 . . if s.allocCount != 0 || s.sweepgen != h.sweepgen { 1734 . . print("mheap.freeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") 1735 . . throw("mheap.freeSpanLocked - invalid free") 1736 . . } 1737 10ms 10ms h.pagesInUse.Add(-s.npages) return Xadduintptr(&u.value, delta) types.go:420 1738 . . 1739 . . // Clear in-use bit in arena page bitmap. 1740 10ms 10ms arena, pageIdx, pageMask := pageIndexOf(s.base()) ai := arenaIndex(p) mheap.go:749 return arenaIdx((p - arenaBaseOffset) / heapArenaBytes) mheap.go:607 1741 . . atomic.And8(&arena.pageInUse[pageIdx], ^pageMask) 1742 . . 1743 . . // Clear small heap span bit if necessary. 1744 20ms 20ms if gcUsesSpanInlineMarkBits(s.elemsize) { 1745 . . atomic.And8(&arena.pageUseSpanInlineMarkBits[pageIdx], ^pageMask) 1746 . . } 1747 . . default: 1748 . . throw("mheap.freeSpanLocked - invalid span state") 1749 . . } 1750 . . 1751 . . // Update stats. 1752 . . // 1753 . . // Mirrors the code in allocSpan. 1754 . . nbytes := s.npages * pageSize 1755 . 20ms gcController.heapFree.add(int64(nbytes)) 1756 . . if typ == spanAllocHeap { 1757 10ms 20ms gcController.heapInUse.add(-int64(nbytes)) 1758 . . } 1759 . . // Update consistent stats. 1760 . 20ms stats := memstats.heapStats.acquire() 1761 . . switch typ { 1762 . . case spanAllocHeap: 1763 40ms 40ms atomic.Xaddint64(&stats.inHeap, -int64(nbytes)) 1764 . . case spanAllocStack: 1765 . . atomic.Xaddint64(&stats.inStacks, -int64(nbytes)) 1766 . . case spanAllocWorkBuf: 1767 . . atomic.Xaddint64(&stats.inWorkBufs, -int64(nbytes)) 1768 . . } 1769 . 10ms memstats.heapStats.release() 1770 . . 1771 . . // Mark the space as free. 1772 . 130ms h.pages.free(s.base(), s.npages) 1773 . . 1774 . . // Free the span structure. We no longer have a use for it. 1775 . . s.state.set(mSpanDead) 1776 30ms 30ms h.freeMSpanLocked(s) if pp != nil && pp.mspancache.len < len(pp.mspancache.buf) { mheap.go:1188 ⋮ h.spanalloc.free(unsafe.Pointer(s)) mheap.go:1195 f.inuse -= f.size mfixalloc.go:105 1777 . . } 1778 . . 1779 . . // scavengeAll acquires the heap lock (blocking any additional 1780 . . // manipulation of the page allocator) and iterates over the whole 1781 . . // heap, scavenging every free page available.
runtime.(*mspan).init
/usr/lib/go/src/runtime/mheap.go
Total: 160ms 160ms (flat, cum) 0.22% 1807 . . } 1808 . . 1809 . . // Initialize a new span with the given start and npages. 1810 . . func (span *mspan) init(base uintptr, npages uintptr) { 1811 . . // span is *not* zeroed. 1812 20ms 20ms span.next = nil 1813 . . span.prev = nil 1814 130ms 130ms span.list = nil 1815 . . span.startAddr = base 1816 . . span.npages = npages 1817 . . span.limit = base + npages*gc.PageSize // see go.dev/issue/74288; adjusted later for heap spans 1818 . . span.allocCount = 0 1819 . . span.spanclass = 0 1820 . . span.elemsize = 0 1821 . . span.speciallock.key = 0 1822 . . span.specials = nil 1823 . . span.needzero = 0 1824 . . span.freeindex = 0 1825 10ms 10ms span.freeIndexForScan = 0 1826 . . span.allocBits = nil 1827 . . span.gcmarkBits = nil 1828 . . span.pinnerBits = nil 1829 . . span.state.set(mSpanDead) 1830 . . lockInit(&span.speciallock, lockRankMspanSpecial)
runtime.spanHasSpecials
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 10ms (flat, cum) 0.014% 1972 . . // spanHasSpecials marks a span as having specials in the arena bitmap. 1973 . . func spanHasSpecials(s *mspan) { 1974 . . arenaPage := (s.base() / pageSize) % pagesPerArena 1975 . . ai := arenaIndex(s.base()) 1976 . . ha := mheap_.arenas[ai.l1()][ai.l2()] 1977 10ms 10ms atomic.Or8(&ha.pageSpecials[arenaPage/8], uint8(1)<<(arenaPage%8)) 1978 . . } 1979 . . 1980 . . // spanHasNoSpecials marks a span as having no specials in the arena bitmap. 1981 . . func spanHasNoSpecials(s *mspan) { 1982 . . arenaPage := (s.base() / pageSize) % pagesPerArena
runtime.addspecial
/usr/lib/go/src/runtime/mheap.go
Total: 20ms 30ms (flat, cum) 0.042% 1999 . . 2000 . . // Ensure that the span is swept. 2001 . . // Sweeping accesses the specials list w/o locks, so we have 2002 . . // to synchronize with it. And it's just much safer. 2003 . . mp := acquirem() 2004 . 10ms span.ensureSwept() 2005 . . 2006 . . offset := uintptr(p) - span.base() 2007 . . kind := s.kind 2008 . . 2009 . . lock(&span.speciallock) 2010 . . 2011 . . // Find splice point, check for existing record. 2012 . . iter, exists := span.specialFindSplicePoint(offset, kind) 2013 . . if !exists || force { 2014 . . // Splice in record, fill in offset. 2015 . . s.offset = offset 2016 . . s.next = *iter 2017 . . *iter = s 2018 10ms 10ms spanHasSpecials(span) atomic.Or8(&ha.pageSpecials[arenaPage/8], uint8(1)<<(arenaPage%8)) mheap.go:1977 2019 . . } 2020 . . 2021 . . unlock(&span.speciallock) 2022 10ms 10ms releasem(mp) if mp.locks == 0 && gp.preempt { runtime1.go:622 2023 . . // We're converting p to a uintptr and looking it up, and we 2024 . . // don't want it to die and get swept while we're doing so. 2025 . . KeepAlive(p) 2026 . . return !exists || force // already exists or addition was forced 2027 . . }
runtime.removespecial
/usr/lib/go/src/runtime/mheap.go
Total: 20ms 70ms (flat, cum) 0.098% 2028 . . 2029 . . // Removes the Special record of the given kind for the object p. 2030 . . // Returns the record if the record existed, nil otherwise. 2031 . . // The caller must FixAlloc_Free the result. 2032 . . func removespecial(p unsafe.Pointer, kind uint8) *special { 2033 . 20ms span := spanOfHeap(uintptr(p)) 2034 . . if span == nil { 2035 . . throw("removespecial on invalid pointer") 2036 . . } 2037 . . 2038 . . // Ensure that the span is swept. 2039 . . // Sweeping accesses the specials list w/o locks, so we have 2040 . . // to synchronize with it. And it's just much safer. 2041 . . mp := acquirem() 2042 . 10ms span.ensureSwept() 2043 . . 2044 . . offset := uintptr(p) - span.base() 2045 . . 2046 . . var result *special 2047 . 20ms lock(&span.speciallock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 2048 . . 2049 . . iter, exists := span.specialFindSplicePoint(offset, kind) 2050 . . if exists { 2051 . . s := *iter 2052 . . *iter = s.next 2053 . . result = s 2054 . . } 2055 . . if span.specials == nil { 2056 . . spanHasNoSpecials(span) 2057 . . } 2058 20ms 20ms unlock(&span.speciallock) 2059 . . releasem(mp) 2060 . . return result 2061 . . } 2062 . . 2063 . . // Find a splice point in the sorted list and check for an already existing
runtime.addfinalizer
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 110ms (flat, cum) 0.15% 2096 . . fint *_type // May be a heap pointer, but always live. 2097 . . ot *ptrtype // May be a heap pointer, but always live. 2098 . . } 2099 . . 2100 . . // Adds a finalizer to the object p. Returns true if it succeeded. 2101 10ms 10ms func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *ptrtype) bool { 2102 . 20ms lock(&mheap_.speciallock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 2103 . 30ms s := (*specialfinalizer)(mheap_.specialfinalizeralloc.alloc()) 2104 . 30ms unlock(&mheap_.speciallock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 2105 . . s.special.kind = _KindSpecialFinalizer 2106 . . s.fn = f 2107 . . s.nret = nret 2108 . . s.fint = fint 2109 . . s.ot = ot 2110 . 20ms if addspecial(p, &s.special, false) { 2111 . . // This is responsible for maintaining the same 2112 . . // GC-related invariants as markrootSpans in any 2113 . . // situation where it's possible that markrootSpans 2114 . . // has already run but mark termination hasn't yet. 2115 . . if gcphase != _GCoff {
runtime.removefinalizer
/usr/lib/go/src/runtime/mheap.go
Total: 0 110ms (flat, cum) 0.15% 2136 . . return false 2137 . . } 2138 . . 2139 . . // Removes the finalizer (if any) from the object p. 2140 . . func removefinalizer(p unsafe.Pointer) { 2141 . 70ms s := (*specialfinalizer)(unsafe.Pointer(removespecial(p, _KindSpecialFinalizer))) 2142 . . if s == nil { 2143 . . return // there wasn't a finalizer to remove 2144 . . } 2145 . 20ms lock(&mheap_.speciallock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 2146 . . mheap_.specialfinalizeralloc.free(unsafe.Pointer(s)) 2147 . 20ms unlock(&mheap_.speciallock) unlockWithRank(l) lock_spinbit.go:270 unlock2(l) lockrank_off.go:36 2148 . . } 2149 . . 2150 . . // The described object has a cleanup set for it. 2151 . . type specialCleanup struct { 2152 . . _ sys.NotInHeap
runtime.setprofilebucket
/usr/lib/go/src/runtime/mheap.go
Total: 10ms 30ms (flat, cum) 0.042% 2703 . . special special 2704 . . b *bucket 2705 . . } 2706 . . 2707 . . // Set the heap profile bucket associated with addr to b. 2708 10ms 10ms func setprofilebucket(p unsafe.Pointer, b *bucket) { 2709 . . lock(&mheap_.speciallock) 2710 . 10ms s := (*specialprofile)(mheap_.specialprofilealloc.alloc()) 2711 . . unlock(&mheap_.speciallock) 2712 . . s.special.kind = _KindSpecialProfile 2713 . . s.b = b 2714 . 10ms if !addspecial(p, &s.special, false) { 2715 . . throw("setprofilebucket: profile already set") 2716 . . } 2717 . . } 2718 . . 2719 . . // specialReachable tracks whether an object is reachable on the next
runtime.freeSpecial
/usr/lib/go/src/runtime/mheap.go
Total: 0 20ms (flat, cum) 0.028% 2788 . . lock(&mheap_.speciallock) 2789 . . mheap_.specialWeakHandleAlloc.free(unsafe.Pointer(s)) 2790 . . unlock(&mheap_.speciallock) 2791 . . case _KindSpecialProfile: 2792 . . sp := (*specialprofile)(unsafe.Pointer(s)) 2793 . 20ms mProf_Free(sp.b) 2794 . . lock(&mheap_.speciallock) 2795 . . mheap_.specialprofilealloc.free(unsafe.Pointer(sp)) 2796 . . unlock(&mheap_.speciallock) 2797 . . case _KindSpecialReachable: 2798 . . sp := (*specialReachable)(unsafe.Pointer(s))
runtime.(*gcBitsArena).tryAlloc
/usr/lib/go/src/runtime/mheap.go
Total: 180ms 180ms (flat, cum) 0.25% 2891 . . } 2892 . . 2893 . . // tryAlloc allocates from b or returns nil if b does not have enough room. 2894 . . // This is safe to call concurrently. 2895 . . func (b *gcBitsArena) tryAlloc(bytes uintptr) *gcBits { 2896 50ms 50ms if b == nil || atomic.Loaduintptr(&b.free)+bytes > uintptr(len(b.bits)) { 2897 . . return nil 2898 . . } 2899 . . // Try to allocate from this block. 2900 130ms 130ms end := atomic.Xadduintptr(&b.free, bytes) 2901 . . if end > uintptr(len(b.bits)) { 2902 . . return nil 2903 . . } 2904 . . // There was enough room. 2905 . . start := end - bytes
runtime.newMarkBits
/usr/lib/go/src/runtime/mheap.go
Total: 190ms 210ms (flat, cum) 0.29% 2906 . . return &b.bits[start] 2907 . . } 2908 . . 2909 . . // newMarkBits returns a pointer to 8 byte aligned bytes 2910 . . // to be used for a span's mark bits. 2911 10ms 10ms func newMarkBits(nelems uintptr) *gcBits { 2912 . . blocksNeeded := (nelems + 63) / 64 2913 . . bytesNeeded := blocksNeeded * 8 2914 . . 2915 . . // Try directly allocating from the current head arena. 2916 . . head := (*gcBitsArena)(atomic.Loadp(unsafe.Pointer(&gcBitsArenas.next))) 2917 180ms 180ms if p := head.tryAlloc(bytesNeeded); p != nil { if b == nil || atomic.Loaduintptr(&b.free)+bytes > uintptr(len(b.bits)) { mheap.go:2896 ⋮ ⋮ end := atomic.Xadduintptr(&b.free, bytes) mheap.go:2900 2918 . . return p 2919 . . } 2920 . . 2921 . . // There's not enough room in the head arena. We may need to 2922 . . // allocate a new arena. 2923 . 10ms lock(&gcBitsArenas.lock) lockWithRank(l, getLockRank(l)) lock_spinbit.go:161 lock2(l) lockrank_off.go:24 2924 . . // Try the head arena again, since it may have changed. Now 2925 . . // that we hold the lock, the list head can't change, but its 2926 . . // free position still can. 2927 . . if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 2928 . . unlock(&gcBitsArenas.lock) 2929 . . return p 2930 . . } 2931 . . 2932 . . // Allocate a new arena. This may temporarily drop the lock. 2933 . 10ms fresh := newArenaMayUnlock() 2934 . . // If newArenaMayUnlock dropped the lock, another thread may 2935 . . // have put a fresh arena on the "next" list. Try allocating 2936 . . // from next again. 2937 . . if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 2938 . . // Put fresh back on the free list.
runtime.newAllocBits
/usr/lib/go/src/runtime/mheap.go
Total: 0 10ms (flat, cum) 0.014% 2963 . . // newAllocBits is used to provide newly initialized spans 2964 . . // allocation bits. For spans not being initialized the 2965 . . // mark bits are repurposed as allocation bits when 2966 . . // the span is swept. 2967 . . func newAllocBits(nelems uintptr) *gcBits { 2968 . 10ms return newMarkBits(nelems) 2969 . . } 2970 . . 2971 . . // nextMarkBitArenaEpoch establishes a new epoch for the arenas 2972 . . // holding the mark bits. The arenas are named relative to the 2973 . . // current GC cycle which is demarcated by the call to finishweep_m.
runtime.newArenaMayUnlock
/usr/lib/go/src/runtime/mheap.go
Total: 0 10ms (flat, cum) 0.014% 3006 . . // The caller must hold gcBitsArena.lock. This may temporarily release it. 3007 . . func newArenaMayUnlock() *gcBitsArena { 3008 . . var result *gcBitsArena 3009 . . if gcBitsArenas.free == nil { 3010 . . unlock(&gcBitsArenas.lock) 3011 . 10ms result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gcMiscSys, "gc bits")) 3012 . . if result == nil { 3013 . . throw("runtime: cannot allocate memory") 3014 . . } 3015 . . lock(&gcBitsArenas.lock) 3016 . . } else {
runtime.(*unwinder).init
/usr/lib/go/src/runtime/traceback.go
Total: 0 390ms (flat, cum) 0.55% 125 . . // provide a "valid" method. Alternatively, this could start in a "before 126 . . // the first frame" state and "next" could return whether it was able to 127 . . // move to the next frame, but that's both more awkward to use in a "for" 128 . . // loop and is harder to implement because we have to do things differently 129 . . // for the first frame. 130 . 390ms u.initAt(^uintptr(0), ^uintptr(0), ^uintptr(0), gp, flags) 131 . . }
runtime.(*unwinder).initAt
/usr/lib/go/src/runtime/traceback.go
Total: 40ms 40ms (flat, cum) 0.056% 132 . . 133 30ms 30ms func (u *unwinder) initAt(pc0, sp0, lr0 uintptr, gp *g, flags unwindFlags) { 134 . . // Don't call this "g"; it's too easy get "g" and "gp" confused. 135 10ms 10ms if ourg := getg(); ourg == gp && ourg == ourg.m.curg { 136 . . // The starting sp has been passed in as a uintptr, and the caller may 137 . . // have other uintptr-typed stack references as well. 138 . . // If during one of the calls that got us here or during one of the 139 . . // callbacks below the stack must be grown, all these uintptr references 140 . . // to the stack will not be updated, and traceback will continue
runtime.(*unwinder).initAt
/usr/lib/go/src/runtime/traceback.go
Total: 20ms 360ms (flat, cum) 0.5% 196 . . // LR are not touched. 197 . . frame.pc = frame.lr 198 . . frame.lr = 0 199 . . } 200 . . 201 10ms 30ms f := findfunc(frame.pc) 202 . . if !f.valid() { 203 . . if flags&unwindSilentErrors == 0 { 204 . . print("runtime: g ", gp.goid, " gp=", gp, ": unknown pc ", hex(frame.pc), "\n") 205 . . tracebackHexdump(gp.stack, &frame, 0) 206 . . } 207 . . if flags&(unwindPrintErrors|unwindSilentErrors) == 0 { 208 . . throw("unknown pc") 209 . . } 210 . . *u = unwinder{} 211 . . return 212 . . } 213 . . frame.fn = f 214 . . 215 . . // Populate the unwinder. 216 10ms 10ms *u = unwinder{ 217 . . frame: frame, 218 . . g: gp.guintptr(), 219 . . cgoCtxt: len(gp.cgoCtxt) - 1, 220 . . calleeFuncID: abi.FuncIDNormal, 221 . . flags: flags, 222 . . } 223 . . 224 . . isSyscall := frame.pc == pc0 && frame.sp == sp0 && pc0 == gp.syscallpc && sp0 == gp.syscallsp 225 . 320ms u.resolveInternal(true, isSyscall) 226 . . }
runtime.(*unwinder).valid
/usr/lib/go/src/runtime/traceback.go
Total: 10ms 10ms (flat, cum) 0.014% 228 . . func (u *unwinder) valid() bool { 229 10ms 10ms return u.frame.pc != 0 230 . . } 231 . . 232 . . // resolveInternal fills in u.frame based on u.frame.fn, pc, and sp. 233 . . // 234 . . // innermost indicates that this is the first resolve on this stack. If
runtime.(*unwinder).resolveInternal
/usr/lib/go/src/runtime/traceback.go
Total: 440ms 440ms (flat, cum) 0.62% 248 . . // 249 . . // If fn is a stack-jumping function, resolveInternal can change the entire 250 . . // frame state to follow that stack jump. 251 . . // 252 . . // This is internal to unwinder. 253 20ms 20ms func (u *unwinder) resolveInternal(innermost, isSyscall bool) { 254 . . frame := &u.frame 255 . . gp := u.g.ptr() 256 . . 257 . . f := frame.fn 258 350ms 350ms if f.pcsp == 0 { 259 . . // No frame information, must be external function, like race support. 260 . . // See golang.org/issue/13568. 261 . . u.finishInternal() 262 . . return 263 . . } 264 . . 265 . . // Compute function info flags. 266 . . flag := f.flag 267 50ms 50ms if f.funcID == abi.FuncID_cgocallback { 268 . . // cgocallback does write SP to switch from the g0 to the curg stack, 269 . . // but it carefully arranges that during the transition BOTH stacks 270 . . // have cgocallback frame valid for unwinding through. 271 . . // So we don't need to exclude it with the other SP-writing functions. 272 10ms 10ms flag &^= abi.FuncFlagSPWrite 273 . . } 274 . . if isSyscall { 275 . . // Some Syscall functions write to SP, but they do so only after 276 . . // saving the entry PC/SP using entersyscall. 277 . . // Since we are using the entry PC/SP, the later SP write doesn't matter. 278 . . flag &^= abi.FuncFlagSPWrite 279 . . } 280 . . 281 . . // Found an actual function. 282 . . // Derive frame pointer. 283 . . if frame.fp == 0 { 284 . . // Jump over system stack transitions. If we're on g0 and there's a user 285 . . // goroutine, try to jump. Otherwise this is a regular call. 286 . . // We also defensively check that this won't switch M's on us, 287 . . // which could happen at critical points in the scheduler. 288 . . // This ensures gp.m doesn't change from a stack jump. 289 10ms 10ms if u.flags&unwindJumpStack != 0 && gp == gp.m.g0 && gp.m.curg != nil && gp.m.curg.m == gp.m { 290 . . switch f.funcID { 291 . . case abi.FuncID_morestack: 292 . . // morestack does not return normally -- newstack() 293 . . // gogo's to curg.sched. Match that. 294 . . // This keeps morestack() from showing up in the backtrace,
runtime.(*unwinder).resolveInternal
/usr/lib/go/src/runtime/traceback.go
Total: 50ms 2.52s (flat, cum) 3.52% 322 . . frame.sp = gp.sched.sp 323 . . u.cgoCtxt = len(gp.cgoCtxt) - 1 324 . . flag &^= abi.FuncFlagSPWrite 325 . . } 326 . . } 327 40ms 2.51s frame.fp = frame.sp + uintptr(funcspdelta(f, frame.pc)) ⋮ ⋮ ⋮ x, _ := pcvalue(f, f.pcsp, targetpc, true) symtab.go:1210 328 . . if !usesLR { 329 . . // On x86, call instruction pushes return PC before entering new function. 330 . . frame.fp += goarch.PtrSize 331 . . } 332 . . } 333 . . 334 . . // Derive link register. 335 10ms 10ms if flag&abi.FuncFlagTopFrame != 0 { 336 . . // This function marks the top of the stack. Stop the traceback. 337 . . frame.lr = 0 338 . . } else if flag&abi.FuncFlagSPWrite != 0 && (!innermost || u.flags&(unwindPrintErrors|unwindSilentErrors) != 0) { 339 . . // The function we are in does a write to SP that we don't know 340 . . // how to encode in the spdelta table. Examples include context
runtime.(*unwinder).resolveInternal
/usr/lib/go/src/runtime/traceback.go
Total: 100ms 100ms (flat, cum) 0.14% 367 . . } 368 . . frame.lr = 0 369 . . } else { 370 . . var lrPtr uintptr 371 . . if usesLR { 372 20ms 20ms if innermost && frame.sp < frame.fp || frame.lr == 0 { 373 . . lrPtr = frame.sp 374 60ms 60ms frame.lr = *(*uintptr)(unsafe.Pointer(lrPtr)) 375 . . } 376 . . } else { 377 . . if frame.lr == 0 { 378 . . lrPtr = frame.fp - goarch.PtrSize 379 . . frame.lr = *(*uintptr)(unsafe.Pointer(lrPtr)) 380 . . } 381 . . } 382 . . } 383 . . 384 20ms 20ms frame.varp = frame.fp 385 . . if !usesLR { 386 . . // On x86, call instruction pushes return PC before entering new function. 387 . . frame.varp -= goarch.PtrSize 388 . . } 389 . .
runtime.(*unwinder).resolveInternal
/usr/lib/go/src/runtime/traceback.go
Total: 60ms 60ms (flat, cum) 0.084% 403 . . // (with R29 = RSP - 8 in Go functions). 404 . . // This is technically ABI-compatible but not standard. 405 . . // And it happens to end up mimicking the x86 layout. 406 . . // Other architectures may make different decisions. 407 . . if frame.varp > frame.sp && framepointer_enabled { 408 10ms 10ms frame.varp -= goarch.PtrSize 409 . . } 410 . . 411 20ms 20ms frame.argp = frame.fp + sys.MinFrameSize 412 . . 413 . . // Determine frame's 'continuation PC', where it can continue. 414 . . // Normally this is the return address on the stack, but if sigpanic 415 . . // is immediately below this function on the stack, then the frame 416 . . // stopped executing due to a trap, and frame.pc is probably not 417 . . // a safe point for looking up liveness information. In this panicking case, 418 . . // the function either doesn't return at all (if it has no defers or if the 419 . . // defers do not recover) or it returns from one of the calls to 420 . . // deferproc a second time (if the corresponding deferred func recovers). 421 . . // In the latter case, use a deferreturn call site as the continuation pc. 422 . . frame.continpc = frame.pc 423 10ms 10ms if u.calleeFuncID == abi.FuncID_sigpanic { 424 . . if frame.fn.deferreturn != 0 { 425 . . frame.continpc = frame.fn.entry() + uintptr(frame.fn.deferreturn) + 1 426 . . // Note: this may perhaps keep return variables alive longer than 427 . . // strictly necessary, as we are using "function has a defer statement" 428 . . // as a proxy for "function actually deferred something". It seems 429 . . // to be a minor drawback. (We used to actually look through the 430 . . // gp._defer for a defer corresponding to this function, but that 431 . . // is hard to do with defer records on the stack during a stack copy.) 432 . . // Note: the +1 is to offset the -1 that 433 . . // (*stkframe).getStackMap does to back up a return 434 . . // address make sure the pc is in the CALL instruction. 435 . . } else { 436 . . frame.continpc = 0 437 . . } 438 . . } 439 20ms 20ms } 440 . .
runtime.isInjectedCall
/usr/lib/go/src/runtime/traceback.go
Total: 60ms 60ms (flat, cum) 0.084% 441 . . func isInjectedCall(id abi.FuncID) bool { 442 60ms 60ms return id == abi.FuncID_sigpanic || id == abi.FuncID_asyncPreempt || id == abi.FuncID_debugCallV2 443 . . }
runtime.(*unwinder).next
/usr/lib/go/src/runtime/traceback.go
Total: 60ms 1.01s (flat, cum) 1.41% 444 . . 445 40ms 40ms func (u *unwinder) next() { 446 . . frame := &u.frame 447 10ms 10ms f := frame.fn 448 . . gp := u.g.ptr() 449 . . 450 . . // Do not unwind past the bottom of the stack. 451 10ms 10ms if frame.lr == 0 { 452 . 20ms u.finishInternal() 453 . . return 454 . . } 455 . 930ms flr := findfunc(frame.lr) 456 . . if !flr.valid() { 457 . . // This happens if you get a profiling interrupt at just the wrong time. 458 . . // In that context it is okay to stop early. 459 . . // But if no error flags are set, we're doing a garbage collection and must 460 . . // get everything, so crash loudly.
runtime.(*unwinder).next
/usr/lib/go/src/runtime/traceback.go
Total: 80ms 2.88s (flat, cum) 4.03% 477 . . frame.lr = 0 478 . . u.finishInternal() 479 . . return 480 . . } 481 . . 482 20ms 20ms if frame.pc == frame.lr && frame.sp == frame.fp { 483 . . // If the next frame is identical to the current frame, we cannot make progress. 484 . . print("runtime: traceback stuck. pc=", hex(frame.pc), " sp=", hex(frame.sp), "\n") 485 . . tracebackHexdump(gp.stack, frame, frame.sp) 486 . . throw("traceback stuck") 487 . . } 488 . . 489 30ms 30ms injectedCall := isInjectedCall(f.funcID) return id == abi.FuncID_sigpanic || id == abi.FuncID_asyncPreempt || id == abi.FuncID_debugCallV2 traceback.go:442 490 . . if injectedCall { 491 . . u.flags |= unwindTrap 492 . . } else { 493 . . u.flags &^= unwindTrap 494 . . } 495 . . 496 . . // Unwind to next frame. 497 . . u.calleeFuncID = f.funcID 498 . . frame.fn = flr 499 . . frame.pc = frame.lr 500 . . frame.lr = 0 501 10ms 10ms frame.sp = frame.fp 502 . . frame.fp = 0 503 . . 504 . . // On link register architectures, sighandler saves the LR on stack 505 . . // before faking a call. 506 . . if usesLR && injectedCall { 507 . . x := *(*uintptr)(unsafe.Pointer(frame.sp)) 508 . . // same as the size bump used in scanframeworker. 509 . . frame.sp += alignUp(sys.MinFrameSize, sys.StackAlign) 510 . . f = findfunc(frame.pc) 511 . . frame.fn = f 512 . . if !f.valid() { 513 . . frame.pc = x 514 . . } else if funcspdelta(f, frame.pc) == 0 { 515 . . frame.lr = x 516 . . } 517 . . } 518 . . 519 10ms 2.81s u.resolveInternal(false, false) 520 10ms 10ms } 521 . . 522 . . // finishInternal is an unwinder-internal helper called after the stack has been
runtime.(*unwinder).finishInternal
/usr/lib/go/src/runtime/traceback.go
Total: 20ms 20ms (flat, cum) 0.028% 523 . . // exhausted. It sets the unwinder to an invalid state and checks that it 524 . . // successfully unwound the entire stack. 525 20ms 20ms func (u *unwinder) finishInternal() { 526 . . u.frame.pc = 0 527 . . 528 . . // Note that panic != nil is okay here: there can be leftover panics, 529 . . // because the defers on the panic stack do not nest in frame order as 530 . . // they do on the defer stack. If you have:
runtime.tracebackPCs
/usr/lib/go/src/runtime/traceback.go
Total: 0 140ms (flat, cum) 0.2% 624 . . // 625 . . // Callers should set the unwindSilentErrors flag on u. 626 . . func tracebackPCs(u *unwinder, skip int, pcBuf []uintptr) int { 627 . . var cgoBuf [32]uintptr 628 . . n := 0 629 . 70ms for ; n < len(pcBuf) && u.valid(); u.next() { 630 . . f := u.frame.fn 631 . . cgoN := u.cgoCallers(cgoBuf[:]) 632 . . 633 . . // TODO: Why does &u.cache cause u to escape? (Same in traceback2) 634 . 70ms for iu, uf := newInlineUnwinder(f, u.symPC()); n < len(pcBuf) && uf.valid(); uf = iu.next(uf) { 635 . . sf := iu.srcFunc(uf) 636 . . if sf.funcID == abi.FuncIDWrapper && elideWrapperCalling(u.calleeFuncID) { 637 . . // ignore wrappers 638 . . } else if skip > 0 { 639 . . skip--
runtime.callers
/usr/lib/go/src/runtime/traceback.go
Total: 0 150ms (flat, cum) 0.21% 1152 . . func callers(skip int, pcbuf []uintptr) int { 1153 . . sp := sys.GetCallerSP() 1154 . . pc := sys.GetCallerPC() 1155 . . gp := getg() 1156 . . var n int 1157 . 150ms systemstack(func() {
runtime.callers.func1
/usr/lib/go/src/runtime/traceback.go
Total: 0 150ms (flat, cum) 0.21% 1159 . 10ms u.initAt(pc, sp, 0, gp, unwindSilentErrors) 1160 . 140ms n = tracebackPCs(&u, skip, pcbuf) 1161 . . }) 1162 . . return n 1163 . . } 1164 . . 1165 . . func gcallers(gp *g, skip int, pcbuf []uintptr) int {
runtime.isSystemGoroutine
/usr/lib/go/src/runtime/traceback.go
Total: 80ms 150ms (flat, cum) 0.21% 1487 . . // If fixed is true, any goroutine that can vary between user and 1488 . . // system (that is, the finalizer goroutine) is considered a user 1489 . . // goroutine. 1490 . . func isSystemGoroutine(gp *g, fixed bool) bool { 1491 . . // Keep this in sync with internal/trace.IsSystemGoroutine. 1492 70ms 140ms f := findfunc(gp.startpc) 1493 . . if !f.valid() { 1494 . . return false 1495 . . } 1496 10ms 10ms if f.funcID == abi.FuncID_runtime_main || f.funcID == abi.FuncID_corostart || f.funcID == abi.FuncID_handleAsyncEvent { 1497 . . return false 1498 . . } 1499 . . if f.funcID == abi.FuncID_runFinalizers { 1500 . . // We include the finalizer goroutine if it's calling 1501 . . // back into user code.
runtime.isSystemGoroutine
/usr/lib/go/src/runtime/traceback.go
Total: 0 140ms (flat, cum) 0.2% 1514 . . // always consider it a user goroutine. 1515 . . return false 1516 . . } 1517 . . return !gp.runningCleanups.Load() 1518 . . } 1519 . 140ms return stringslite.HasPrefix(funcname(f), "runtime.") return f.datap.funcName(f.nameOff) symtab.go:1149 1520 . . } 1521 . . 1522 . . // SetCgoTraceback records three C functions to use to gather 1523 . . // traceback information from C code and to convert that traceback 1524 . . // information into symbolic information. These are used when printing
runtime.memhash8
/usr/lib/go/src/runtime/alg.go
Total: 0 10ms (flat, cum) 0.014% 34 . . func memhash0(p unsafe.Pointer, h uintptr) uintptr { 35 . . return h 36 . . } 37 . . 38 . . func memhash8(p unsafe.Pointer, h uintptr) uintptr { 39 . 10ms return memhash(p, h, 1) return maps.MemHash(p, h, s) alg.go:82 40 . . } 41 . . 42 . . func memhash16(p unsafe.Pointer, h uintptr) uintptr { 43 . . return memhash(p, h, 2) 44 . . }
runtime.memhash
/usr/lib/go/src/runtime/alg.go
Total: 0 10ms (flat, cum) 0.014% 77 . . // See go.dev/issue/67401. 78 . . // 79 . . //go:nosplit 80 . . //go:linkname memhash 81 . . func memhash(p unsafe.Pointer, h, s uintptr) uintptr { 82 . 10ms return maps.MemHash(p, h, s) 83 . . } 84 . .
runtime.memhash64
/usr/lib/go/src/runtime/alg.go
Total: 20ms 90ms (flat, cum) 0.13% 85 . . //go:nosplit 86 . . func memhash64(p unsafe.Pointer, seed uintptr) uintptr { 87 20ms 90ms return maps.MemHash64(readUnaligned64(p), seed) return byteorder.LEUint64(q[:]) alg.go:401 ⋮ 88 . . } 89 . . 90 . . //go:nosplit 91 . . func memhash32(p unsafe.Pointer, seed uintptr) uintptr { 92 . . return maps.MemHash32(readUnaligned32(p), seed)
runtime.strhash
/usr/lib/go/src/runtime/alg.go
Total: 10ms 90ms (flat, cum) 0.13% 104 . . // Do not remove or change the type signature. 105 . . // See go.dev/issue/67401. 106 . . // 107 . . //go:nosplit 108 . . //go:linkname strhash 109 10ms 10ms func strhash(p unsafe.Pointer, h uintptr) uintptr { 110 . 80ms return maps.StrHash(*(*string)(p), h) return MemHash(unsafe.Pointer(unsafe.StringData(s)), h, uintptr(len(s))) memhash_aes.go:37 111 . . } 112 . . 113 . . // NOTE: Because NaN != NaN, a map can contain any 114 . . // number of (mostly useless) entries keyed with NaNs. 115 . . // To avoid long hash chains, we assign a random number
runtime.interhash
/usr/lib/go/src/runtime/alg.go
Total: 280ms 410ms (flat, cum) 0.57% 147 . . func c128hash(p unsafe.Pointer, h uintptr) uintptr { 148 . . x := (*[2]float64)(p) 149 . . return f64hash(unsafe.Pointer(&x[1]), f64hash(unsafe.Pointer(&x[0]), h)) 150 . . } 151 . . 152 40ms 40ms func interhash(p unsafe.Pointer, h uintptr) uintptr { 153 . . a := (*iface)(p) 154 . . tab := a.tab 155 30ms 30ms if tab == nil { 156 . . return h 157 . . } 158 . . t := tab.Type 159 160ms 160ms if t.Equal == nil { 160 . . // Check hashability here. We could do this check inside 161 . . // typehash, but we want to report the topmost type in 162 . . // the error text (e.g. in a struct with a field of slice type 163 . . // we want to report the struct, not the slice). 164 . . panic(errorString("hash of unhashable type " + toRType(t).string())) 165 . . } 166 10ms 10ms if t.IsDirectIface() { 167 40ms 170ms return trimHash(c1 * typehash(t, unsafe.Pointer(&a.data), h^c0)) 168 . . } else { 169 . . return trimHash(c1 * typehash(t, a.data, h^c0)) 170 . . } 171 . . } 172 . .
runtime.nilinterhash
/usr/lib/go/src/runtime/alg.go
Total: 0 50ms (flat, cum) 0.07% 178 . . // 179 . . // Do not remove or change the type signature. 180 . . // See go.dev/issue/67401. 181 . . // 182 . . //go:linkname nilinterhash 183 . 30ms func nilinterhash(p unsafe.Pointer, h uintptr) uintptr { 184 . . a := (*eface)(p) 185 . . t := a._type 186 . . if t == nil { 187 . . return h 188 . . } 189 . . if t.Equal == nil { 190 . . // See comment in interhash above. 191 . . panic(errorString("hash of unhashable type " + toRType(t).string())) 192 . . } 193 . . if t.IsDirectIface() { 194 . 20ms return trimHash(c1 * typehash(t, unsafe.Pointer(&a.data), h^c0)) 195 . . } else { 196 . . return trimHash(c1 * typehash(t, a.data, h^c0)) 197 . . } 198 . . } 199 . .
runtime.typehash
/usr/lib/go/src/runtime/alg.go
Total: 80ms 150ms (flat, cum) 0.21% 214 . . // 215 . . // Do not remove or change the type signature. 216 . . // See go.dev/issue/67401. 217 . . // 218 . . //go:linkname typehash 219 20ms 20ms func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { 220 . . if t.TFlag&abi.TFlagRegularMemory != 0 { 221 . . // Handle ptr sizes specially, see issue 37086. 222 . . switch t.Size_ { 223 30ms 30ms case 4: 224 . . return memhash32(p, h) 225 . . case 8: 226 30ms 100ms return memhash64(p, h) ⋮ return maps.MemHash64(readUnaligned64(p), seed) alg.go:87 return byteorder.LEUint64(q[:]) alg.go:401 ⋮ 227 . . default: 228 . . return memhash(p, h, t.Size_) 229 . . } 230 . . } 231 . . switch t.Kind() {
runtime.strequal
/usr/lib/go/src/runtime/alg.go
Total: 20ms 30ms (flat, cum) 0.042% 300 . . return *(*complex64)(p) == *(*complex64)(q) 301 . . } 302 . . func c128equal(p, q unsafe.Pointer) bool { 303 . . return *(*complex128)(p) == *(*complex128)(q) 304 . . } 305 20ms 20ms func strequal(p, q unsafe.Pointer) bool { 306 . 10ms return *(*string)(p) == *(*string)(q)
runtime.interequal
/usr/lib/go/src/runtime/alg.go
Total: 90ms 90ms (flat, cum) 0.13% 308 10ms 10ms func interequal(p, q unsafe.Pointer) bool { 309 . . x := *(*iface)(p) 310 . . y := *(*iface)(q) 311 80ms 80ms return x.tab == y.tab && ifaceeq(x.tab, x.data, y.data) 312 . . } 313 . . func nilinterequal(p, q unsafe.Pointer) bool { 314 . . x := *(*eface)(p) 315 . . y := *(*eface)(q) 316 . . return x._type == y._type && efaceeq(x._type, x.data, y.data)
runtime.efaceeq
/usr/lib/go/src/runtime/alg.go
Total: 10ms 30ms (flat, cum) 0.042% 327 . . // Direct interface types are ptr, chan, map, func, and single-element structs/arrays thereof. 328 . . // Maps and funcs are not comparable, so they can't reach here. 329 . . // Ptrs, chans, and single-element items can be compared directly using ==. 330 . . return x == y 331 . . } 332 10ms 30ms return eq(x, y)
runtime.ifaceeq
/usr/lib/go/src/runtime/alg.go
Total: 490ms 490ms (flat, cum) 0.69% 334 170ms 170ms func ifaceeq(tab *itab, x, y unsafe.Pointer) bool { 335 . . if tab == nil { 336 . . return true 337 . . } 338 . . t := tab.Type 339 20ms 20ms eq := t.Equal 340 220ms 220ms if eq == nil { 341 . . panic(errorString("comparing uncomparable type " + toRType(t).string())) 342 . . } 343 70ms 70ms if t.IsDirectIface() { ⋮ return t.TFlag&TFlagDirectIface != 0 type.go:208 344 . . // See comment in efaceeq. 345 10ms 10ms return x == y 346 . . } 347 . . return eq(x, y) 348 . . } 349 . . 350 . . // Testing adapters for hash quality tests (see hash_test.go)
runtime.readUnaligned64
/usr/lib/go/src/runtime/alg.go
Total: 20ms 20ms (flat, cum) 0.028% 396 . . func readUnaligned64(p unsafe.Pointer) uint64 { 397 . . q := (*[8]byte)(p) 398 . . if goarch.BigEndian { 399 . . return byteorder.BEUint64(q[:]) 400 . . } 401 20ms 20ms return byteorder.LEUint64(q[:]) 402 . . }
runtime.(*mspan).initHeapBits
/usr/lib/go/src/runtime/mbitmap.go
Total: 20ms 570ms (flat, cum) 0.8% 511 . . for i := range b { 512 . . b[i] = ^uintptr(0) 513 . . } 514 . . } else if (!s.spanclass.noscan() && heapBitsInSpan(s.elemsize)) || s.isUserArenaChunk { 515 . . b := s.heapBits() 516 . 520ms clear(b) 517 . . } 518 . . if goexperiment.GreenTeaGC && gcUsesSpanInlineMarkBits(s.elemsize) { 519 10ms 40ms s.initInlineMarkBits() s.inlineMarkBits().init(s.spanclass, s.needzero != 0) mgcmark_greenteagc.go:190 ⋮ 520 . . } 521 10ms 10ms } 522 . . 523 . . // heapBits returns the heap ptr/scalar bits stored at the end of the span for 524 . . // small object spans and heap arena spans. 525 . . // 526 . . // Note that the uintptr of each element means something different for small object
runtime.spanHeapBitsRange
/usr/lib/go/src/runtime/mbitmap.go
Total: 20ms 20ms (flat, cum) 0.028% 569 . . //go:nosplit 570 . . func spanHeapBitsRange(spanBase, spanSize, elemsize uintptr) (base, size uintptr) { 571 . . size = spanSize / goarch.PtrSize / 8 572 . . base = spanBase + spanSize - size 573 . . if goexperiment.GreenTeaGC && gcUsesSpanInlineMarkBits(elemsize) { 574 20ms 20ms base -= unsafe.Sizeof(spanInlineMarkBits{}) 575 . . } 576 . . return 577 . . } 578 . . 579 . . // heapBitsSmallForAddr loads the heap bits for the object stored at addr from span.heapBits.
runtime.(*mspan).writeHeapBitsSmall
/usr/lib/go/src/runtime/mbitmap.go
Total: 500ms 500ms (flat, cum) 0.7% 619 . . // 620 . . // Assumes dataSize is <= ptrBits*goarch.PtrSize. x must be a pointer into the span. 621 . . // heapBitsInSpan(dataSize) must be true. dataSize must be >= typ.Size_. 622 . . // 623 . . //go:nosplit 624 10ms 10ms func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize uintptr) { 625 . . // The objects here are always really small, so a single load is sufficient. 626 20ms 20ms src0 := readUintptr(getGCMask(typ)) return t.GCData type.go:92 627 . . 628 . . // Create repetitions of the bitmap if we have a small slice backing store. 629 . . src := src0 630 80ms 80ms if typ.Size_ == goarch.PtrSize { 631 . . src = (1 << (dataSize / goarch.PtrSize)) - 1 632 . . // This object is all pointers, so scanSize is just dataSize. 633 . . scanSize = dataSize 634 . . } else { 635 . . // N.B. We rely on dataSize being an exact multiple of the type size. 636 . . // The alternative is to be defensive and mask out src to the length 637 . . // of dataSize. The purpose is to save on one additional masking operation. 638 . . if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { 639 . . throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") 640 . . } 641 . . scanSize = typ.PtrBytes 642 50ms 50ms for i := typ.Size_; i < dataSize; i += typ.Size_ { 643 50ms 50ms src |= src0 << (i / goarch.PtrSize) 644 10ms 10ms scanSize += typ.Size_ 645 . . } 646 . . if asanenabled { 647 . . // Mask src down to dataSize. dataSize is going to be a strange size because of 648 . . // the redzone required for allocations when asan is enabled. 649 . . src &= (1 << (dataSize / goarch.PtrSize)) - 1 650 . . } 651 . . } 652 . . 653 . . // Since we're never writing more than one uintptr's worth of bits, we're either going 654 . . // to do one or two writes. 655 50ms 50ms dstBase, _ := spanHeapBitsRange(span.base(), pageSize, span.elemsize) return s.startAddr mheap.go:519 ⋮ base -= unsafe.Sizeof(spanInlineMarkBits{}) mbitmap.go:574 ⋮ return s.startAddr mheap.go:519 656 . . dst := unsafe.Pointer(dstBase) 657 . . o := (x - span.base()) / goarch.PtrSize 658 . . i := o / ptrBits 659 . . j := o % ptrBits 660 20ms 20ms bits := span.elemsize / goarch.PtrSize 661 . . if j+bits > ptrBits { 662 . . // Two writes. 663 30ms 30ms bits0 := ptrBits - j 664 . . bits1 := bits - bits0 665 . . dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) 666 40ms 40ms dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) return unsafe.Pointer(uintptr(p) + x) stubs.go:25 667 . . *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) 668 . . *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0) 669 . . } else { 670 . . // One write. 671 110ms 110ms dst := (*uintptr)(add(dst, i*goarch.PtrSize)) 672 . . *dst = (*dst)&^(((1<<bits)-1)<<j) | (src << j) 673 . . } 674 . . 675 . . const doubleCheck = false 676 . . if doubleCheck { 677 . . srcRead := span.heapBitsSmallForAddr(x) 678 . . if srcRead != src { 679 . . print("runtime: x=", hex(x), " i=", i, " j=", j, " bits=", bits, "\n") 680 . . print("runtime: dataSize=", dataSize, " typ.Size_=", typ.Size_, " typ.PtrBytes=", typ.PtrBytes, "\n") 681 . . print("runtime: src0=", hex(src0), " src=", hex(src), " srcRead=", hex(srcRead), "\n") 682 . . throw("bad pointer bits written for small object") 683 . . } 684 . . } 685 30ms 30ms return 686 . . } 687 . . 688 . . // heapSetType* functions record that the new allocation [x, x+size) 689 . . // holds in [x, x+dataSize) one or more values of type typ. 690 . . // (The number of values is given by dataSize / typ.Size.)
runtime.heapSetTypeNoHeader
/usr/lib/go/src/runtime/mbitmap.go
Total: 60ms 560ms (flat, cum) 0.78% 706 . . 707 . . func heapSetTypeNoHeader(x, dataSize uintptr, typ *_type, span *mspan) uintptr { 708 . . if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(span.elemsize)) { 709 . . throw("tried to write heap bits, but no heap bits in span") 710 . . } 711 60ms 560ms scanSize := span.writeHeapBitsSmall(x, dataSize, typ) 712 . . if doubleCheckHeapSetType { 713 . . doubleCheckHeapType(x, dataSize, typ, nil, span) 714 . . } 715 . . return scanSize 716 . . }
runtime.heapSetTypeSmallHeader
/usr/lib/go/src/runtime/mbitmap.go
Total: 80ms 80ms (flat, cum) 0.11% 734 . . // 735 . . // See go.dev/issue/74375 for details of a similar issue in 736 . . // spanInlineMarkBits. 737 . . throw("runtime: pointer to heap type header nil?") 738 . . } 739 80ms 80ms *header = typ 740 . . if doubleCheckHeapSetType { 741 . . doubleCheckHeapType(x, dataSize, typ, header, span) 742 . . } 743 . . return span.elemsize 744 . . }
runtime.addb
/usr/lib/go/src/runtime/mbitmap.go
Total: 10ms 10ms (flat, cum) 0.014% 1029 . . //go:nosplit 1030 . . func addb(p *byte, n uintptr) *byte { 1031 . . // Note: wrote out full expression instead of calling add(p, n) 1032 . . // to reduce the number of temporaries generated by the 1033 . . // compiler for this trivial expression during inlining. 1034 10ms 10ms return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) 1035 . . } 1036 . . 1037 . . // subtractb returns the byte pointer p-n. 1038 . . // 1039 . . //go:nowritebarrier
runtime.(*mspan).refillAllocCache
/usr/lib/go/src/runtime/mbitmap.go
Total: 180ms 180ms (flat, cum) 0.25% 1093 . . // refillAllocCache takes 8 bytes s.allocBits starting at whichByte 1094 . . // and negates them so that ctz (count trailing zeros) instructions 1095 . . // can be used. It then places these 8 bytes into the cached 64 bit 1096 . . // s.allocCache. 1097 . . func (s *mspan) refillAllocCache(whichByte uint16) { 1098 160ms 160ms bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(uintptr(whichByte)))) 1099 . . aCache := uint64(0) 1100 . . aCache |= uint64(bytes[0]) 1101 . . aCache |= uint64(bytes[1]) << (1 * 8) 1102 . . aCache |= uint64(bytes[2]) << (2 * 8) 1103 . . aCache |= uint64(bytes[3]) << (3 * 8) 1104 . . aCache |= uint64(bytes[4]) << (4 * 8) 1105 . . aCache |= uint64(bytes[5]) << (5 * 8) 1106 . . aCache |= uint64(bytes[6]) << (6 * 8) 1107 . . aCache |= uint64(bytes[7]) << (7 * 8) 1108 20ms 20ms s.allocCache = ^aCache 1109 . . } 1110 . . 1111 . . // nextFreeIndex returns the index of the next free object in s at
runtime.(*mspan).nextFreeIndex
/usr/lib/go/src/runtime/mbitmap.go
Total: 20ms 240ms (flat, cum) 0.34% 1112 . . // or after s.freeindex. 1113 . . // There are hardware instructions that can be used to make this 1114 . . // faster if profiling warrants it. 1115 20ms 240ms func (s *mspan) nextFreeIndex() uint16 { 1116 . . sfreeindex := s.freeindex 1117 . . snelems := s.nelems 1118 . . if sfreeindex == snelems { 1119 . . return sfreeindex 1120 . . }
runtime.(*mspan).nextFreeIndex
/usr/lib/go/src/runtime/mbitmap.go
Total: 0 150ms (flat, cum) 0.21% 1154 . . // As each 1 in s.allocCache was encountered and used for allocation 1155 . . // it was shifted away. At this point s.allocCache contains all 0s. 1156 . . // Refill s.allocCache so that it corresponds 1157 . . // to the bits at s.allocBits starting at s.freeindex. 1158 . . whichByte := sfreeindex / 8 1159 . 150ms s.refillAllocCache(whichByte) 1160 . . } 1161 . . s.freeindex = sfreeindex 1162 . . return result 1163 . . } 1164 . .
runtime.typeBitsBulkBarrier
/usr/lib/go/src/runtime/mbitmap.go
Total: 10ms 10ms (flat, cum) 0.014% 1459 . . // and the GC must observe them as an atomic action. 1460 . . // 1461 . . // Callers must perform cgo checks if goexperiment.CgoCheck2. 1462 . . // 1463 . . //go:nosplit 1464 10ms 10ms func typeBitsBulkBarrier(typ *_type, dst, src, size uintptr) { 1465 . . if typ == nil { 1466 . . throw("runtime: typeBitsBulkBarrier without type") 1467 . . } 1468 . . if typ.Size_ != size { 1469 . . println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size)
runtime.(*mspan).countAlloc
/usr/lib/go/src/runtime/mbitmap.go
Total: 20ms 20ms (flat, cum) 0.028% 1504 . . for i := uintptr(0); i < bytes; i += 8 { 1505 . . // Extract 64 bits from the byte pointer and get a OnesCount. 1506 . . // Note that the unsafe cast here doesn't preserve endianness, 1507 . . // but that's OK. We only care about how many bits are 1, not 1508 . . // about the order we discover them in. 1509 20ms 20ms mrkBits := *(*uint64)(unsafe.Pointer(s.gcmarkBits.bytep(i))) 1510 . . count += sys.OnesCount64(mrkBits) 1511 . . } 1512 . . return count 1513 . . }
runtime.readUintptr
/usr/lib/go/src/runtime/mbitmap.go
Total: 10ms 10ms (flat, cum) 0.014% 1514 . . 1515 . . // Read the bytes starting at the aligned pointer p into a uintptr. 1516 . . // Read is little-endian. 1517 . . func readUintptr(p *byte) uintptr { 1518 10ms 10ms x := *(*uintptr)(unsafe.Pointer(p)) 1519 . . if goarch.BigEndian { 1520 . . if goarch.PtrSize == 8 { 1521 . . return uintptr(sys.Bswap64(uint64(x))) 1522 . . } 1523 . . return uintptr(sys.Bswap32(uint32(x)))
slices.Index[go.shape.[]*git.urbach.dev/cli/q/src/codegen.Step,go.shape.*git.urbach.dev/cli/q/src/codegen.Step]
/usr/lib/go/src/slices/slices.go
Total: 230ms 230ms (flat, cum) 0.32% 92 . . } 93 . . 94 . . // Index returns the index of the first occurrence of v in s, 95 . . // or -1 if not present. 96 . . func Index[S ~[]E, E comparable](s S, v E) int { 97 230ms 230ms for i := range s {
slices.Index[go.shape.[]go.shape.*git.urbach.dev/cli/q/src/core.Function,go.shape.*git.urbach.dev/cli/q/src/core.Function]
/usr/lib/go/src/slices/slices.go
Total: 340ms 370ms (flat, cum) 0.52% 98 340ms 370ms if v == s[i] { 99 . . return i 100 . . } 101 . . } 102 . . return -1
slices.IndexFunc[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }]
/usr/lib/go/src/slices/slices.go
Total: 90ms 110ms (flat, cum) 0.15% 104 . . 105 . . // IndexFunc returns the first index i satisfying f(s[i]), 106 . . // or -1 if none do. 107 . . func IndexFunc[S ~[]E, E any](s S, f func(E) bool) int { 108 40ms 40ms for i := range s { 109 50ms 70ms if f(s[i]) { 110 . . return i 111 . . } 112 . . } 113 . . return -1
slices.Contains[go.shape.[]go.shape.*git.urbach.dev/cli/q/src/core.Function,go.shape.*git.urbach.dev/cli/q/src/core.Function]
/usr/lib/go/src/slices/slices.go
Total: 630ms 660ms (flat, cum) 0.92% 114 . . } 115 . . 116 . . // Contains reports whether v is present in s. 117 . . func Contains[S ~[]E, E comparable](s S, v E) bool { 118 630ms 660ms return Index(s, v) >= 0 if v == s[i] { slices.go:98 ⋮ ⋮ ⋮ for i := range s { slices.go:97 ⋮ ⋮ if v == s[i] { slices.go:98 ⋮ ⋮ ⋮ ⋮ ⋮ if v == s[i] { slices.go:98 ⋮ for i := range s { slices.go:97 ⋮ if v == s[i] { slices.go:98 ⋮ ⋮ ⋮ ⋮ ⋮ for i := range s { slices.go:97 ⋮ if v == s[i] { slices.go:98 ⋮ for i := range s { slices.go:97 ⋮ if v == s[i] { slices.go:98 ⋮ 119 . . } 120 . . 121 . . // ContainsFunc reports whether at least one 122 . . // element e of s satisfies f(e). 123 . . // It stops as soon as a call to f returns true.
slices.Insert[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }]
/usr/lib/go/src/slices/slices.go
Total: 30ms 30ms (flat, cum) 0.042% 135 . . // If the result is empty, it has the same nilness as s. 136 . . func Insert[S ~[]E, E any](s S, i int, v ...E) S { 137 . . _ = s[i:] // bounds check 138 . . 139 . . m := len(v) 140 10ms 10ms if m == 0 { 141 . . return s 142 . . } 143 . . n := len(s) 144 . . if i == n { 145 10ms 10ms return append(s, v...) 146 . . } 147 10ms 10ms if n+m > cap(s) { 148 . . // Use append rather than make so that we bump the size of 149 . . // the slice up to the next storage class. 150 . . // This is what Grow does but we don't call Grow because 151 . . // that might copy the values twice. 152 . . s2 := append(s[:i], make(S, n+m-i)...)
slices.Insert[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }]
/usr/lib/go/src/slices/slices.go
Total: 0 20ms (flat, cum) 0.028% 173 . . if !overlaps(v, s[i+m:]) { 174 . . // Easy case - v does not overlap either the c or d regions. 175 . . // (It might be in some of a or b, or elsewhere entirely.) 176 . . // The data we copy up doesn't write to v at all, so just do it. 177 . . 178 . 20ms copy(s[i+m:], s[i:]) 179 . . 180 . . // Now we have 181 . . // s: aaaaaaaabbbbbbbbcccccccc 182 . . // ^ ^ ^ ^ 183 . . // i i+m n n+m
slices.DeleteFunc[go.shape.[]*git.urbach.dev/cli/q/src/ssa.Block,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 20ms 20ms (flat, cum) 0.028% 236 . . 237 . . // DeleteFunc modifies s in place by removing any elements for which del returns true, 238 . . // and returns the modified slice. 239 . . // DeleteFunc zeroes the elements between the new length and the original length. 240 . . // If the result is empty, it has the same nilness as s. 241 20ms 20ms func DeleteFunc[S ~[]E, E any](s S, del func(E) bool) S {
slices.DeleteFunc[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }]
/usr/lib/go/src/slices/slices.go
Total: 90ms 110ms (flat, cum) 0.15% 242 90ms 110ms i := IndexFunc(s, del) if f(s[i]) { slices.go:109 ⋮ for i := range s { slices.go:108 ⋮ if f(s[i]) { slices.go:109 ⋮ for i := range s { slices.go:108 ⋮ if f(s[i]) { slices.go:109 ⋮
slices.DeleteFunc[go.shape.[]*git.urbach.dev/cli/q/src/ssa.Block,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 40ms 40ms (flat, cum) 0.056% 244 40ms 40ms return s 245 . . }
slices.DeleteFunc[go.shape.[]git.urbach.dev/cli/q/src/ssa.Value,go.shape.interface { AddUser; Equals bool; Inputs []git.urbach.dev/cli/q/src/ssa.Value; IsPure bool; RemoveUser; Replace; String string; Type git.urbach.dev/cli/q/src/types.Type; Users []git.urbach.dev/cli/q/src/ssa.Value }]
/usr/lib/go/src/slices/slices.go
Total: 10ms 10ms (flat, cum) 0.014% 247 . . for j := i + 1; j < len(s); j++ { 248 10ms 10ms if v := s[j]; !del(v) { 249 . . s[i] = v 250 . . i++
slices.DeleteFunc[go.shape.[]*git.urbach.dev/cli/q/src/ssa.Block,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 0 30ms (flat, cum) 0.042% 251 . . } 252 . . } 253 . 30ms clear(s[i:]) // zero/nil out the obsolete elements, for GC 254 . . return s[:i] 255 . . } 256 . . 257 . . // Replace modifies s in place by replacing the elements s[i:j] with the given v, 258 . . // and returns the modified slice.
slices.Clone[go.shape.[]*git.urbach.dev/cli/q/src/ssa.Block,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 0 40ms (flat, cum) 0.056% 357 . . if s == nil { 358 . . return nil 359 . . } 360 . . // Avoid s[:0:0] as it leads to unwanted liveness when cloning a 361 . . // zero-length slice of a large array; see https://go.dev/issue/68488. 362 . 40ms return append(S{}, s...) 363 . . } 364 . . 365 . . // Compact replaces consecutive runs of equal elements with a single copy. 366 . . // This is like the uniq command found on Unix. 367 . . // Compact modifies the contents of the slice s and returns the modified slice,
slices.Grow[go.shape.[]*git.urbach.dev/cli/q/src/codegen.Step,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 0 110ms (flat, cum) 0.15% 423 . . if n < 0 { 424 . . panic("cannot be negative") 425 . . } 426 . . if n -= cap(s) - len(s); n > 0 { 427 . . // This expression allocates only once (see test). 428 . 110ms s = append(s[:cap(s)], make([]E, n)...)[:len(s)] 429 . . } 430 . . return s 431 . . } 432 . . 433 . . // Clip removes unused capacity from the slice, returning s[:len(s):len(s)].
slices.Concat[go.shape.[]*git.urbach.dev/cli/q/src/codegen.Step,go.shape.*uint8]
/usr/lib/go/src/slices/slices.go
Total: 30ms 200ms (flat, cum) 0.28% 490 . . // If the concatenation is empty, the result is nil. 491 . . func Concat[S ~[]E, E any](slices ...S) S { 492 . . size := 0 493 . . for _, s := range slices { 494 . . size += len(s) 495 10ms 10ms if size < 0 { 496 . . panic("len out of range") 497 . . } 498 . . } 499 . . // Use Grow, not make, to round up to the size class: 500 . . // the extra space is otherwise unused and helps 501 . . // callers that append a few elements to the result. 502 . 110ms newslice := Grow[S](nil, size) s = append(s[:cap(s)], make([]E, n)...)[:len(s)] slices.go:428 ⋮ ⋮ 503 . . for _, s := range slices { 504 20ms 80ms newslice = append(newslice, s...) 505 . . } 506 . . return newslice 507 . . } 508 . . 509 . . // Repeat returns a new slice that repeats the provided slice the given number of times.
internal/runtime/maps.memHashAES
/usr/lib/go/src/internal/runtime/maps/memhash_arm64.s
Total: 770ms 770ms (flat, cum) 1.08% 44 . . TEXT ·memHashAES<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32 45 . . // R0: data 46 . . // R1: seed data 47 . . // R2: length 48 . . // At return, R0 = return value 49 30ms 30ms VEOR V30.B16, V30.B16, V30.B16 50 . . VMOV R1, V30.D[0] 51 10ms 10ms VMOV R2, V30.D[1] // load length into seed 52 . . 53 30ms 30ms MOVD $·aeskeysched+0(SB), R4 54 . . VLD1.P 16(R4), [V0.B16] 55 40ms 40ms AESE V30.B16, V0.B16 56 . . AESMC V0.B16, V0.B16 57 10ms 10ms CMP $16, R2 58 10ms 10ms BLO aes0to15 59 . . BEQ aes16 60 10ms 10ms CMP $32, R2 61 . . BLS aes17to32 62 . . CMP $64, R2 63 . . BLS aes33to64 64 . . CMP $128, R2 65 . . BLS aes65to128 66 . . B aes129plus 67 . . 68 . . aes0to15: 69 20ms 20ms CBZ R2, aes0 70 10ms 10ms VEOR V2.B16, V2.B16, V2.B16 71 . . TBZ $3, R2, less_than_8 72 . . VLD1.P 8(R0), V2.D[0] 73 . . 74 . . less_than_8: 75 100ms 100ms TBZ $2, R2, less_than_4 76 30ms 30ms VLD1.P 4(R0), V2.S[2] 77 . . 78 . . less_than_4: 79 130ms 130ms TBZ $1, R2, less_than_2 80 30ms 30ms VLD1.P 2(R0), V2.H[6] 81 . . 82 . . less_than_2: 83 100ms 100ms TBZ $0, R2, done 84 30ms 30ms VLD1 (R0), V2.B[14] 85 . . done: 86 50ms 50ms AESE V0.B16, V2.B16 87 . . AESMC V2.B16, V2.B16 88 40ms 40ms AESE V0.B16, V2.B16 89 . . AESMC V2.B16, V2.B16 90 . . AESE V0.B16, V2.B16 91 . . AESMC V2.B16, V2.B16 92 . . 93 . . VMOV V2.D[0], R0 94 50ms 50ms RET 95 . . 96 . . aes0: 97 . . VMOV V0.D[0], R0 98 . . RET 99 . . 100 . . aes16: 101 . . VLD1 (R0), [V2.B16] 102 . . B done 103 . . 104 . . aes17to32: 105 . . // make second seed 106 . . VLD1 (R4), [V1.B16] 107 . . AESE V30.B16, V1.B16 108 . . AESMC V1.B16, V1.B16 109 . . SUB $16, R2, R10 110 . . VLD1.P (R0)(R10), [V2.B16] 111 30ms 30ms VLD1 (R0), [V3.B16] 112 . . 113 10ms 10ms AESE V0.B16, V2.B16 114 . . AESMC V2.B16, V2.B16 115 . . AESE V1.B16, V3.B16 116 . . AESMC V3.B16, V3.B16 117 . . 118 . . AESE V0.B16, V2.B16
internal/runtime/maps.bitset.first
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 30ms 30ms (flat, cum) 0.042% 45 . . // first returns the relative index of the first control byte in the group that 46 . . // is in the set. 47 . . // 48 . . // Preconditions: b is not 0 (empty). 49 . . func (b bitset) first() uintptr { 50 30ms 30ms return bitsetFirst(b) return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 group.go:58 ⋮ ⋮ 51 . . } 52 . . 53 . . // Portable implementation of first.
internal/runtime/maps.bitsetFirst
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 30ms 30ms (flat, cum) 0.042% 55 . . // On AMD64, this is replaced with an intrisic that simply does 56 . . // TrailingZeros64. There is no need to shift as the bitset is packed. 57 . . func bitsetFirst(b bitset) uintptr { 58 30ms 30ms return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 59 . . } 60 . .
internal/runtime/maps.bitset.removeFirst
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 10ms 10ms (flat, cum) 0.014% 62 . . // set bit to 0). 63 . . func (b bitset) removeFirst() bitset { 64 10ms 10ms return b & (b - 1) 65 . . } 66 . . 67 . . // removeBelow clears all set bits below slot i (non-inclusive). 68 . . func (b bitset) removeBelow(i uintptr) bitset { 69 . . return bitsetRemoveBelow(b, i)
internal/runtime/maps.(*ctrlGroup).set
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 40ms 40ms (flat, cum) 0.056% 138 . . func (g *ctrlGroup) set(i uintptr, c ctrl) { 139 . . if goarch.BigEndian { 140 . . *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) = c 141 . . return 142 . . } 143 40ms 40ms *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c 144 . . } 145 . .
internal/runtime/maps.(*ctrlGroup).setEmpty
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 80ms 80ms (flat, cum) 0.11% 146 . . // setEmpty sets all the control bytes to empty. 147 . . func (g *ctrlGroup) setEmpty() { 148 80ms 80ms *g = ctrlGroup(bitsetEmpty) 149 . . } 150 . .
internal/runtime/maps.ctrlGroup.matchH2
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 120ms 120ms (flat, cum) 0.17% 152 . . // matches the given value. May return false positives. 153 . . func (g ctrlGroup) matchH2(h uintptr) bitset { 154 120ms 120ms return ctrlGroupMatchH2(g, h) v = ^v group.go:164 ⋮ ⋮ return bitset((v&bitsetL7B + bitsetLSB) & (v & bitsetMSB)) group.go:165 ⋮ v = ^v group.go:164 ⋮ 155 . . } 156 . . 157 . . // Portable implementation of matchH2. 158 . . //
internal/runtime/maps.ctrlGroupMatchH2
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 110ms 110ms (flat, cum) 0.15% 160 . . // note on bitset about the packed intrinsified return value. 161 . . func ctrlGroupMatchH2(g ctrlGroup, h uintptr) bitset { 162 . . v := uint64(g) ^ (bitsetLSB * uint64(h)) 163 . . if goarch.IsArm64 == 1 { 164 90ms 90ms v = ^v 165 20ms 20ms return bitset((v&bitsetL7B + bitsetLSB) & (v & bitsetMSB)) 166 . . } 167 . . // NB: This generic matching routine produces false positive matches when 168 . . // h is 2^N and the control bytes have a seq of 2^N followed by 2^N+1. For 169 . . // example: if ctrls==0x0302 and h=02, we'll compute v as 0x0100. When we 170 . . // subtract off 0x0101 the first 2 bytes we'll become 0xffff and both be
internal/runtime/maps.ctrlGroup.matchEmpty
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 10ms 10ms (flat, cum) 0.014% 175 . . return bitset(((v - bitsetLSB) &^ v) & bitsetMSB) 176 . . } 177 . . 178 . . // matchEmpty returns the set of slots in the group that are empty. 179 . . func (g ctrlGroup) matchEmpty() bitset { 180 10ms 10ms return ctrlGroupMatchEmpty(g) return bitset((v &^ (v << 6)) & bitsetMSB) group.go:195 181 . . } 182 . . 183 . . // Portable implementation of matchEmpty. 184 . . // 185 . . // Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See
internal/runtime/maps.ctrlGroupMatchEmpty
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 10ms 10ms (flat, cum) 0.014% 190 . . // A full slot is 0??? ???? 191 . . // 192 . . // A slot is empty iff bit 7 is set and bit 1 is not. We could select any 193 . . // of the other bits here (e.g. v << 1 would also work). 194 . . v := uint64(g) 195 10ms 10ms return bitset((v &^ (v << 6)) & bitsetMSB) 196 . . } 197 . .
internal/runtime/maps.ctrlGroup.matchEmptyOrDeleted
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 20ms 20ms (flat, cum) 0.028% 199 . . // deleted. 200 . . func (g ctrlGroup) matchEmptyOrDeleted() bitset { 201 20ms 20ms return ctrlGroupMatchEmptyOrDeleted(g) 202 . . } 203 . . 204 . . // Portable implementation of matchEmptyOrDeleted. 205 . . // 206 . . // Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See
internal/runtime/maps.(*groupReference).ctrls
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 50ms 50ms (flat, cum) 0.07% 289 . . return v, false 290 . . } 291 . . 292 . . // ctrls returns the group control word. 293 . . func (g *groupReference) ctrls() *ctrlGroup { 294 50ms 50ms return (*ctrlGroup)(g.data) 295 . . } 296 . .
internal/runtime/maps.(*groupReference).key
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 230ms 230ms (flat, cum) 0.32% 297 . . // key returns a pointer to the key at index i. 298 . . func (g *groupReference) key(typ *abi.MapType, i uintptr) unsafe.Pointer { 299 230ms 230ms offset := typ.KeysOff + i*typ.KeyStride 300 . . 301 . . return unsafe.Pointer(uintptr(g.data) + offset) 302 . . }
internal/runtime/maps.(*groupReference).elem
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 110ms 110ms (flat, cum) 0.15% 303 . . 304 . . // elem returns a pointer to the element at index i. 305 . . func (g *groupReference) elem(typ *abi.MapType, i uintptr) unsafe.Pointer { 306 50ms 50ms offset := typ.ElemsOff + i*typ.ElemStride 307 . . 308 60ms 60ms return unsafe.Pointer(uintptr(g.data) + offset) 309 . . } 310 . . 311 . . // groupsReference is a wrapper type describing an array of groups stored at 312 . . // data. 313 . . type groupsReference struct {
internal/runtime/maps.newGroups
/usr/lib/go/src/internal/runtime/maps/group.go
Total: 0 1.69s (flat, cum) 2.36% 325 . . // 326 . . // Length must be a power of two. 327 . . func newGroups(typ *abi.MapType, length uint64) groupsReference { 328 . . return groupsReference{ 329 . . // TODO: make the length type the same throughout. 330 . 1.69s data: newarray(typ.Group, int(length)), 331 . . lengthMask: length - 1, 332 . . } 333 . . } 334 . . 335 . . // group returns the group at index i.
git.urbach.dev/cli/q/src/token.Tokenize
/home/user/q/src/token/Tokenize.go
Total: 640ms 1.88s (flat, cum) 2.63% 1 . . package token 2 . . 3 . . // Tokenize turns the file contents into a list of tokens. 4 10ms 10ms func Tokenize(buffer []byte) List { 5 . . var ( 6 . . i Position 7 . 430ms tokens = make(List, 0, 8+len(buffer)/2) 8 . . ) 9 . . 10 40ms 40ms for i < Position(len(buffer)) { 11 260ms 260ms switch buffer[i] { 12 30ms 30ms case ' ', '\t', '\r': 13 10ms 10ms case ',': 14 . . tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1}) 15 . . case '(': 16 . . tokens = append(tokens, Token{Kind: GroupStart, Position: i, Length: 1}) 17 10ms 10ms case ')': 18 10ms 10ms tokens = append(tokens, Token{Kind: GroupEnd, Position: i, Length: 1}) 19 10ms 10ms case '{': 20 20ms 20ms tokens = append(tokens, Token{Kind: BlockStart, Position: i, Length: 1}) 21 10ms 10ms case '}': 22 20ms 20ms tokens = append(tokens, Token{Kind: BlockEnd, Position: i, Length: 1}) 23 . . case '[': 24 10ms 10ms tokens = append(tokens, Token{Kind: ArrayStart, Position: i, Length: 1}) 25 10ms 10ms case ']': 26 . . tokens = append(tokens, Token{Kind: ArrayEnd, Position: i, Length: 1}) 27 . . case '\n': 28 10ms 10ms tokens = append(tokens, Token{Kind: NewLine, Position: i, Length: 1}) 29 . . case '-': 30 10ms 30ms tokens, i = dash(tokens, buffer, i) 31 30ms 30ms case '/': 32 . . tokens, i = slash(tokens, buffer, i) 33 . . continue 34 . . case '"', '\'': 35 . 20ms tokens, i = quote(tokens, buffer, i) 36 . . continue 37 10ms 10ms case '0': 38 20ms 70ms tokens, i = zero(tokens, buffer, i) 39 . . continue 40 . . case '#': 41 . . tokens, i = hash(tokens, buffer, i) 42 . . continue 43 . . default: 44 10ms 10ms if isIdentifierStart(buffer[i]) { return isLetter(c) || c == '_' identifier.go:66 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') identifier.go:72 45 50ms 560ms tokens, i = identifier(tokens, buffer, i) 46 . . continue 47 . . } 48 . . 49 . . if isDigit(buffer[i]) { 50 10ms 50ms tokens, i = digit(tokens, buffer, i) 51 . . continue 52 . . } 53 . . 54 30ms 30ms if isOperator(buffer[i]) { case '=', ':', '.', '+', '-', '*', '/', '<', '>', '&', '|', '^', '%', '!': operator.go:40 ⋮ 55 . 170ms tokens, i = operator(tokens, buffer, i) 56 . . continue 57 . . } 58 . . 59 . . tokens = append(tokens, Token{Kind: Invalid, Position: i, Length: 1}) 60 . . } 61 . . 62 . . i++ 63 . . } 64 . . 65 10ms 10ms tokens = append(tokens, Token{Kind: EOF, Position: i, Length: 0}) 66 . . return tokens 67 . . }
git.urbach.dev/cli/q/src/core.(*Function).deleteResources
/home/user/q/src/core/deleteResources.go
Total: 110ms 1.85s (flat, cum) 2.59% 9 . . 10 . . // deleteResources inserts delete calls for all resources. 11 . . func (f *Function) deleteResources(filter map[string]ssa.Value) { 12 . . var ( 13 . . names []string 14 20ms 20ms identifiers = f.Block().Identifiers.After return ir.Blocks[len(ir.Blocks)-1] IR.go:27 15 . . ) 16 . . 17 . . if len(identifiers) == 0 { 18 . . return 19 . . } 20 . . 21 70ms 260ms for name := range identifiers { 22 10ms 10ms if filter != nil { 23 . 120ms _, inherited := filter[name] 24 . . 25 . . if inherited { 26 . . continue 27 . . } 28 . . } 29 . . 30 . . if names == nil { 31 . 130ms names = make([]string, 0, len(identifiers)) 32 . . } 33 . . 34 10ms 10ms names = append(names, name) 35 . . } 36 . . 37 . 1.30s slices.SortFunc(names, func(a string, b string) int { pdqsortCmpFunc(x, 0, n, bits.Len(uint(n)), cmp) sort.go:32
git.urbach.dev/cli/q/src/core.(*Function).deleteResources.func1
/home/user/q/src/core/deleteResources.go
Total: 450ms 1.19s (flat, cum) 1.66% 38 . 280ms aValue := identifiers[a] 39 10ms 240ms bValue := identifiers[b] 40 . . 41 30ms 30ms aStruct, aIsStruct := aValue.(*ssa.Struct) 42 . . 43 10ms 10ms if aIsStruct && len(aStruct.Arguments) > 0 { 44 . . aValue = aStruct.Arguments[0] 45 . . } 46 . . 47 . . bStruct, bIsStruct := bValue.(*ssa.Struct) 48 . . 49 10ms 10ms if bIsStruct && len(bStruct.Arguments) > 0 { 50 . . bValue = bStruct.Arguments[0] 51 . . } 52 . . 53 20ms 20ms for _, block := range f.IR.Blocks { 54 190ms 340ms aIndex := block.Index(aValue) for i, value := range b.Instructions { Block.go:77 ⋮ if value == search { Block.go:78 ⋮ ⋮ for i, value := range b.Instructions { Block.go:77 ⋮ if value == search { Block.go:78 ⋮ for i, value := range b.Instructions { Block.go:77 ⋮ if value == search { Block.go:78 ⋮ ⋮ for i, value := range b.Instructions { Block.go:77 ⋮ ⋮ ⋮ if value == search { Block.go:78 55 140ms 220ms bIndex := block.Index(bValue) for i, value := range b.Instructions { Block.go:77 ⋮ ⋮ ⋮ if value == search { Block.go:78 ⋮ for i, value := range b.Instructions { Block.go:77 ⋮ if value == search { Block.go:78 ⋮ ⋮ for i, value := range b.Instructions { Block.go:77 ⋮ ⋮ 56 . . 57 . . switch { 58 10ms 10ms case aIndex != -1 && bIndex != -1: 59 10ms 10ms return bIndex - aIndex 60 . . case aIndex != -1: 61 . . return 1 62 10ms 10ms case bIndex != -1: 63 10ms 10ms return -1 64 . . } 65 . . } 66 . .
git.urbach.dev/cli/q/src/core.(*Function).deleteResources
/home/user/q/src/core/deleteResources.go
Total: 50ms 360ms (flat, cum) 0.5% 67 . . panic("non-deterministic memory deallocation order") 68 . . }) 69 . . 70 10ms 10ms for _, name := range names { 71 10ms 140ms value := identifiers[name] 72 . . 73 10ms 10ms if value == nil { 74 . . continue 75 . . } 76 . . 77 10ms 120ms resource, isResource := value.Type().(*types.Resource) 78 . . 79 . . if !isResource { 80 . . continue 81 . . } 82 . . 83 . . _, isPointer := resource.Of.(*types.Pointer) 84 . . _, isStruct := resource.Of.(*types.Struct) 85 . . 86 . . if !isPointer && !isStruct { 87 . . continue 88 . . } 89 . . 90 . . _, isParam := value.(*ssa.Parameter) 91 . . 92 . . if isParam { 93 . . continue 94 . . } 95 . . 96 . 70ms f.delete(value) 97 . . } 98 10ms 10ms }
runtime.mapaccess1
/usr/lib/go/src/internal/runtime/maps/runtime.go
Total: 10ms 980ms (flat, cum) 1.37% 52 . . // the key is not in the map. 53 . . // NOTE: The returned pointer may keep the whole map live, so don't 54 . . // hold onto it for very long. 55 . . // 56 . . //go:linkname runtime_mapaccess1 runtime.mapaccess1 57 10ms 230ms func runtime_mapaccess1(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { 58 . 750ms p, _ := runtime_mapaccess2(typ, m, key) 59 . . return p 60 . . }
runtime.mapaccess2
/usr/lib/go/src/internal/runtime/maps/runtime.go
Total: 260ms 830ms (flat, cum) 1.16% 61 . . 62 . . //go:linkname runtime_mapaccess2 runtime.mapaccess2 63 20ms 260ms func runtime_mapaccess2(typ *abi.MapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { 64 . . if race.Enabled && m != nil { 65 . . callerpc := sys.GetCallerPC() 66 . . pc := abi.FuncPCABIInternal(runtime_mapaccess2) 67 . . race.ReadPC(unsafe.Pointer(m), callerpc, pc) 68 . . race.ReadObjectPC(typ.Key, key, callerpc, pc) 69 . . } 70 . . if msan.Enabled && m != nil { 71 . . msan.Read(key, typ.Key.Size_) 72 . . } 73 . . if asan.Enabled && m != nil { 74 . . asan.Read(key, typ.Key.Size_) 75 . . } 76 . . 77 20ms 20ms if m == nil || m.Used() == 0 { 78 20ms 30ms if err := mapKeyError(typ, key); err != nil { return mapKeyError2(t.Key, p) map.go:846 ⋮ if !t.HashMightPanic() { map.go:843 79 . . panic(err) // see issue 23734 80 . . } 81 . . return unsafe.Pointer(&zeroVal[0]), false 82 . . } 83 . . 84 . . if m.writing != 0 { 85 . . fatal("concurrent map read and map write") 86 . . } 87 . . 88 40ms 250ms hash := typ.Hasher(key, m.seed) 89 . . 90 . . if m.dirLen == 0 { 91 . 40ms _, elem, ok := m.getWithKeySmall(typ, hash, key) 92 . . if !ok { 93 . . return unsafe.Pointer(&zeroVal[0]), false 94 . . } 95 . . return elem, true 96 . . } 97 . . 98 . . // Select table. 99 30ms 30ms idx := m.directoryIndex(hash) ⋮ if m.dirLen == 1 { map.go:358 100 . . t := m.directoryAt(idx) 101 . . 102 . . // Probe table. 103 . . seq := makeProbeSeq(h1(hash), t.groups.lengthMask) 104 . . h2Hash := h2(hash) 105 . . for ; ; seq = seq.next() { 106 . . g := t.groups.group(typ, seq.offset) 107 . . 108 70ms 70ms match := g.ctrls().matchH2(h2Hash) return ctrlGroupMatchH2(g, h) group.go:154 v = ^v group.go:164 109 . . 110 30ms 30ms for match != 0 { 111 . . i := match.first() 112 . . 113 . . slotKey := g.key(typ, i) 114 . . slotKeyOrig := slotKey 115 . . if typ.IndirectKey() { 116 . . slotKey = *((*unsafe.Pointer)(slotKey)) 117 . . } 118 10ms 80ms if typ.Key.Equal(key, slotKey) { 119 . . var slotElem unsafe.Pointer 120 . . if goexperiment.MapSplitGroup { 121 . . slotElem = g.elem(typ, i) 122 . . } else { 123 . . slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) 124 . . } 125 . . if typ.IndirectElem() { 126 . . slotElem = *((*unsafe.Pointer)(slotElem)) 127 . . } 128 10ms 10ms return slotElem, true 129 . . } 130 10ms 10ms match = match.removeFirst() return b & (b - 1) group.go:64 131 . . } 132 . . 133 . . match = g.ctrls().matchEmpty() 134 . . if match != 0 { 135 . . // Finding an empty slot means we've reached the end of
runtime.mapassign
/usr/lib/go/src/internal/runtime/maps/runtime.go
Total: 240ms 1.04s (flat, cum) 1.45% 138 . . } 139 . . } 140 . . } 141 . . 142 . . //go:linkname runtime_mapassign runtime.mapassign 143 30ms 30ms func runtime_mapassign(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { 144 . . if m == nil { 145 . . panic(errNilAssign) 146 . . } 147 . . if race.Enabled { 148 . . callerpc := sys.GetCallerPC() 149 . . pc := abi.FuncPCABIInternal(runtime_mapassign) 150 . . race.WritePC(unsafe.Pointer(m), callerpc, pc) 151 . . race.ReadObjectPC(typ.Key, key, callerpc, pc) 152 . . } 153 . . if msan.Enabled { 154 . . msan.Read(key, typ.Key.Size_) 155 . . } 156 . . if asan.Enabled { 157 . . asan.Read(key, typ.Key.Size_) 158 . . } 159 10ms 10ms if m.writing != 0 { 160 . . fatal("concurrent map writes") 161 . . } 162 . . 163 10ms 220ms hash := typ.Hasher(key, m.seed) 164 . . 165 . . // Set writing after calling Hasher, since Hasher may panic, in which 166 . . // case we have not actually done a write. 167 50ms 50ms m.writing ^= 1 // toggle, see comment on writing 168 . . 169 10ms 10ms if m.dirPtr == nil { 170 . 200ms m.growToSmall(typ) 171 . . } 172 . . 173 . . if m.dirLen == 0 { 174 10ms 260ms elem := m.putSlotSmall(typ, hash, key) 175 . . if elem == nil { 176 . . // Can't fit another entry, grow to full size map. 177 . 40ms tab := m.growToTable(typ) 178 . . 179 . 10ms elem = tab.uncheckedPutSlotForAssign(typ, hash, key) 180 . . m.used++ 181 . . 182 . . tab.checkInvariants(typ, m) 183 . . } 184 . . 185 . . if m.writing == 0 { 186 . . fatal("concurrent map writes") 187 . . } 188 . . m.writing ^= 1 189 . . 190 10ms 10ms return elem 191 . . } 192 . . 193 . . var slotElem unsafe.Pointer 194 . . outer: 195 . . for { 196 . . // Select table. 197 . . idx := m.directoryIndex(hash) 198 . . t := m.directoryAt(idx) 199 . . 200 20ms 20ms seq := makeProbeSeq(h1(hash), t.groups.lengthMask) 201 . . 202 . . // As we look for a match, keep track of the first deleted slot 203 . . // we find, which we'll use to insert the new entry if 204 . . // necessary. 205 . . var firstDeletedGroup groupReference 206 . . var firstDeletedSlot uintptr 207 . . 208 . . h2Hash := h2(hash) 209 . . for ; ; seq = seq.next() { 210 . . g := t.groups.group(typ, seq.offset) 211 10ms 10ms match := g.ctrls().matchH2(h2Hash) return ctrlGroupMatchH2(g, h) group.go:154 v = ^v group.go:164 212 . . 213 . . // Look for an existing slot containing this key. 214 10ms 10ms for match != 0 { 215 . . i := match.first() 216 . . 217 . . slotKey := g.key(typ, i) 218 . . slotKeyOrig := slotKey 219 . . if typ.IndirectKey() { 220 . . slotKey = *((*unsafe.Pointer)(slotKey)) 221 . . } 222 . . if typ.Key.Equal(key, slotKey) { 223 . . if typ.NeedKeyUpdate() { 224 . 10ms typedmemmove(typ.Key, slotKey, key) 225 . . } 226 . . 227 . . if goexperiment.MapSplitGroup { 228 . . slotElem = g.elem(typ, i) 229 . . } else { 230 . . slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) 231 . . } 232 . . if typ.IndirectElem() { 233 . . slotElem = *((*unsafe.Pointer)(slotElem)) 234 . . } 235 . . 236 . . t.checkInvariants(typ, m) 237 . . break outer 238 . . } 239 . . match = match.removeFirst() 240 . . } 241 . . 242 . . // No existing slot for this key in this group. Is this the end 243 . . // of the probe sequence? 244 20ms 20ms match = g.ctrls().matchEmpty() return (*ctrlGroup)(g.data) group.go:294 245 . . if match != 0 { 246 . . // Finding an empty slot means we've reached the end of 247 . . // the probe sequence. 248 . . 249 . . var i uintptr 250 . . 251 . . // If we found a deleted slot along the way, we 252 . . // can replace it without consuming growthLeft. 253 10ms 10ms if firstDeletedGroup.data != nil { 254 . . g = firstDeletedGroup 255 . . i = firstDeletedSlot 256 . . t.growthLeft++ // will be decremented below to become a no-op. 257 . . } else { 258 . . // Otherwise, use the empty slot. 259 . . i = match.first() 260 . . } 261 . . 262 . . // If there is room left to grow, just insert the new entry. 263 . . if t.growthLeft > 0 { 264 20ms 20ms slotKey := g.key(typ, i) ⋮ offset := typ.KeysOff + i*typ.KeyStride group.go:299 265 . . slotKeyOrig := slotKey 266 . . if typ.IndirectKey() { 267 . . kmem := newobject(typ.Key) 268 . . *(*unsafe.Pointer)(slotKey) = kmem 269 . . slotKey = kmem 270 . . } 271 . 50ms typedmemmove(typ.Key, slotKey, key) 272 . . 273 . . if goexperiment.MapSplitGroup { 274 . . slotElem = g.elem(typ, i) 275 . . } else { 276 10ms 10ms slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) 277 . . } 278 . . if typ.IndirectElem() { 279 . . emem := newobject(typ.Elem) 280 . . *(*unsafe.Pointer)(slotElem) = emem 281 . . slotElem = emem 282 . . } 283 . . 284 10ms 10ms g.ctrls().set(i, ctrl(h2Hash)) 285 . . t.growthLeft-- 286 . . t.used++ 287 . . m.used++ 288 . . 289 . . t.checkInvariants(typ, m) 290 . . break outer 291 . . } 292 . . 293 . 30ms t.rehash(typ, m) 294 . . continue outer 295 . . } 296 . . 297 . . // No empty slots in this group. Check for a deleted 298 . . // slot, which we'll use if we don't find a match later
runtime.mapassign
/usr/lib/go/src/internal/runtime/maps/runtime.go
Total: 50ms 50ms (flat, cum) 0.07% 1352 50ms 50ms ???
runtime.acquirem
/usr/lib/go/src/runtime/runtime1.go
Total: 430ms 430ms (flat, cum) 0.6% 608 . . 609 . . // Helpers for Go. Must be NOSPLIT, must only call NOSPLIT functions, and must not block. 610 . . 611 . . //go:nosplit 612 . . func acquirem() *m { 613 10ms 10ms gp := getg() 614 330ms 330ms gp.m.locks++ 615 90ms 90ms return gp.m 616 . . } 617 . .
runtime.releasem
/usr/lib/go/src/runtime/runtime1.go
Total: 120ms 120ms (flat, cum) 0.17% 619 . . func releasem(mp *m) { 620 . . gp := getg() 621 10ms 10ms mp.locks-- 622 110ms 110ms if mp.locks == 0 && gp.preempt { 623 . . // restore the preemption request in case we've cleared it in newstack 624 . . gp.stackguard0 = stackPreempt 625 . . } 626 . . } 627 . .
runtime.memmove
/usr/lib/go/src/runtime/memmove_arm64.s
Total: 400ms 400ms (flat, cum) 0.56% 34 . . BLE copy16 35 . . 36 . . // Large copies 37 . . CMP $128, R2 38 . . BHI copy_long 39 10ms 10ms CMP $32, R2 40 . . BHI copy32_128 41 . . 42 . . // Small copies: 17..32 bytes. 43 . . LDP (R1), (R6, R7) 44 10ms 10ms ADD R1, R2, R4 // R4 points just past the last source byte 45 . . LDP -16(R4), (R12, R13) 46 30ms 30ms STP (R6, R7), (R0) 47 10ms 10ms ADD R0, R2, R5 // R5 points just past the last destination byte 48 . . STP (R12, R13), -16(R5) 49 . . RET 50 . . 51 . . // Small copies: 1..16 bytes. 52 . . copy16: 53 40ms 40ms ADD R1, R2, R4 // R4 points just past the last source byte 54 . . ADD R0, R2, R5 // R5 points just past the last destination byte 55 10ms 10ms CMP $8, R2 56 . . BLT copy7 57 10ms 10ms MOVD (R1), R6 58 80ms 80ms MOVD -8(R4), R7 59 40ms 40ms MOVD R6, (R0) 60 30ms 30ms MOVD R7, -8(R5) 61 . . RET 62 . . 63 . . copy7: 64 10ms 10ms TBZ $2, R2, copy3 65 . . MOVWU (R1), R6 66 40ms 40ms MOVWU -4(R4), R7 67 . . MOVW R6, (R0) 68 . . MOVW R7, -4(R5) 69 . . RET 70 . . 71 . . copy3: 72 . . TBZ $1, R2, copy1 73 . . MOVHU (R1), R6 74 30ms 30ms MOVHU -2(R4), R7 75 . . MOVH R6, (R0) 76 . . MOVH R7, -2(R5) 77 . . RET 78 . . 79 . . copy1: 80 . . MOVBU (R1), R6 81 20ms 20ms MOVB R6, (R0) 82 . . 83 . . copy0: 84 10ms 10ms RET 85 . . 86 . . // Medium copies: 33..128 bytes. 87 . . copy32_128: 88 . . ADD R1, R2, R4 // R4 points just past the last source byte 89 . . ADD R0, R2, R5 // R5 points just past the last destination byte 90 . . LDP (R1), (R6, R7) 91 . . LDP 16(R1), (R8, R9) 92 . . LDP -32(R4), (R10, R11) 93 . . LDP -16(R4), (R12, R13) 94 10ms 10ms CMP $64, R2 95 . . BHI copy128 96 10ms 10ms STP (R6, R7), (R0) 97 . . STP (R8, R9), 16(R0) 98 . . STP (R10, R11), -32(R5) 99 . . STP (R12, R13), -16(R5) 100 . . RET 101 . .
runtime.memmove
/usr/lib/go/src/runtime/memmove_arm64.s
Total: 10ms 10ms (flat, cum) 0.014% 128 . . 129 . . CMP $1024, R2 130 . . BLT backward_check 131 . . // feature detect to decide how to align 132 . . MOVBU runtime·arm64UseAlignedLoads(SB), R6 133 10ms 10ms CBNZ R6, use_aligned_loads 134 . . MOVD R0, R7 135 . . MOVD R5, R8 136 . . B backward_check 137 . . use_aligned_loads: 138 . . MOVD R1, R7
runtime.memmove
/usr/lib/go/src/runtime/memmove_arm64.s
Total: 140ms 140ms (flat, cum) 0.2% 165 . . SUBS $144, R2, R2 166 . . BLS copy64_from_end 167 . . 168 . . loop64: 169 . . STP (R6, R7), 16(R3) // Store B 170 10ms 10ms LDP 16(R1), (R6, R7) // Load B (next iteration) 171 . . STP (R8, R9), 32(R3) // Store C 172 . . LDP 32(R1), (R8, R9) // Load C 173 80ms 80ms STP (R10, R11), 48(R3) // Store D 174 . . LDP 48(R1), (R10, R11) // Load D 175 20ms 20ms STP.W (R12, R13), 64(R3) // Store E 176 10ms 10ms LDP.W 64(R1), (R12, R13) // Load E 177 10ms 10ms SUBS $64, R2, R2 178 . . BHI loop64 179 . . 180 . . // Write the last iteration and copy 64 bytes from the end. 181 . . copy64_from_end: 182 . . LDP -64(R4), (R14, R15) // Load F 183 . . STP (R6, R7), 16(R3) // Store B 184 . . LDP -48(R4), (R6, R7) // Load G 185 . . STP (R8, R9), 32(R3) // Store C 186 10ms 10ms LDP -32(R4), (R8, R9) // Load H 187 . . STP (R10, R11), 48(R3) // Store D 188 . . LDP -16(R4), (R10, R11) // Load I 189 . . STP (R12, R13), 64(R3) // Store E 190 . . STP (R14, R15), -64(R5) // Store F 191 . . STP (R6, R7), -48(R5) // Store G
git.urbach.dev/cli/q/src/codegen.(*Function).markAlive
/home/user/q/src/codegen/markAlive.go
Total: 470ms 1.20s (flat, cum) 1.68% 6 . . "git.urbach.dev/cli/q/src/ssa" 7 . . ) 8 . . 9 . . // markAlive marks the `live` value in the `block` as alive and recursively 10 . . // proceeds in the predecessors of `block` if they can reach the definition. 11 . 270ms func (f *Function) markAlive(live *Step, block *ssa.Block, use *Step, first bool) { 12 10ms 10ms if use.Block == block { 13 10ms 10ms phi, isPhi := use.Value.(*ssa.Phi) 14 . . 15 . . if isPhi { 16 10ms 10ms for index, value := range phi.Arguments { 17 . . if value == live.Value { 18 10ms 190ms f.markAlive(live, block.Predecessors[index], use, false) 19 . . } 20 . . } 21 . . 22 . . return 23 . . } 24 . . } 25 . . 26 . 120ms region := f.BlockToRegion[block] 27 . . 28 . . if first && use.Block == block && (block.Loop == nil || live.Block.Loop != nil) { 29 . . region.End = uint32(use.Index) 30 . . } 31 . . 32 . . var ( 33 . . steps = f.Steps[region.Start:region.End] 34 . . field *ssa.Field 35 . . isField bool 36 . . ) 37 . . 38 430ms 590ms for _, current := range slices.Backward(steps) { if !yield(i, s[i]) { iter.go:29 if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 if slices.Contains(current.Live, live) { markAlive.go:39 return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ if live.Value == current.Value { markAlive.go:49 ⋮ current.Live = append(current.Live, live) markAlive.go:47 ⋮ ⋮ ⋮ if slices.Contains(current.Live, live) { markAlive.go:39 ⋮ return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ } markAlive.go:58 ⋮ if live.Value == current.Value { markAlive.go:49 ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 for _, current := range slices.Backward(steps) { markAlive.go:38 ⋮ if live.Value == current.Value { markAlive.go:49
git.urbach.dev/cli/q/src/codegen.(*Function).markAlive-range1
/home/user/q/src/codegen/markAlive.go
Total: 360ms 520ms (flat, cum) 0.73% 39 220ms 220ms if slices.Contains(current.Live, live) { return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ ⋮ if v == s[i] { slices.go:98 ⋮ ⋮ return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 40 . . return 41 . . } 42 . . 43 . . if isField && field.Tuple == current.Value { 44 . . return 45 . . } 46 . . 47 90ms 210ms current.Live = append(current.Live, live) 48 . . 49 50ms 90ms if live.Value == current.Value { 50 . . field, isField = current.Value.(*ssa.Field) 51 . . _, isParam := current.Value.(*ssa.Parameter) 52 . . _, isPhi := current.Value.(*ssa.Phi) 53 . .
git.urbach.dev/cli/q/src/codegen.(*Function).markAlive
/home/user/q/src/codegen/markAlive.go
Total: 110ms 890ms (flat, cum) 1.24% 54 . . if !isParam && !isPhi && !isField { 55 . . return 56 . . } 57 . . } 58 30ms 30ms } 59 . . 60 30ms 30ms for _, pre := range block.Predecessors { 61 10ms 10ms if pre == block { 62 . . continue 63 . . } 64 . . 65 30ms 240ms if !pre.CanReachPredecessor(live.Block) { return b.canReachPredecessor(other, make(map[*Block]bool)) Block.go:44 ⋮ ⋮ ⋮ ⋮ 66 . . continue 67 . . } 68 . . 69 10ms 580ms f.markAlive(live, pre, use, false) 70 . . } 71 . . }
git.urbach.dev/cli/q/src/expression.Parse
/home/user/q/src/expression/Parse.go
Total: 540ms 5.21s (flat, cum) 7.28% 3 . . import ( 4 . . "git.urbach.dev/cli/q/src/token" 5 . . ) 6 . . 7 . . // Parse generates an expression tree from tokens. 8 40ms 1s func Parse(tokens token.List) *Expression { 9 . . var ( 10 . . cursor *Expression 11 . . root *Expression 12 . . i uint 13 . . ) 14 . . 15 20ms 20ms loop: 16 20ms 20ms for i < uint(len(tokens)) { 17 . . t := tokens[i] 18 . . 19 30ms 30ms if startsType(t.Kind) { case token.Identifier, token.Mul, token.Not, token.ArrayStart: type.go:43 ⋮ ⋮ switch kind { type.go:42 20 110ms 110ms j, isType := parseType(tokens, i) if !allowedInType(t.Kind) { type.go:30 case token.Identifier, token.Dot, token.Mul, token.Not, token.ArrayStart, token.ArrayEnd: type.go:8 ⋮ t := tokens[i] type.go:28 ⋮ ⋮ if !allowedInType(t.Kind) { type.go:30 ⋮ switch kind { type.go:7 ⋮ case token.Identifier, token.Dot, token.Mul, token.Not, token.ArrayStart, token.ArrayEnd: type.go:8 21 . . 22 . . if isType { 23 . . typeToken := makeTypeToken(tokens[i:j]) 24 . 10ms root, cursor = handleLiteral(root, cursor, typeToken) 25 . . i = j 26 . . t = tokens[i] 27 . . } 28 . . } 29 . . 30 30ms 30ms switch t.Kind { 31 20ms 20ms case token.GroupStart, token.ArrayStart, token.BlockStart: 32 10ms 10ms i++ 33 . . groupLevel := 1 34 . . groupPosition := i 35 . . 36 . . for i < uint(len(tokens)) { 37 10ms 10ms t = tokens[i] 38 . . 39 . . switch t.Kind { 40 20ms 20ms case token.GroupStart, token.ArrayStart, token.BlockStart: 41 . . groupLevel++ 42 . . case token.GroupEnd, token.ArrayEnd, token.BlockEnd: 43 . . groupLevel-- 44 . . 45 . . if groupLevel == 0 { 46 . 1.90s root, cursor = handleGroupEnd(tokens, root, cursor, groupPosition, i, t) 47 10ms 10ms i++ 48 . . continue loop 49 . . } 50 . . } 51 . . 52 . . i++ 53 . . } 54 . . 55 . . break loop 56 . . } 57 . . 58 . . switch { 59 50ms 50ms case cursor != nil && cursor.Token.Kind == token.Cast && len(cursor.Children) < 2: 60 10ms 10ms typ := tokens[i:] 61 . . 62 . . if len(typ) > 0 { 63 . . typeToken := makeTypeToken(typ) 64 10ms 10ms cursor.AddChild(newLeaf(typeToken)) expr.Children = append(expr.Children, child) Expression.go:24 65 . . } 66 . . 67 . . return root 68 20ms 20ms case t.Kind.IsLiteral(): return k == Identifier || k == Number || k == String || k == Rune || k.IsBuiltin() Kind.go:128 69 . 1.11s root, cursor = handleLiteral(root, cursor, t) 70 . . case t.Kind.IsKeyword(): 71 . . leaf := newLeaf(t) 72 . . leaf.Token.Kind = token.Invalid 73 . . 74 . . if cursor == nil { 75 . . cursor = leaf 76 . . root = leaf 77 . . } else { 78 . . cursor.AddChild(leaf) 79 . . cursor = leaf 80 . . } 81 . . case !t.Kind.IsOperator(): 82 . . // do nothing 83 . . case cursor == nil: 84 . . cursor = newLeaf(t) 85 . . cursor.precedence = precedence(t.Kind) 86 . . root = cursor 87 . . default: 88 20ms 290ms node := newLeaf(t) return &Expression{Token: t} newLeaf.go:7 ⋮ 89 10ms 10ms node.precedence = precedence(t.Kind) return Operators[symbol].Precedence operator.go:70 90 . . 91 . . if cursor.Token.Kind.IsOperator() { 92 10ms 250ms root = handleOperator(root, cursor, node) 93 . . } else { 94 50ms 230ms node.AddChild(cursor) ⋮ expr.Children = append(expr.Children, child) Expression.go:24 ⋮ if expr.Children == nil { Expression.go:20 ⋮ expr.Children = make([]*Expression, 0, 2) Expression.go:21 95 . . root = node 96 . . } 97 . . 98 . . if !cursor.Token.Kind.IsUnaryOperator() || !node.Token.Kind.IsUnaryOperator() { 99 . . cursor = node 100 . . } 101 . . } 102 . . 103 20ms 20ms i++ 104 . . } 105 . . 106 . . if root == nil { 107 . . root = New() 108 . . } 109 . . 110 20ms 20ms return root 111 . . }
sync.(*WaitGroup).Add
/usr/lib/go/src/sync/waitgroup.go
Total: 320ms 340ms (flat, cum) 0.48% 72 . . // Typically this means the calls to Add should execute before the statement 73 . . // creating the goroutine or other event to be waited for. 74 . . // If a WaitGroup is reused to wait for several independent sets of events, 75 . . // new Add calls must happen after all previous Wait calls have returned. 76 . . // See the WaitGroup example. 77 10ms 10ms func (wg *WaitGroup) Add(delta int) { 78 . . if race.Enabled { 79 . . if delta < 0 { 80 . . // Synchronize decrements with Wait. 81 . . race.ReleaseMerge(unsafe.Pointer(wg)) 82 . . } 83 . . race.Disable() 84 . . defer race.Enable() 85 . . } 86 . . bubbled := false 87 . 20ms if synctest.IsInBubble() { 88 . . // If Add is called from within a bubble, then all Add calls must be made 89 . . // from the same bubble. 90 . . switch synctest.Associate(wg) { 91 . . case synctest.Unbubbled: 92 . . case synctest.OtherBubble: 93 . . // wg is already associated with a different bubble. 94 . . fatal("sync: WaitGroup.Add called from multiple synctest bubbles") 95 . . case synctest.CurrentBubble: 96 . . bubbled = true 97 . . state := wg.state.Or(waitGroupBubbleFlag) 98 . . if state != 0 && state&waitGroupBubbleFlag == 0 { 99 . . // Add has been called from outside this bubble. 100 . . fatal("sync: WaitGroup.Add called from inside and outside synctest bubble") 101 . . } 102 . . } 103 . . } 104 300ms 300ms state := wg.state.Add(uint64(delta) << 32) 105 . . if state&waitGroupBubbleFlag != 0 && !bubbled { 106 . . // Add has been called from within a synctest bubble (and we aren't in one). 107 . . fatal("sync: WaitGroup.Add called from inside and outside synctest bubble") 108 . . } 109 . . v := int32(state >> 32) 110 10ms 10ms w := uint32(state & 0x7fffffff) 111 . . if race.Enabled && delta > 0 && v == int32(delta) { 112 . . // The first increment must be synchronized with Wait. 113 . . // Need to model this as a read, because there can be 114 . . // several concurrent wg.counter transitions from 0. 115 . . race.Read(unsafe.Pointer(&wg.sema))
sync.(*WaitGroup).Add
/usr/lib/go/src/sync/waitgroup.go
Total: 0 350ms (flat, cum) 0.49% 137 . . // Adds must not happen concurrently with wait when counter is 0, 138 . . // so we can safely disassociate wg from its current bubble. 139 . . synctest.Disassociate(wg) 140 . . } 141 . . for ; w != 0; w-- { 142 . 350ms runtime_Semrelease(&wg.sema, false, 0) 143 . . } 144 . . } 145 . . 146 . . // Done decrements the [WaitGroup] task counter by one. 147 . . // It is equivalent to Add(-1).
sync.(*WaitGroup).Done
/usr/lib/go/src/sync/waitgroup.go
Total: 0 480ms (flat, cum) 0.67% 151 . . // In the terminology of [the Go memory model], a call to Done 152 . . // "synchronizes before" the return of any Wait call that it unblocks. 153 . . // 154 . . // [the Go memory model]: https://go.dev/ref/mem 155 . . func (wg *WaitGroup) Done() { 156 . 480ms wg.Add(-1) 157 . . }
sync.(*WaitGroup).Wait
/usr/lib/go/src/sync/waitgroup.go
Total: 10ms 10ms (flat, cum) 0.014% 159 . . // Wait blocks until the [WaitGroup] task counter is zero. 160 10ms 10ms func (wg *WaitGroup) Wait() { 161 . . if race.Enabled { 162 . . race.Disable() 163 . . } 164 . . for { 165 . . state := wg.state.Load()
sync.(*WaitGroup).Wait
/usr/lib/go/src/sync/waitgroup.go
Total: 0 50ms (flat, cum) 0.07% 201 . . } 202 . . if race.Enabled { 203 . . race.Disable() 204 . . } 205 . . } 206 . 50ms runtime_SemacquireWaitGroup(&wg.sema, synctestDurable) 207 . . isReset := wg.state.Load() != 0 208 . . if race.Enabled { 209 . . race.Enable() 210 . . race.Acquire(unsafe.Pointer(wg)) 211 . . }
sync.(*WaitGroup).Go
/usr/lib/go/src/sync/waitgroup.go
Total: 0 210ms (flat, cum) 0.29% 232 . . // In the terminology of [the Go memory model], the return from f 233 . . // "synchronizes before" the return of any Wait call that it unblocks. 234 . . // 235 . . // [the Go memory model]: https://go.dev/ref/mem 236 . . func (wg *WaitGroup) Go(f func()) { 237 . 210ms wg.Add(1)
sync.(*WaitGroup).Go.func1
/usr/lib/go/src/sync/waitgroup.go
Total: 140ms 1.54s (flat, cum) 2.15% 238 70ms 1.47s go func() { 239 70ms 70ms defer func() {
sync.(*WaitGroup).Go.func1.1
/usr/lib/go/src/sync/waitgroup.go
Total: 0 500ms (flat, cum) 0.7% 240 . 20ms if x := recover(); x != nil { 241 . . // f panicked, which will be fatal because 242 . . // this is a new goroutine. 243 . . // 244 . . // Calling Done will unblock Wait in the main goroutine, 245 . . // allowing it to race with the fatal panic and 246 . . // possibly even exit the process (os.Exit(0)) 247 . . // before the panic completes. 248 . . // 249 . . // This is almost certainly undesirable, 250 . . // so instead avoid calling Done and simply panic. 251 . . panic(x) 252 . . } 253 . . 254 . . // f completed normally, or abruptly using goexit. 255 . . // Either way, decrement the semaphore. 256 . 480ms wg.Done() wg.Add(-1) waitgroup.go:156
sync.(*WaitGroup).Go.func1
/usr/lib/go/src/sync/waitgroup.go
Total: 60ms 57.69s (flat, cum) 80.65% 258 40ms 57.15s f() 259 20ms 540ms }() 260 . . }
git.urbach.dev/cli/q/src/token.identifier
/home/user/q/src/token/identifier.go
Total: 440ms 440ms (flat, cum) 0.62% 1 . . package token 2 . . 3 . . // identifier handles all tokens that qualify as an identifier. 4 20ms 20ms func identifier(tokens List, buffer []byte, i Position) (List, Position) { 5 . . position := i 6 10ms 10ms i++ 7 . . 8 190ms 190ms for i < Position(len(buffer)) && isIdentifier(buffer[i]) { ⋮ ⋮ return isLetter(c) || isDigit(c) || c == '_' identifier.go:59 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') identifier.go:72 ⋮ ⋮ ⋮ ⋮ 9 150ms 150ms i++ 10 . . } 11 . . 12 30ms 30ms identifier := buffer[position:i] 13 . . kind := Identifier 14 . . 15 . . switch string(identifier) { 16 10ms 10ms case "as": 17 . . kind = Cast 18 10ms 10ms case "assert": 19 . . kind = Assert 20 . . case "cas": 21 . . kind = Cas 22 20ms 20ms case "const": 23 . . kind = Const 24 . . case "delete": 25 . . kind = Delete 26 . . case "if": 27 . . kind = If
git.urbach.dev/cli/q/src/token.identifier
/home/user/q/src/token/identifier.go
Total: 70ms 70ms (flat, cum) 0.098% 41 . . kind = Loop 42 . . case "new": 43 . . kind = New 44 . . case "return": 45 . . kind = Return 46 10ms 10ms case "syscall": 47 . . kind = Syscall 48 . . case "switch": 49 . . kind = Switch 50 . . } 51 . . 52 50ms 50ms tokens = append(tokens, Token{Kind: kind, Position: position, Length: Length(len(identifier))}) 53 10ms 10ms return tokens, i 54 . . } 55 . .
git.urbach.dev/cli/q/src/token.isIdentifier
/home/user/q/src/token/identifier.go
Total: 90ms 90ms (flat, cum) 0.13% 57 . . // a digit or an underscore. 58 . . func isIdentifier(c byte) bool { 59 90ms 90ms return isLetter(c) || isDigit(c) || c == '_' return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') identifier.go:72 ⋮ ⋮ ⋮ 60 . . } 61 . . 62 . . // isIdentifierStart returns true if the character is the
git.urbach.dev/cli/q/src/token.isIdentifierStart
/home/user/q/src/token/identifier.go
Total: 10ms 10ms (flat, cum) 0.014% 63 . . // start of an identifier which is either a letter or an 64 . . // underscore. 65 . . func isIdentifierStart(c byte) bool { 66 10ms 10ms return isLetter(c) || c == '_' return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') identifier.go:72 67 . . } 68 . .
git.urbach.dev/cli/q/src/token.isLetter
/home/user/q/src/token/identifier.go
Total: 100ms 100ms (flat, cum) 0.14% 70 . . // or uppercase letter in the English alphabet. 71 . . func isLetter(c byte) bool { 72 100ms 100ms return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') 73 . . }
runtime.gclinkptr.ptr
/usr/lib/go/src/runtime/mcache.go
Total: 50ms 50ms (flat, cum) 0.07% 81 . . 82 . . // ptr returns the *gclink form of p. 83 . . // The result should be used for accessing fields, not stored 84 . . // in other data structures. 85 . . func (p gclinkptr) ptr() *gclink { 86 50ms 50ms return (*gclink)(unsafe.Pointer(p)) 87 . . } 88 . . 89 . . type stackfreelist struct { 90 . . list gclinkptr // linked list of free stacks 91 . . size uintptr // total size of stacks in list
runtime.getMCache
/usr/lib/go/src/runtime/mcache.go
Total: 350ms 350ms (flat, cum) 0.49% 136 . . // 137 . . // Returns nil if we're not bootstrapping or we don't have a P. The caller's 138 . . // P must not change, so we must be in a non-preemptible state. 139 . . func getMCache(mp *m) *mcache { 140 . . // Grab the mcache, since that's where stats live. 141 90ms 90ms pp := mp.p.ptr() 142 . . var c *mcache 143 30ms 30ms if pp == nil { 144 . . // We will be called without a P while bootstrapping, 145 . . // in which case we use mcache0, which is set in mallocinit. 146 . . // mcache0 is cleared when bootstrapping is complete, 147 . . // by procresize. 148 . . c = mcache0 149 . . } else { 150 230ms 230ms c = pp.mcache 151 . . } 152 . . return c 153 . . } 154 . .
runtime.(*mcache).refill
/usr/lib/go/src/runtime/mcache.go
Total: 80ms 2.98s (flat, cum) 4.17% 156 . . // have at least one free object. The current span in c must be full. 157 . . // 158 . . // Must run in a non-preemptible context since otherwise the owner of 159 . . // c could change. 160 10ms 10ms func (c *mcache) refill(spc spanClass) { 161 . . // Return the current cached span to the central lists. 162 . . s := c.alloc[spc] 163 . . 164 . . if s.allocCount != s.nelems { 165 . . throw("refill of span with free space remaining") 166 . . } 167 . . 168 . . // TODO(thepudds): we might be able to allow mallocgcTiny to reuse 16 byte objects from spc==5, 169 . . // but for now, just clear our reusable objects for tinySpanClass. 170 . . if spc == tinySpanClass { 171 . . c.reusableNoscan[spc] = 0 172 . . } 173 30ms 30ms if c.reusableNoscan[spc] != 0 { 174 . . throw("refill of span with reusable pointers remaining on pointer free list") 175 . . } 176 . . 177 . . if s != &emptymspan { 178 . . // Mark this span as no longer cached. 179 . . if s.sweepgen != mheap_.sweepgen+3 { 180 . . throw("bad sweepgen in refill") 181 . . } 182 . 1s mheap_.central[spc].mcentral.uncacheSpan(s) 183 . . 184 . . // Count up how many slots were used and record it. 185 . 100ms stats := memstats.heapStats.acquire() 186 . . slotsUsed := int64(s.allocCount) - int64(s.allocCountBeforeCache) 187 30ms 30ms atomic.Xadd64(&stats.smallAllocCount[spc.sizeclass()], slotsUsed) 188 . . 189 . . // Flush tinyAllocs. 190 . . if spc == tinySpanClass { 191 . . atomic.Xadd64(&stats.tinyAllocCount, int64(c.tinyAllocs)) 192 . . c.tinyAllocs = 0 193 . . } 194 . . memstats.heapStats.release() 195 . . 196 . . // Count the allocs in inconsistent, internal stats. 197 . . bytesAllocated := slotsUsed * int64(s.elemsize) 198 10ms 10ms gcController.totalAlloc.Add(bytesAllocated) return Xadd64(&u.value, delta) types.go:344 199 . . 200 . . // Clear the second allocCount just to be safe. 201 . . s.allocCountBeforeCache = 0 202 . . } 203 . . 204 . . // Get a new cached span from the central lists. 205 . 1.80s s = mheap_.central[spc].mcentral.cacheSpan() 206 . . if s == nil { 207 . . throw("out of memory") 208 . . } 209 . . 210 . . if s.allocCount == s.nelems {
runtime.(*mcache).refill
/usr/lib/go/src/runtime/mcache.go
Total: 10ms 130ms (flat, cum) 0.18% 230 . . // We pick an overestimate here because an underestimate leads 231 . . // the pacer to believe that it's in better shape than it is, 232 . . // which appears to lead to more memory used. See #53738 for 233 . . // more details. 234 . . usedBytes := uintptr(s.allocCount) * s.elemsize 235 10ms 130ms gcController.update(int64(s.npages*pageSize)-int64(usedBytes), int64(c.scanAlloc)) 236 . . c.scanAlloc = 0 237 . . 238 . . c.alloc[spc] = s 239 . . } 240 . .
runtime.(*mcache).releaseAll
/usr/lib/go/src/runtime/mcache.go
Total: 0 10ms (flat, cum) 0.014% 317 . . // recomputed since caching this span, so we don't do this for stale spans. 318 . . dHeapLive -= int64(s.nelems-s.allocCount) * int64(s.elemsize) 319 . . } 320 . . 321 . . // Release the span to the mcentral. 322 . 10ms mheap_.central[i].mcentral.uncacheSpan(s) 323 . . c.alloc[i] = &emptymspan 324 . . } 325 . . } 326 . . // Clear tinyalloc pool. 327 . . c.tiny = 0
runtime.(*mcache).prepareForSweep
/usr/lib/go/src/runtime/mcache.go
Total: 10ms 20ms (flat, cum) 0.028% 353 . . // could leave allocate-black on, allow allocation to continue 354 . . // as usual, use a ragged barrier at the beginning of sweep to 355 . . // ensure all cached spans are swept, and then disable 356 . . // allocate-black. However, with this approach it's difficult 357 . . // to avoid spilling mark bits into the *next* GC cycle. 358 10ms 10ms sg := mheap_.sweepgen 359 . . flushGen := c.flushGen.Load() 360 . . if flushGen == sg { 361 . . return 362 . . } else if flushGen != sg-2 { 363 . . println("bad flushGen", flushGen, "in prepareForSweep; sweepgen", sg) 364 . . throw("bad flushGen") 365 . . } 366 . 10ms c.releaseAll() 367 . . stackcache_clear(c) 368 . . c.flushGen.Store(mheap_.sweepgen) // Synchronizes with gcStart 369 . . } 370 . . 371 . . // addReusableNoscan adds a noscan object pointer to the reusable pointer free list
runtime.(*spanSet).push
/usr/lib/go/src/runtime/mspanset.go
Total: 160ms 770ms (flat, cum) 1.08% 78 . . spans [spanSetBlockEntries]atomicMSpanPointer 79 . . } 80 . . 81 . . // push adds span s to buffer b. push is safe to call concurrently 82 . . // with other push and pop operations. 83 . 90ms func (b *spanSet) push(s *mspan) { 84 . . // Obtain our slot. 85 10ms 530ms cursor := uintptr(b.index.incTail().tail() - 1) 86 140ms 140ms top, bottom := cursor/spanSetBlockEntries, cursor%spanSetBlockEntries 87 . . 88 . . // Do we need to add a block? 89 . . spineLen := b.spineLen.Load() 90 . . var block *spanSetBlock 91 . . retry: 92 . . if top < spineLen { 93 10ms 10ms block = b.spine.Load().lookup(top).Load() return spanSetSpinePointer{s.a.Load()} mspanset.go:304 return Loadp(unsafe.Pointer(&u.value)) types.go:466 94 . . } else { 95 . . // Add a new block to the spine, potentially growing 96 . . // the spine. 97 . . lock(&b.spineLock) 98 . . // spineLen cannot change until we release the lock,
runtime.(*spanSet).push
/usr/lib/go/src/runtime/mspanset.go
Total: 100ms 100ms (flat, cum) 0.14% 140 . . unlock(&b.spineLock) 141 . . } 142 . . 143 . . // We have a block. Insert the span atomically, since there may be 144 . . // concurrent readers via the block API. 145 90ms 90ms block.spans[bottom].StoreNoWB(s) ⋮ p.p.StoreNoWB(unsafe.Pointer(s)) mspanset.go:425 StorepNoWB(unsafe.Pointer(&u.value), value) types.go:479 146 10ms 10ms } 147 . . 148 . . // pop removes and returns a span from buffer b, or nil if b is empty. 149 . . // pop is safe to call concurrently with other pop and push operations. 150 . . func (b *spanSet) pop() *mspan { 151 . . var head, tail uint32
runtime.(*spanSet).pop
/usr/lib/go/src/runtime/mspanset.go
Total: 140ms 140ms (flat, cum) 0.2% 154 . . if GOARCH == "arm64" { 155 . . backoff = 128 156 . . } 157 . . claimLoop: 158 . . for { 159 30ms 30ms headtail := b.index.load() return headTailIndex(h.u.Load()) mspanset.go:379 return Load64(&u.value) types.go:309 160 30ms 30ms head, tail = headtail.split() return h.head(), h.tail() mspanset.go:369 return uint32(h >> 32) mspanset.go:359 161 . . if head >= tail { 162 . . // The buf is empty, as far as we can tell. 163 20ms 20ms return nil 164 . . } 165 . . // Check if the head position we want to claim is actually 166 . . // backed by a block. 167 . . spineLen := b.spineLen.Load() 168 . . if spineLen <= uintptr(head)/spanSetBlockEntries { 169 . . // We're racing with a spine growth and the allocation of 170 . . // a new block (and maybe a new spine!), and trying to grab 171 . . // the span at the index which is currently being pushed. 172 . . // Instead of spinning, let's just notify the caller that 173 . . // there's nothing currently here. Spinning on this is 174 . . // almost definitely not worth it. 175 . . return nil 176 . . } 177 . . // Try to claim the current head by CASing in an updated head. 178 . . // This may fail transiently due to a push which modifies the 179 . . // tail, so keep trying while the head isn't changing. 180 . . want := head 181 . . for want == head { 182 60ms 60ms if b.index.cas(headtail, makeHeadTailIndex(want+1, tail)) { return h.u.CompareAndSwap(uint64(old), uint64(new)) mspanset.go:384 return Cas64(&u.value, old, new) types.go:325 183 . . break claimLoop 184 . . } 185 . . // Use a backoff approach to reduce demand to the shared memory location 186 . . // decreases memory contention and allows for other threads to make quicker 187 . . // progress.
runtime.(*spanSet).pop
/usr/lib/go/src/runtime/mspanset.go
Total: 20ms 20ms (flat, cum) 0.028% 230 . . // popping its corresponding mspan) by the time we get here. Because 231 . . // we're the last popper, we also don't have to worry about concurrent 232 . . // pushers (there can't be any). Note that we may not be the popper 233 . . // which claimed the last slot in the block, we're just the last one 234 . . // to finish popping. 235 10ms 10ms if block.popped.Add(1) == spanSetBlockEntries { return Xadd(&u.value, delta) types.go:291 236 . . // Clear the block's pointer. 237 . . blockp.StoreNoWB(nil) 238 . . 239 . . // Return the block to the block pool. 240 . . spanSetBlockPool.free(block) 241 . . } 242 10ms 10ms return s 243 . . } 244 . . 245 . . // reset resets a spanSet which is empty. It will also clean up 246 . . // any left over blocks. 247 . . //
runtime.(*atomicSpanSetSpinePointer).Load
/usr/lib/go/src/runtime/mspanset.go
Total: 10ms 10ms (flat, cum) 0.014% 299 . . 300 . . // Loads the spanSetSpinePointer and returns it. 301 . . // 302 . . // It has the same semantics as atomic.UnsafePointer. 303 . . func (s *atomicSpanSetSpinePointer) Load() spanSetSpinePointer { 304 10ms 10ms return spanSetSpinePointer{s.a.Load()} return Loadp(unsafe.Pointer(&u.value)) types.go:466 305 . . } 306 . . 307 . . // Stores the spanSetSpinePointer. 308 . . // 309 . . // It has the same semantics as [atomic.UnsafePointer].
runtime.headTailIndex.head
/usr/lib/go/src/runtime/mspanset.go
Total: 30ms 30ms (flat, cum) 0.042% 354 . . return headTailIndex(uint64(head)<<32 | uint64(tail)) 355 . . } 356 . . 357 . . // head returns the head of a headTailIndex value. 358 . . func (h headTailIndex) head() uint32 { 359 30ms 30ms return uint32(h >> 32) 360 . . } 361 . . 362 . . // tail returns the tail of a headTailIndex value. 363 . . func (h headTailIndex) tail() uint32 {
runtime.headTailIndex.split
/usr/lib/go/src/runtime/mspanset.go
Total: 30ms 30ms (flat, cum) 0.042% 365 . . } 366 . . 367 . . // split splits the headTailIndex value into its parts. 368 . . func (h headTailIndex) split() (head uint32, tail uint32) { 369 30ms 30ms return h.head(), h.tail() return uint32(h >> 32) mspanset.go:359 370 . . } 371 . . 372 . . // atomicHeadTailIndex is an atomically-accessed headTailIndex. 373 . . type atomicHeadTailIndex struct {
runtime.(*atomicHeadTailIndex).load
/usr/lib/go/src/runtime/mspanset.go
Total: 30ms 30ms (flat, cum) 0.042% 375 . . } 376 . . 377 . . // load atomically reads a headTailIndex value. 378 . . func (h *atomicHeadTailIndex) load() headTailIndex { 379 30ms 30ms return headTailIndex(h.u.Load()) return Load64(&u.value) types.go:309 380 . . } 381 . .
runtime.(*atomicHeadTailIndex).cas
/usr/lib/go/src/runtime/mspanset.go
Total: 60ms 60ms (flat, cum) 0.084% 382 . . // cas atomically compares-and-swaps a headTailIndex value. 383 . . func (h *atomicHeadTailIndex) cas(old, new headTailIndex) bool { 384 60ms 60ms return h.u.CompareAndSwap(uint64(old), uint64(new)) return Cas64(&u.value, old, new) types.go:325 385 . . } 386 . . 387 . . // incHead atomically increments the head of a headTailIndex. 388 . . func (h *atomicHeadTailIndex) incHead() headTailIndex { 389 . . return headTailIndex(h.u.Add(1 << 32))
runtime.(*atomicHeadTailIndex).incTail
/usr/lib/go/src/runtime/mspanset.go
Total: 60ms 520ms (flat, cum) 0.73% 393 . . func (h *atomicHeadTailIndex) decHead() headTailIndex { 394 . . return headTailIndex(h.u.Add(-(1 << 32))) 395 . . } 396 . . 397 . . // incTail atomically increments the tail of a headTailIndex. 398 . 460ms func (h *atomicHeadTailIndex) incTail() headTailIndex { 399 60ms 60ms ht := headTailIndex(h.u.Add(1)) 400 . . // Check for overflow. 401 . . if ht.tail() == 0 { 402 . . print("runtime: head = ", ht.head(), ", tail = ", ht.tail(), "\n") 403 . . throw("headTailIndex overflow") 404 . . }
runtime.(*atomicMSpanPointer).StoreNoWB
/usr/lib/go/src/runtime/mspanset.go
Total: 20ms 20ms (flat, cum) 0.028% 420 . . return (*mspan)(p.p.Load()) 421 . . } 422 . . 423 . . // StoreNoWB stores an *mspan. 424 . . func (p *atomicMSpanPointer) StoreNoWB(s *mspan) { 425 20ms 20ms p.p.StoreNoWB(unsafe.Pointer(s)) StorepNoWB(unsafe.Pointer(&u.value), value) types.go:479 426 . . }
git.urbach.dev/cli/q/src/expression.(*Expression).AddChild
/home/user/q/src/expression/Expression.go
Total: 160ms 570ms (flat, cum) 0.8% 15 . . precedence int8 16 . . } 17 . . 18 . . // AddChild adds a child to the expression. 19 . . func (expr *Expression) AddChild(child *Expression) { 20 40ms 40ms if expr.Children == nil { 21 10ms 370ms expr.Children = make([]*Expression, 0, 2) 22 . . } 23 . . 24 110ms 160ms expr.Children = append(expr.Children, child) 25 . . child.Parent = expr 26 . . }
git.urbach.dev/cli/q/src/expression.(*Expression).EachLeaf
/home/user/q/src/expression/Expression.go
Total: 100ms 330ms (flat, cum) 0.46% 27 . . 28 . . // EachLeaf iterates through all leaves in the tree. 29 20ms 20ms func (expr *Expression) EachLeaf(yield func(*Expression) bool) bool { 30 30ms 30ms if expr.IsLeaf() { return len(expr.Children) == 0 Expression.go:68 ⋮ 31 20ms 120ms return yield(expr) 32 . . } 33 . . 34 30ms 30ms for _, child := range expr.Children { 35 . 130ms if !child.EachLeaf(yield) { 36 . . return false 37 . . } 38 . . } 39 . . 40 . . return true
git.urbach.dev/cli/q/src/expression.(*Expression).InsertAbove
/home/user/q/src/expression/Expression.go
Total: 60ms 240ms (flat, cum) 0.34% 58 . . if expr.Parent != nil { 59 . . expr.Parent.Children[len(expr.Parent.Children)-1] = tree 60 . . tree.Parent = expr.Parent 61 . . } 62 . . 63 60ms 240ms tree.AddChild(expr) expr.Children = append(expr.Children, child) Expression.go:24 ⋮ expr.Children = make([]*Expression, 0, 2) Expression.go:21 ⋮ ⋮ expr.Children = append(expr.Children, child) Expression.go:24 ⋮ expr.Children = make([]*Expression, 0, 2) Expression.go:21 ⋮ expr.Children = append(expr.Children, child) Expression.go:24 ⋮ expr.Children = make([]*Expression, 0, 2) Expression.go:21 ⋮ expr.Children = append(expr.Children, child) Expression.go:24 ⋮ ⋮ expr.Children = make([]*Expression, 0, 2) Expression.go:21 64 . . } 65 . .
git.urbach.dev/cli/q/src/expression.(*Expression).IsLeaf
/home/user/q/src/expression/Expression.go
Total: 40ms 40ms (flat, cum) 0.056% 66 . . // IsLeaf returns true if the expression has no children. 67 . . func (expr *Expression) IsLeaf() bool { 68 40ms 40ms return len(expr.Children) == 0 69 . . } 70 . . 71 . . // LastChild returns the last child. 72 . . func (expr *Expression) LastChild() *Expression { 73 . . return expr.Children[len(expr.Children)-1]
git.urbach.dev/cli/q/src/expression.(*Expression).Leaves.func1
/home/user/q/src/expression/Expression.go
Total: 0 200ms (flat, cum) 0.28% 74 . . } 75 . . 76 . . // Leaves iterates through all leaves in the tree. 77 . . func (expr *Expression) Leaves() iter.Seq[*Expression] { 78 . . return func(yield func(*Expression) bool) { 79 . 200ms expr.EachLeaf(yield) 80 . . } 81 . . } 82 . . 83 . . // RemoveChild removes a child from the expression. 84 . . func (expr *Expression) RemoveChild(child *Expression) {
git.urbach.dev/cli/q/src/expression.(*Expression).Source
/home/user/q/src/expression/Expression.go
Total: 60ms 60ms (flat, cum) 0.084% 102 . . expr.Token.Reset() 103 . . expr.precedence = 0 104 . . } 105 . . 106 . . // Source returns the start and end positions in the source file. 107 60ms 60ms func (expr *Expression) Source() token.Source { 108 . . start := expr.Token.Position
git.urbach.dev/cli/q/src/expression.(*Expression).Source-range1
/home/user/q/src/expression/Expression.go
Total: 110ms 310ms (flat, cum) 0.43% 110 . . 111 60ms 260ms for leaf := range expr.Leaves() { ⋮ expr.EachLeaf(yield) Expression.go:79 ⋮ ⋮ ⋮ ⋮ 112 10ms 10ms if leaf.Token.Position < start { 113 . . start = leaf.Token.Position 114 10ms 10ms } else if leaf.Token.End() > end { 115 . . end = leaf.Token.End() 116 . . } 117 30ms 30ms } 118 . . 119 . . return token.NewSource(start, end) 120 . . }
git.urbach.dev/cli/q/src/expression.(*Expression).SourceString
/home/user/q/src/expression/Expression.go
Total: 0 10ms (flat, cum) 0.014% 121 . . 122 . . // SourceString returns the string that was parsed in this expression. 123 . . func (expr *Expression) SourceString(source []byte) string { 124 . 10ms region := expr.Source() 125 . . open := 0 126 . . left := token.Position(0) 127 . . right := token.Position(0) 128 . . 129 . . for i := region.Start(); i < region.End(); i++ {
git.urbach.dev/cli/q/src/expression.(*Expression).SourceString
/home/user/q/src/expression/Expression.go
Total: 0 10ms (flat, cum) 0.014% 150 . . 151 . . right++ 152 . . open-- 153 . . } 154 . . 155 . 10ms return string(source[region.Start()-left : region.End()+right]) 156 . . } 157 . .
git.urbach.dev/cli/q/src/expression.(*Expression).String
/home/user/q/src/expression/Expression.go
Total: 0 120ms (flat, cum) 0.17% 159 . . func (expr *Expression) String(source []byte) string { 160 . . builder := strings.Builder{} 161 . 120ms expr.write(&builder, source) 162 . . return builder.String() 163 . . }
internal/runtime/maps.h2
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 10ms 10ms (flat, cum) 0.014% 205 . . 206 . . // Extracts the H2 portion of a hash: the 7 bits not used for h1. 207 . . // 208 . . // These are used as an occupied control byte. 209 . . func h2(h uintptr) uintptr { 210 10ms 10ms return h & 0x7f 211 . . } 212 . . 213 . . // Note: changes here must be reflected in cmd/compile/internal/reflectdata/map.go:MapType. 214 . . type Map struct { 215 . . // The number of filled slots (i.e. the number of elements in all
internal/runtime/maps.NewMap
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 340ms (flat, cum) 0.48% 280 . . // maxAlloc should be runtime.maxAlloc. 281 . . // 282 . . // TODO(prattmic): Put maxAlloc somewhere accessible. 283 . . func NewMap(mt *abi.MapType, hint uintptr, m *Map, maxAlloc uintptr) *Map { 284 . . if m == nil { 285 . 300ms m = new(Map) 286 . . } 287 . . 288 . 40ms m.seed = uintptr(rand()) 289 . . 290 . . if hint <= abi.MapGroupSlots { 291 . . // A small map can fill all 8 slots, so no need to increase 292 . . // target capacity. 293 . . //
internal/runtime/maps.NewMap
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 10ms 340ms (flat, cum) 0.48% 332 . . } 333 . . 334 . . m.globalDepth = uint8(sys.TrailingZeros64(dirSize)) 335 . . m.globalShift = depthToShift(m.globalDepth) 336 . . 337 . 40ms directory := make([]*table, dirSize) 338 . . 339 10ms 10ms for i := range directory { 340 . . // TODO: Think more about initial table capacity. 341 . 290ms directory[i] = newTable(mt, uint64(targetCapacity)/dirSize, i, m.globalDepth) 342 . . } 343 . . 344 . . m.dirPtr = unsafe.Pointer(&directory[0]) 345 . . m.dirLen = len(directory)
internal/runtime/maps.NewEmptyMap
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 220ms (flat, cum) 0.31% 347 . . return m 348 . . } 349 . . 350 . . func NewEmptyMap() *Map { 351 . 200ms m := new(Map) 352 . 20ms m.seed = uintptr(rand()) 353 . . // See comment in NewMap. No need to eager allocate a group. 354 . . return m
internal/runtime/maps.(*Map).directoryIndex
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 40ms 40ms (flat, cum) 0.056% 356 . . 357 . . func (m *Map) directoryIndex(hash uintptr) uintptr { 358 40ms 40ms if m.dirLen == 1 { 359 . . return 0 360 . . } 361 . . return hash >> (m.globalShift & 63)
internal/runtime/maps.(*Map).directoryAt
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 40ms 40ms (flat, cum) 0.056% 362 . . } 363 . . 364 . . func (m *Map) directoryAt(i uintptr) *table { 365 40ms 40ms return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) 366 . . } 367 . . 368 . . func (m *Map) directorySet(i uintptr, nt *table) { 369 . . *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) = nt 370 . . }
internal/runtime/maps.(*Map).getWithKeySmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 40ms 40ms (flat, cum) 0.056% 460 . . 461 . . idx := m.directoryIndex(hash) 462 . . return m.directoryAt(idx).getWithoutKey(typ, hash, key) 463 . . } 464 . . 465 10ms 10ms func (m *Map) getWithKeySmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { 466 . . g := groupReference{ 467 . . data: m.dirPtr, 468 . . } 469 . . 470 . . match := g.ctrls().matchH2(h2(hash)) 471 . . 472 10ms 10ms for match != 0 { 473 . . i := match.first() 474 . . 475 . . slotKey := g.key(typ, i) 476 . . if typ.IndirectKey() { 477 . . slotKey = *((*unsafe.Pointer)(slotKey)) 478 . . } 479 . . 480 . . if typ.Key.Equal(key, slotKey) { 481 20ms 20ms slotElem := g.elem(typ, i) offset := typ.ElemsOff + i*typ.ElemStride group.go:306 ⋮ 482 . . if typ.IndirectElem() { 483 . . slotElem = *((*unsafe.Pointer)(slotElem)) 484 . . } 485 . . return slotKey, slotElem, true 486 . . }
internal/runtime/maps.(*Map).putSlotSmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 110ms 130ms (flat, cum) 0.18% 551 . . 552 . . return elem 553 . . } 554 . . } 555 . . 556 60ms 60ms func (m *Map) putSlotSmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { 557 . . g := groupReference{ 558 10ms 10ms data: m.dirPtr, 559 . . } 560 . . 561 10ms 10ms match := g.ctrls().matchH2(h2(hash)) return h & 0x7f map.go:210 562 . . 563 . . // Look for an existing slot containing this key. 564 30ms 30ms for match != 0 { 565 . . i := match.first() 566 . . 567 . . slotKey := g.key(typ, i) 568 . . if typ.IndirectKey() { 569 . . slotKey = *((*unsafe.Pointer)(slotKey)) 570 . . } 571 . 20ms if typ.Key.Equal(key, slotKey) { 572 . . if typ.NeedKeyUpdate() { 573 . . typedmemmove(typ.Key, slotKey, key) 574 . . } 575 . . 576 . . slotElem := g.elem(typ, i)
internal/runtime/maps.(*Map).putSlotSmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 90ms 120ms (flat, cum) 0.17% 593 . . } 594 . . 595 . . i := match.first() 596 . . 597 . . slotKey := g.key(typ, i) 598 10ms 10ms if typ.IndirectKey() { 599 . . kmem := newobject(typ.Key) 600 . . *(*unsafe.Pointer)(slotKey) = kmem 601 . . slotKey = kmem 602 . . } 603 . 30ms typedmemmove(typ.Key, slotKey, key) 604 . . 605 10ms 10ms slotElem := g.elem(typ, i) offset := typ.ElemsOff + i*typ.ElemStride group.go:306 606 . . if typ.IndirectElem() { 607 . . emem := newobject(typ.Elem) 608 . . *(*unsafe.Pointer)(slotElem) = emem 609 . . slotElem = emem 610 . . } 611 . . 612 40ms 40ms g.ctrls().set(i, ctrl(h2(hash))) *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c group.go:143 ⋮ return (*ctrlGroup)(g.data) group.go:294 613 20ms 20ms m.used++ 614 . . 615 10ms 10ms return slotElem 616 . . }
internal/runtime/maps.(*Map).growToSmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 90ms 1.36s (flat, cum) 1.90% 617 . . 618 20ms 20ms func (m *Map) growToSmall(typ *abi.MapType) { 619 20ms 1.29s grp := newGroups(typ, 1) data: newarray(typ.Group, int(length)), group.go:330 ⋮ 620 . . m.dirPtr = grp.data 621 . . 622 . . g := groupReference{ 623 . . data: m.dirPtr, 624 . . } 625 30ms 30ms g.ctrls().setEmpty() *g = ctrlGroup(bitsetEmpty) group.go:148 626 20ms 20ms } 627 . .
internal/runtime/maps.(*Map).growToTable
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 100ms (flat, cum) 0.14% 628 . . func (m *Map) growToTable(typ *abi.MapType) *table { 629 . 100ms tab := newTable(typ, 2*abi.MapGroupSlots, 0, 0) 630 . . 631 . . g := groupReference{ 632 . . data: m.dirPtr, 633 . . } 634 . .
internal/runtime/maps.(*Map).growToTable
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 10ms 70ms (flat, cum) 0.098% 646 . . elem := g.elem(typ, i) 647 . . if typ.IndirectElem() { 648 . . elem = *((*unsafe.Pointer)(elem)) 649 . . } 650 . . 651 10ms 30ms hash := typ.Hasher(key, m.seed) 652 . . 653 . 20ms tab.uncheckedPutSlot(typ, hash, key, elem) 654 . . } 655 . . 656 . 20ms directory := make([]*table, 1) 657 . . 658 . . directory[0] = tab 659 . . 660 . . m.dirPtr = unsafe.Pointer(&directory[0]) 661 . . m.dirLen = len(directory)
internal/runtime/maps.(*Map).Delete
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 20ms (flat, cum) 0.028% 682 . . // Set writing after calling Hasher, since Hasher may panic, in which 683 . . // case we have not actually done a write. 684 . . m.writing ^= 1 // toggle, see comment on writing 685 . . 686 . . if m.dirLen == 0 { 687 . 20ms m.deleteSmall(typ, hash, key) 688 . . } else { 689 . . idx := m.directoryIndex(hash) 690 . . if m.directoryAt(idx).Delete(typ, m, hash, key) { 691 . . m.tombstonePossible = true 692 . . }
internal/runtime/maps.(*Map).deleteSmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 10ms (flat, cum) 0.014% 717 . . slotKey := g.key(typ, i) 718 . . origSlotKey := slotKey 719 . . if typ.IndirectKey() { 720 . . slotKey = *((*unsafe.Pointer)(slotKey)) 721 . . } 722 . 10ms if typ.Key.Equal(key, slotKey) { 723 . . m.used-- 724 . . 725 . . if typ.IndirectKey() { 726 . . // Clearing the pointer is sufficient. 727 . . *(*unsafe.Pointer)(origSlotKey) = nil
internal/runtime/maps.(*Map).deleteSmall
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 0 10ms (flat, cum) 0.014% 738 . . // Unlike keys, always clear the elem (even if 739 . . // it contains no pointers), as compound 740 . . // assignment operations depend on cleared 741 . . // deleted values. See 742 . . // https://go.dev/issue/25936. 743 . 10ms typedmemclr(typ.Elem, slotElem) 744 . . } 745 . . 746 . . // We only have 1 group, so it is OK to immediately 747 . . // reuse deleted slots. 748 . . g.ctrls().set(i, ctrlEmpty)
internal/runtime/maps.mapKeyError
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 20ms 30ms (flat, cum) 0.042% 838 . . 839 . . return m 840 . . } 841 . . 842 . . func mapKeyError(t *abi.MapType, p unsafe.Pointer) error { 843 20ms 20ms if !t.HashMightPanic() { 844 . . return nil 845 . . } 846 . 10ms return mapKeyError2(t.Key, p) 847 . . }
internal/runtime/maps.mapKeyError2
/usr/lib/go/src/internal/runtime/maps/map.go
Total: 10ms 10ms (flat, cum) 0.014% 848 . . 849 10ms 10ms func mapKeyError2(t *abi.Type, p unsafe.Pointer) error { 850 . . if t.TFlag&abi.TFlagRegularMemory != 0 { 851 . . return nil 852 . . } 853 . . switch t.Kind() { 854 . . case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
runtime.(*sweepClass).update
/usr/lib/go/src/runtime/mgcsweep.go
Total: 10ms 10ms (flat, cum) 0.014% 66 . . 67 . . func (s *sweepClass) update(sNew sweepClass) { 68 . . // Only update *s if its current value is less than sNew, 69 . . // since *s increases monotonically. 70 . . sOld := s.load() 71 10ms 10ms for sOld < sNew && !atomic.Cas((*uint32)(s), uint32(sOld), uint32(sNew)) { 72 . . sOld = s.load() 73 . . } 74 . . // TODO(mknyszek): This isn't the only place we have 75 . . // an atomic monotonically increasing counter. It would 76 . . // be nice to have an "atomic max" which is just implemented
runtime.(*mheap).nextSpanForSweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 10ms 130ms (flat, cum) 0.18% 98 . . for sc := sweep.centralIndex.load(); sc < numSweepClasses; sc++ { 99 . . spc, full := sc.split() 100 . . c := &h.central[spc].mcentral 101 . . var s *mspan 102 . . if full { 103 . 120ms s = c.fullUnswept(sg).pop() 104 . . } else { 105 . . s = c.partialUnswept(sg).pop() 106 . . } 107 . . if s != nil { 108 . . // Write down that we found something so future sweepers 109 . . // can start from here. 110 10ms 10ms sweep.centralIndex.update(sc) for sOld < sNew && !atomic.Cas((*uint32)(s), uint32(sOld), uint32(sNew)) { mgcsweep.go:71 111 . . return s 112 . . } 113 . . } 114 . . // Write down that we found nothing. 115 . . sweep.centralIndex.update(sweepClassDone)
runtime.(*activeSweep).begin
/usr/lib/go/src/runtime/mgcsweep.go
Total: 80ms 80ms (flat, cum) 0.11% 146 . . // 147 . . // Even if the sweepLocker is invalid, its sweepGen is always valid. 148 . . func (a *activeSweep) begin() sweepLocker { 149 . . for { 150 . . state := a.state.Load() 151 60ms 60ms if state&sweepDrainedMask != 0 { 152 . . return sweepLocker{mheap_.sweepgen, false} 153 . . } 154 20ms 20ms if a.state.CompareAndSwap(state, state+1) { return Cas(&u.value, old, new) types.go:236 155 . . return sweepLocker{mheap_.sweepgen, true} 156 . . } 157 . . } 158 . . } 159 . .
runtime.(*activeSweep).end
/usr/lib/go/src/runtime/mgcsweep.go
Total: 30ms 30ms (flat, cum) 0.042% 166 . . for { 167 . . state := a.state.Load() 168 . . if (state&^sweepDrainedMask)-1 >= sweepDrainedMask { 169 . . throw("mismatched begin/end of activeSweep") 170 . . } 171 20ms 20ms if a.state.CompareAndSwap(state, state-1) { return Cas(&u.value, old, new) types.go:236 172 10ms 10ms if state-1 != sweepDrainedMask { 173 . . return 174 . . } 175 . . // We're the last sweeper, and there's nothing left to sweep. 176 . . if debug.gcpacertrace > 0 { 177 . . live := gcController.heapLive.Load()
runtime.bgsweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 0 830ms (flat, cum) 1.16% 295 . . // isn't spare idle time available on other cores. If there's available idle 296 . . // time, helping to sweep can reduce allocation latencies by getting ahead of 297 . . // the proportional sweeper and having spans ready to go for allocation. 298 . . const sweepBatchSize = 10 299 . . nSwept := 0 300 . 830ms for sweepone() != ^uintptr(0) { 301 . . nSwept++ 302 . . if nSwept%sweepBatchSize == 0 { 303 . . goschedIfBusy() 304 . . } 305 . . }
runtime.(*sweepLocker).tryAcquire
/usr/lib/go/src/runtime/mgcsweep.go
Total: 40ms 40ms (flat, cum) 0.056% 342 . . func (l *sweepLocker) tryAcquire(s *mspan) (sweepLocked, bool) { 343 . . if !l.valid { 344 . . throw("use of invalid sweepLocker") 345 . . } 346 . . // Check before attempting to CAS. 347 40ms 40ms if atomic.Load(&s.sweepgen) != l.sweepGen-2 { 348 . . return sweepLocked{}, false 349 . . } 350 . . // Attempt to acquire sweep ownership of s. 351 . . if !atomic.Cas(&s.sweepgen, l.sweepGen-2, l.sweepGen-1) { 352 . . return sweepLocked{}, false
runtime.sweepone
/usr/lib/go/src/runtime/mgcsweep.go
Total: 130ms 1.32s (flat, cum) 1.85% 363 . . // in the middle of sweep thus leaving the span in an inconsistent state for next GC 364 . . gp.m.locks++ 365 . . 366 . . // TODO(austin): sweepone is almost always called in a loop; 367 . . // lift the sweepLocker into its callers. 368 40ms 40ms sl := sweep.active.begin() if state&sweepDrainedMask != 0 { mgcsweep.go:151 ⋮ if a.state.CompareAndSwap(state, state+1) { mgcsweep.go:154 return Cas(&u.value, old, new) types.go:236 369 . . if !sl.valid { 370 . . gp.m.locks-- 371 . . return ^uintptr(0) 372 . . } 373 . . 374 . . // Find a span to sweep. 375 . . npages := ^uintptr(0) 376 . . var noMoreWork bool 377 . . for { 378 . 130ms s := mheap_.nextSpanForSweep() 379 . . if s == nil { 380 . . noMoreWork = sweep.active.markDrained() 381 . . break 382 . . } 383 70ms 70ms if state := s.state.get(); state != mSpanInUse { 384 . . // This can happen if direct sweeping already 385 . . // swept this span, but in that case the sweep 386 . . // generation should always be up-to-date. 387 . . if !(s.sweepgen == sl.sweepGen || s.sweepgen == sl.sweepGen+3) { 388 . . print("runtime: bad span s.state=", state, " s.sweepgen=", s.sweepgen, " sweepgen=", sl.sweepGen, "\n") 389 . . throw("non in-use span in unswept list") 390 . . } 391 . . continue 392 . . } 393 . 40ms if s, ok := sl.tryAcquire(s); ok { 394 . . // Sweep the span we found. 395 . . npages = s.npages 396 . 990ms if s.sweep(false) { 397 . . // Whole span was freed. Count it toward the 398 . . // page reclaimer credit since these pages can 399 . . // now be used for span allocation. 400 20ms 20ms mheap_.reclaimCredit.Add(npages) return Xadduintptr(&u.value, delta) types.go:420 401 . . } else { 402 . . // Span is still in-use, so this returned no 403 . . // pages to the heap and the span needs to 404 . . // move to the swept in-use list. 405 . . npages = 0 406 . . } 407 . . break 408 . . } 409 . . } 410 . 30ms sweep.active.end(sl) 411 . . 412 . . if noMoreWork { 413 . . // The sweep list is empty. There may still be 414 . . // concurrent sweeps running, but we're at least very 415 . . // close to done sweeping.
runtime.sweepone
/usr/lib/go/src/runtime/mgcsweep.go
Total: 20ms 20ms (flat, cum) 0.028% 443 . . }) 444 . . } 445 . . scavenger.ready() 446 . . } 447 . . 448 20ms 20ms gp.m.locks-- 449 . . return npages 450 . . } 451 . . 452 . . // isSweepDone reports whether all spans are swept. 453 . . //
runtime.(*mspan).ensureSwept
/usr/lib/go/src/runtime/mgcsweep.go
Total: 20ms 20ms (flat, cum) 0.028% 472 . . } 473 . . 474 . . // If this operation fails, then that means that there are 475 . . // no more spans to be swept. In this case, either s has already 476 . . // been swept, or is about to be acquired for sweeping and swept. 477 10ms 10ms sl := sweep.active.begin() if state&sweepDrainedMask != 0 { mgcsweep.go:151 478 . . if sl.valid { 479 . . // The caller must be sure that the span is a mSpanInUse span. 480 . . if s, ok := sl.tryAcquire(s); ok { 481 . . s.sweep(false) 482 . . sweep.active.end(sl) 483 . . return 484 . . } 485 . . sweep.active.end(sl) 486 . . } 487 . . 488 . . // Unfortunately we can't sweep the span ourselves. Somebody else 489 . . // got to it first. We don't have efficient means to wait, but that's 490 . . // OK, it will be swept fairly soon. 491 . . for { 492 . . spangen := atomic.Load(&s.sweepgen) 493 10ms 10ms if spangen == sl.sweepGen || spangen == sl.sweepGen+3 { 494 . . break 495 . . } 496 . . osyield() 497 . . } 498 . . }
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 40ms 40ms (flat, cum) 0.056% 527 . . if trace.ok() { 528 . . trace.GCSweepSpan(s.npages * pageSize) 529 . . traceRelease(trace) 530 . . } 531 . . 532 40ms 40ms mheap_.pagesSwept.Add(int64(s.npages)) return Xadd64(&u.value, delta) types.go:344 533 . . 534 . . spc := s.spanclass 535 . . size := s.elemsize 536 . . 537 . . // The allocBits indicate which unmarked objects don't need to be
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 20ms 20ms (flat, cum) 0.028% 548 . . // In such case we need to queue finalizer for execution, 549 . . // mark the object as live and preserve the profile special. 550 . . // 2. A tiny object can have several finalizers setup for different offsets. 551 . . // If such object is not marked, we need to queue all finalizers at once. 552 . . // Both 1 and 2 are possible at the same time. 553 20ms 20ms hadSpecials := s.specials != nil 554 . . siter := newSpecialsIter(s) 555 . . for siter.valid() { 556 . . // A finalizer can be set for an inner byte of an object, find object beginning. 557 . . objIndex := siter.s.offset / size 558 . . p := s.base() + objIndex*size
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 0 20ms (flat, cum) 0.028% 594 . . // Find the exact byte for which the special was setup 595 . . // (as opposed to object beginning). 596 . . special := siter.s 597 . . p := s.base() + special.offset 598 . . siter.unlinkAndNext() 599 . 20ms freeSpecial(special, unsafe.Pointer(p), size) 600 . . } 601 . . } 602 . . } else { 603 . . // object is still live 604 . . if siter.s.kind == _KindSpecialReachable {
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 0 240ms (flat, cum) 0.34% 651 . . } 652 . . } 653 . . 654 . . // Copy over and clear the inline mark bits if necessary. 655 . . if gcUsesSpanInlineMarkBits(s.elemsize) { 656 . 240ms s.moveInlineMarks(s.gcmarkBits) 657 . . } 658 . . 659 . . // Check for zombie objects. 660 . . if s.freeindex < s.nelems { 661 . . // Everything < freeindex is allocated and hence
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 20ms 90ms (flat, cum) 0.13% 674 . . } 675 . . } 676 . . } 677 . . 678 . . // Count the number of free objects in this span. 679 20ms 20ms nalloc := uint16(s.countAlloc()) mrkBits := *(*uint64)(unsafe.Pointer(s.gcmarkBits.bytep(i))) mbitmap.go:1509 680 . . nfreed := s.allocCount - nalloc 681 . . if nalloc > s.allocCount { 682 . . // The zombie check above should have caught this in 683 . . // more detail. 684 . . print("runtime: nelems=", s.nelems, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n") 685 . . throw("sweep increased allocation count") 686 . . } 687 . . 688 . . s.allocCount = nalloc 689 . . s.freeindex = 0 // reset allocation index to start of span. 690 . . s.freeIndexForScan = 0 691 . . if traceEnabled() { 692 . . getg().m.p.ptr().trace.reclaimed += uintptr(nfreed) * s.elemsize 693 . . } 694 . . 695 . . // gcmarkBits becomes the allocBits. 696 . . // get a fresh cleared gcmarkBits in preparation for next GC 697 . . s.allocBits = s.gcmarkBits 698 . 70ms s.gcmarkBits = newMarkBits(uintptr(s.nelems)) 699 . . 700 . . // refresh pinnerBits if they exists 701 . . if s.pinnerBits != nil { 702 . . s.refreshPinnerBits() 703 . . }
runtime.(*sweepLocked).sweep
/usr/lib/go/src/runtime/mgcsweep.go
Total: 40ms 590ms (flat, cum) 0.82% 768 . . // Only mark the span as needing zeroing if we've freed any 769 . . // objects, because a fresh span that had been allocated into, 770 . . // wasn't totally filled, but then swept, still has all of its 771 . . // free slots zeroed. 772 . . s.needzero = 1 773 . 20ms stats := memstats.heapStats.acquire() 774 . . atomic.Xadd64(&stats.smallFreeCount[spc.sizeclass()], int64(nfreed)) 775 . . memstats.heapStats.release() 776 . . 777 . . // Count the frees in the inconsistent, internal stats. 778 40ms 40ms gcController.totalFree.Add(int64(nfreed) * int64(s.elemsize)) return Xadd64(&u.value, delta) types.go:344 779 . . } 780 . . if !preserve { 781 . . // The caller may not have removed this span from whatever 782 . . // unswept set its on but taken ownership of the span for 783 . . // sweeping by updating sweepgen. If this span still is in 784 . . // an unswept set, then the mcentral will pop it off the 785 . . // set, check its sweepgen, and ignore it. 786 . . if nalloc == 0 { 787 . . // Free totally free span directly back to the heap. 788 . 520ms mheap_.freeSpan(s) systemstack(func() { mheap.go:1658 789 . . return true 790 . . } 791 . . // Return span back to the right mcentral list. 792 . . if nalloc == s.nelems { 793 . . mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s) 794 . . } else { 795 . 10ms mheap_.central[spc].mcentral.partialSwept(sweepgen).push(s) 796 . . } 797 . . } 798 . . } else if !preserve { 799 . . // Handle spans for large objects. 800 . . if nfreed != 0 {
runtime.deductSweepCredit
/usr/lib/go/src/runtime/mgcsweep.go
Total: 50ms 60ms (flat, cum) 0.084% 908 . . // It uses statistics gathered by the garbage collector to perform 909 . . // enough sweeping so that all pages are swept during the concurrent 910 . . // sweep phase between GC cycles. 911 . . // 912 . . // mheap_ must NOT be locked. 913 . 10ms func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) { 914 30ms 30ms if mheap_.sweepPagesPerByte == 0 { 915 . . // Proportional sweep is done or disabled. 916 . . return 917 . . } 918 . . 919 10ms 10ms trace := traceAcquire() if !traceEnabled() { traceruntime.go:189 return trace.enabled traceruntime.go:152 920 . . if trace.ok() { 921 . . trace.GCSweepStart() 922 . . traceRelease(trace) 923 . . } 924 . . 925 . . // Fix debt if necessary. 926 . . retry: 927 . . sweptBasis := mheap_.pagesSweptBasis.Load() 928 . . live := gcController.heapLive.Load() 929 10ms 10ms liveBasis := mheap_.sweepHeapLiveBasis 930 . . newHeapLive := spanBytes 931 . . if liveBasis < live { 932 . . // Only do this subtraction when we don't overflow. Otherwise, pagesTarget 933 . . // might be computed as something really huge, causing us to get stuck 934 . . // sweeping here until the next mark phase.
git.urbach.dev/cli/q/src/ssa.mergeIdentifiers
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 180ms 3.46s (flat, cum) 4.84% 8 . . "git.urbach.dev/cli/q/src/types" 9 . . ) 10 . . 11 . . // mergeIdentifiers merges identifier mappings into the successor, 12 . . // inserting phi functions where values differ across predecessors. 13 20ms 20ms func mergeIdentifiers(predecessor *Block, successor *Block) { 14 . . if predecessor.Identifiers.After == nil { 15 . . return 16 . . } 17 . . 18 . . if successor.Identifiers.After == nil { 19 10ms 150ms successor.Identifiers.Before = make(map[string]Value, len(predecessor.Identifiers.After)) 20 . 180ms successor.Identifiers.After = make(map[string]Value, len(predecessor.Identifiers.After)) 21 . . 22 . . if len(successor.Predecessors) == 1 { 23 70ms 1.01s maps.Copy(successor.Identifiers.Before, predecessor.Identifiers.After) for k, v := range src { maps.go:63 ⋮ ⋮ ⋮ dst[k] = v maps.go:64 ⋮ ⋮ dst[k] = v maps.go:64 24 50ms 880ms maps.Copy(successor.Identifiers.After, predecessor.Identifiers.After) for k, v := range src { maps.go:63 ⋮ ⋮ dst[k] = v maps.go:64 ⋮ for k, v := range src { maps.go:63 ⋮ ⋮ for k, v := range src { maps.go:63 25 . . return 26 . . } 27 . . } 28 . . 29 . . var ( 30 . . modifiedStructs []string 31 . 540ms names = collectIdentifierNames(predecessor, successor) 32 . . ) 33 . . 34 20ms 20ms for _, name := range names { 35 10ms 630ms mergeIdentifier(predecessor, successor, name, &modifiedStructs) 36 . . } 37 . . 38 . . // Structs that were modified in branches need to be recreated 39 . . // to use the new Phi values as their arguments. 40 . . for _, name := range modifiedStructs { 41 . . structure := successor.Identifiers.Before[name].(*Struct) 42 . . structType := types.Unwrap(structure.Typ).(*types.Struct) 43 . 10ms newStruct := &Struct{Typ: structure.Typ, Arguments: make(Arguments, len(structure.Arguments))} 44 . . 45 . . for i, field := range structType.Fields { 46 . 20ms newStruct.Arguments[i] = successor.Identifiers.Before[name+"."+field.Name] 47 . . } 48 . . 49 . . successor.ReplaceIdentifier(name, structure, newStruct) 50 . . }
git.urbach.dev/cli/q/src/ssa.collectIdentifierNames
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 90ms 540ms (flat, cum) 0.75% 51 . . } 52 . . 53 . . // collectIdentifierNames returns all identifier names from both maps in deterministic order. 54 . . func collectIdentifierNames(predecessor *Block, successor *Block) []string { 55 10ms 60ms keys := make([]string, 0, max(len(predecessor.Identifiers.After), len(successor.Identifiers.After))) 56 . . 57 . 90ms for name := range successor.Identifiers.Before { 58 20ms 20ms if !slices.Contains(keys, name) { return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ 59 20ms 20ms keys = append(keys, name) 60 . . } 61 . . } 62 . . 63 30ms 150ms for name := range predecessor.Identifiers.After { 64 10ms 40ms if !slices.Contains(keys, name) { return Index(s, v) >= 0 slices.go:118 if v == s[i] { slices.go:98 ⋮ 65 . . keys = append(keys, name) 66 . . } 67 . . } 68 . . 69 . 160ms slices.SortFunc(keys, func(a string, b string) int { pdqsortCmpFunc(x, 0, n, bits.Len(uint(n)), cmp) sort.go:32
git.urbach.dev/cli/q/src/ssa.collectIdentifierNames.func1
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 10ms 60ms (flat, cum) 0.084% 70 10ms 60ms return strings.Compare(b, a) return bytealg.CompareString(a, b) compare.go:16 return abigen_runtime_cmpstring(a, b) compare_native.go:15 ⋮ 71 . . }) 72 . . 73 . . return keys
git.urbach.dev/cli/q/src/ssa.mergeIdentifier
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 60ms 620ms (flat, cum) 0.87% 74 . . } 75 . . 76 . . // mergeIdentifier merges a single identifier into the successor. 77 10ms 10ms func mergeIdentifier(predecessor *Block, successor *Block, name string, modifiedStructs *[]string) { 78 10ms 130ms oldValue, oldExists := successor.Identifiers.Before[name] 79 20ms 130ms newValue, newExists := predecessor.Identifiers.After[name] 80 . . 81 20ms 20ms switch { 82 . . case oldExists: 83 . 240ms mergeOldIdentifier(successor, name, oldValue, newValue, newExists, modifiedStructs) 84 . . case newExists: 85 . 90ms mergeNewIdentifier(successor, name, newValue) 86 . . } 87 . . }
git.urbach.dev/cli/q/src/ssa.mergeOldIdentifier
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 60ms 240ms (flat, cum) 0.34% 89 . . // mergeOldIdentifier handles the case where the successor already has a binding for this name. 90 . . func mergeOldIdentifier(successor *Block, name string, oldValue Value, newValue Value, newExists bool, modifiedStructs *[]string) { 91 10ms 20ms if oldValue == newValue { 92 . . return 93 . . } 94 . . 95 10ms 10ms _, isStruct := oldValue.(*Struct) 96 . . 97 . . if isStruct { 98 . . *modifiedStructs = append(*modifiedStructs, name) 99 . . return 100 . . } 101 . . 102 10ms 10ms definedLocally := successor.Index(oldValue) != -1 if value == search { Block.go:78 103 . . 104 . . if definedLocally { 105 . . phi, isPhi := oldValue.(*Phi) 106 . . 107 . . if isPhi { 108 . . if newExists { 109 . . phi.Arguments = append(phi.Arguments, newValue) 110 . . } else { 111 . . phi.Arguments = append(phi.Arguments, Undefined) 112 . . } 113 . . } 114 . . 115 . . return 116 . . } 117 . . 118 . 40ms phi := &Phi{ 119 . . Name: name, 120 . 10ms Arguments: make([]Value, len(successor.Predecessors)-1, len(successor.Predecessors)), 121 . . Typ: oldValue.Type(), 122 . . } 123 . . 124 20ms 20ms for i := range phi.Arguments { 125 . . phi.Arguments[i] = oldValue 126 . . } 127 . . 128 10ms 60ms successor.InsertAt(0, phi) b.Instructions = slices.Insert(b.Instructions, index, values...) Block.go:88 ⋮ 129 . 70ms successor.ReplaceIdentifier(name, oldValue, phi) i.After[name] = newValue Identifiers.go:66 ⋮ if i.After[name] == oldValue { Identifiers.go:65 ⋮ i.Before[name] = newValue Identifiers.go:63 ⋮ if i.After[name] == oldValue { Identifiers.go:65 130 . . 131 . . if newExists { 132 . . phi.Arguments = append(phi.Arguments, newValue) 133 . . } else { 134 . . phi.Arguments = append(phi.Arguments, Undefined)
git.urbach.dev/cli/q/src/ssa.mergeNewIdentifier
/home/user/q/src/ssa/mergeIdentifiers.go
Total: 0 90ms (flat, cum) 0.13% 137 . . 138 . . // mergeNewIdentifier handles the case where only the predecessor has a binding for this name. 139 . . func mergeNewIdentifier(successor *Block, name string, newValue Value) { 140 . . phi := &Phi{ 141 . . Name: name, 142 . 20ms Arguments: make([]Value, len(successor.Predecessors)-1, len(successor.Predecessors)), 143 . 20ms Typ: newValue.Type(), 144 . . } 145 . . 146 . . for i := range phi.Arguments { 147 . . phi.Arguments[i] = Undefined 148 . . } 149 . . 150 . . successor.InsertAt(0, phi) 151 . 50ms successor.ReplaceIdentifier(name, nil, phi) i.After[name] = newValue Identifiers.go:66 ⋮ i.Before[name] = newValue Identifiers.go:63 152 . . phi.Arguments = append(phi.Arguments, newValue) 153 . . }
git.urbach.dev/cli/q/src/ssa.(*Liveness).AddUser
/home/user/q/src/ssa/Liveness.go
Total: 140ms 730ms (flat, cum) 1.02% 8 . . // Force interface implementation. 9 . . var _ HasUsers = (*Liveness)(nil) 10 . . 11 . . // AddUser adds a new user of the value. 12 . . func (l *Liveness) AddUser(user Value) { 13 140ms 730ms l.users = append(l.users, user) 14 . . } 15 . .
git.urbach.dev/cli/q/src/ssa.(*Liveness).RemoveUser
/home/user/q/src/ssa/Liveness.go
Total: 30ms 30ms (flat, cum) 0.042% 16 . . // RemoveUser removes a user of the value. 17 . . func (l *Liveness) RemoveUser(user Value) { 18 10ms 10ms for i, search := range l.users { 19 10ms 10ms if search == user { 20 10ms 10ms l.users = append(l.users[:i], l.users[i+1:]...) 21 . . return 22 . . } 23 . . }
git.urbach.dev/cli/q/src/ssa.(*Liveness).Users
/home/user/q/src/ssa/Liveness.go
Total: 220ms 220ms (flat, cum) 0.31% 25 . . 26 . . // Users returns the users of the value. 27 . . func (l *Liveness) Users() []Value { 28 220ms 220ms return l.users 29 . . }
git.urbach.dev/cli/q/src/codegen.(*Function).fixRegisterConflicts
/home/user/q/src/codegen/fixRegisterConflicts.go
Total: 370ms 1.37s (flat, cum) 1.92% 10 . . ) 11 . . 12 . . // fixRegisterConflicts checks for conflicts where 2 values that are live at the same time use the same register. 13 . . // It then assigns a new register to the value that was defined earlier. 14 . . func (f *Function) fixRegisterConflicts() { 15 30ms 30ms for _, step := range f.Steps { 16 40ms 40ms switch instr := step.Value.(type) { 17 . . case *ssa.BinaryOp: 18 . . if instr.Op == token.Shl || instr.Op == token.Shr { 19 . . if slices.Contains(f.CPU.ShiftRestricted, step.Register) { 20 . . f.assignFreeRegister(step) 21 . . } 22 . . } 23 . . 24 . . if step.Register != -1 { 25 . 20ms right := f.ValueToStep[instr.Right] 26 . . 27 . . if step.Register == right.Register { 28 . 10ms f.assignFreeRegister(right) step.Register = f.findFreeRegister(step) assignFreeRegister.go:6 29 . . } 30 . . 31 . 20ms left := f.ValueToStep[instr.Left] 32 . . 33 . . if instr.Op == token.Mod && step.Register == left.Register { 34 . . f.assignFreeRegister(left) 35 . . } 36 . . } 37 . . case *ssa.Register: 38 . . if f.build.Arch == config.ARM && step.Register == arm.SP { 39 . . f.assignFreeRegister(step) 40 . . } 41 . . } 42 . . 43 . 190ms clobbered := f.clobberedRegisters(step) 44 . . 45 70ms 70ms for i, live := range step.Live { 46 50ms 50ms if live.Register == -1 { 47 . . continue 48 . . } 49 . . 50 . 10ms if live.Value != step.Value { 51 20ms 20ms if slices.Contains(clobbered, live.Register) { 52 . 290ms f.assignFreeRegister(live) step.Register = f.findFreeRegister(step) assignFreeRegister.go:6 53 . . continue 54 . . } 55 . . 56 10ms 10ms field, isField := live.Value.(*ssa.Field) 57 . . 58 . . if isField { 59 . . _, isFieldFromCall := field.Tuple.(*ssa.Call) 60 . . 61 . . if isFieldFromCall && live.Index > step.Index && step.Register == f.CPU.Call.Out[field.Index] { 62 . . f.assignFreeRegister(step) 63 . . break 64 . . } 65 . . } 66 . . } 67 . . 68 80ms 80ms for _, previous := range step.Live[:i] { 69 50ms 50ms if previous.Register == -1 { 70 . . continue 71 . . } 72 . . 73 20ms 20ms if previous.Register != live.Register { 74 . . continue 75 . . } 76 . . 77 . . if previous.Index < live.Index { 78 . . f.assignFreeRegister(previous) 79 . . } else { 80 . 460ms f.assignFreeRegister(live) step.Register = f.findFreeRegister(step) assignFreeRegister.go:6 81 . . break 82 . . } 83 . . } 84 . . } 85 . . }
git.urbach.dev/cli/q/src/ssa.(*BinaryOp).Equals
/home/user/q/src/ssa/BinaryOp.go
Total: 30ms 30ms (flat, cum) 0.042% 16 . . Op token.Kind 17 . . Swapped bool 18 . . } 19 . . 20 . . // Equals returns true if the binary operations are equal. 21 30ms 30ms func (a *BinaryOp) Equals(v Value) bool { 22 . . b, sameType := v.(*BinaryOp) 23 . . 24 . . if !sameType { 25 . . return false 26 . . }
git.urbach.dev/cli/q/src/ssa.(*BinaryOp).Inputs
/home/user/q/src/ssa/BinaryOp.go
Total: 10ms 130ms (flat, cum) 0.18% 35 . . 36 . . return a.Op.IsCommutative() && a.Left == b.Right && a.Right == b.Left 37 . . } 38 . . 39 . . // Inputs returns the left and right operands. 40 10ms 10ms func (op *BinaryOp) Inputs() []Value { 41 . 120ms return []Value{op.Left, op.Right} 42 . . }
git.urbach.dev/cli/q/src/ssa.(*BinaryOp).IsPure
/home/user/q/src/ssa/BinaryOp.go
Total: 290ms 410ms (flat, cum) 0.57% 44 . . // IsPure returns true if both operands are constant. 45 130ms 130ms func (op *BinaryOp) IsPure() bool { 46 160ms 280ms return op.Left.IsPure() && op.Right.IsPure() 47 . . }
git.urbach.dev/cli/q/src/ssa.(*BinaryOp).Replace
/home/user/q/src/ssa/BinaryOp.go
Total: 30ms 40ms (flat, cum) 0.056% 49 . . // Replace replaces the left or right operand if it matches. 50 10ms 10ms func (op *BinaryOp) Replace(old Value, new Value) { 51 10ms 20ms if op.Left == old { 52 . . op.Left = new 53 . . } 54 . . 55 . . if op.Right == old { 56 . . op.Right = new 57 . . } 58 10ms 10ms } 59 . . 60 . . // String returns a human-readable representation of the binary operation. 61 . . func (op *BinaryOp) String() string {
git.urbach.dev/cli/q/src/ssa.(*BinaryOp).Type
/home/user/q/src/ssa/BinaryOp.go
Total: 10ms 20ms (flat, cum) 0.028% 63 . . } 64 . . 65 . . // Type returns the type of the result of the binary operation. 66 10ms 10ms func (op *BinaryOp) Type() types.Type { 67 . . if op.Op.IsComparison() { 68 . . return types.Bool 69 . . } 70 . . 71 . . if op.Swapped { 72 . . return op.Right.Type() 73 . . } 74 . . 75 . 10ms return op.Left.Type() 76 . . }
git.urbach.dev/cli/q/src/token.List.Instructions
/home/user/q/src/token/Instructions.go
Total: 360ms 10.87s (flat, cum) 15.20% 4 . . func (list List) Instructions(yield func(List) bool) { 5 . . start := 0 6 . . groupLevel := 0 7 . . blockLevel := 0 8 . . 9 110ms 110ms for i, t := range list { 10 140ms 140ms switch t.Kind { 11 10ms 10ms case NewLine: 12 10ms 10ms if start == i { 13 . . start = i + 1 14 . . continue 15 . . } 16 . . 17 10ms 10ms if groupLevel > 0 || blockLevel > 0 { 18 . . continue 19 . . } 20 . . 21 10ms 4.54s if !yield(list[start:i]) { 22 . . return 23 . . } 24 . . 25 . . start = i + 1 26 . . case GroupStart: 27 . . groupLevel++ 28 30ms 30ms case GroupEnd: 29 10ms 10ms groupLevel-- 30 20ms 20ms case BlockStart: 31 . . blockLevel++ 32 . . case BlockEnd: 33 . . blockLevel-- 34 . . 35 . . if groupLevel > 0 || blockLevel > 0 { 36 . . continue 37 . . } 38 . . 39 10ms 10ms if !list[start].Kind.IsBlock() { return k > ___BLOCKS___ && k < ___END_BLOCKS___ Kind.go:103 40 . . continue 41 . . } 42 . . 43 . 5.94s if !yield(list[start : i+1]) { 44 . . return 45 . . } 46 . . 47 . . start = i + 1 48 . . case EOF: 49 . . if start < i { 50 . . yield(list[start:i]) 51 . . } 52 . . 53 . . return 54 . . } 55 . . } 56 . . 57 . . if start < len(list) { 58 . 40ms yield(list[start:]) 59 . . } 60 . . }
git.urbach.dev/cli/q/src/core.(*Function).removeDeadCode
/home/user/q/src/core/removeDeadCode.go
Total: 200ms 900ms (flat, cum) 1.26% 5 . . 6 . . "git.urbach.dev/cli/q/src/ssa" 7 . . ) 8 . . 9 . . // removeDeadCode checks for dead values and also removes the ones that are partially allowed. 10 20ms 20ms func (f *Function) removeDeadCode(folded map[ssa.Value]struct{}) error { 11 . . for { 12 . . count := f.CountValues() 13 . . 14 180ms 880ms for _, block := range slices.Backward(f.Blocks) { if !yield(i, s[i]) { iter.go:29 for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ } removeDeadCode.go:23 ⋮ for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ if len(errors) > 0 { removeDeadCode.go:25 ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ } removeDeadCode.go:23 ⋮ ⋮ } removeDeadCode.go:23 ⋮ block.RemoveNilValues() removeDeadCode.go:29 ⋮ for i, value := range slices.Backward(block.Instructions) { removeDeadCode.go:17 if !yield(i, s[i]) { iter.go:29 ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 for _, block := range slices.Backward(f.Blocks) { removeDeadCode.go:14 15 . . var errors []error
git.urbach.dev/cli/q/src/core.(*Function).removeDeadCode-range1
/home/user/q/src/core/removeDeadCode.go
Total: 170ms 710ms (flat, cum) 0.99% 16 . . 17 170ms 710ms for i, value := range slices.Backward(block.Instructions) { for i := len(s) - 1; i >= 0; i-- { iter.go:28 ⋮ if !yield(i, s[i]) { iter.go:29 ⋮ ⋮ ⋮ ⋮ if !yield(i, s[i]) { iter.go:29 ⋮ ⋮
git.urbach.dev/cli/q/src/core.(*Function).removeDeadCode-range1-range1
/home/user/q/src/core/removeDeadCode.go
Total: 100ms 490ms (flat, cum) 0.69% 18 . 390ms err := f.removeDeadValue(block, i, value, folded) 19 . . 20 20ms 20ms if err != nil { 21 . . errors = append(errors, err) 22 . . } 23 80ms 80ms }
git.urbach.dev/cli/q/src/core.(*Function).removeDeadCode-range1
/home/user/q/src/core/removeDeadCode.go
Total: 10ms 170ms (flat, cum) 0.24% 25 10ms 10ms if len(errors) > 0 { 26 . . return errors[len(errors)-1] 27 . . } 28 . . 29 . 160ms block.RemoveNilValues() 30 . . } 31 . . 32 . . if f.CountValues() == count { 33 . . return nil 34 . . }
runtime.mapaccess1_fast64
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 10ms 190ms (flat, cum) 0.27% 11 . . "internal/runtime/sys" 12 . . "unsafe" 13 . . ) 14 . . 15 . . //go:linkname runtime_mapaccess1_fast64 runtime.mapaccess1_fast64 16 10ms 10ms func runtime_mapaccess1_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Pointer { 17 . 180ms p, _ := runtime_mapaccess2_fast64(typ, m, key) 18 . . return p 19 . . }
runtime.mapaccess2_fast64
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 130ms 130ms (flat, cum) 0.18% 20 . . 21 . . //go:linkname runtime_mapaccess2_fast64 runtime.mapaccess2_fast64 22 30ms 30ms func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Pointer, bool) { 23 . . if race.Enabled && m != nil { 24 . . callerpc := sys.GetCallerPC() 25 . . pc := abi.FuncPCABIInternal(runtime_mapaccess2_fast64) 26 . . race.ReadPC(unsafe.Pointer(m), callerpc, pc) 27 . . } 28 . . 29 . . if m == nil || m.Used() == 0 { 30 . . return unsafe.Pointer(&zeroVal[0]), false 31 . . } 32 . . 33 10ms 10ms if m.writing != 0 { 34 . . fatal("concurrent map read and map write") 35 . . return nil, false 36 . . } 37 . . 38 . . if m.dirLen == 0 { 39 . . g := groupReference{ 40 . . data: m.dirPtr, 41 . . } 42 10ms 10ms full := g.ctrls().matchFull() 43 . . slotKey := g.key(typ, 0) 44 . . var keyStride uintptr 45 . . if goexperiment.MapSplitGroup { 46 . . keyStride = 8 // keys are contiguous in split layout 47 . . } else { 48 . . keyStride = typ.KeyStride // == SlotSize in interleaved layout 49 . . } 50 . . var i uintptr 51 10ms 10ms for full != 0 { 52 30ms 30ms if key == *(*uint64)(slotKey) && full.lowestSet() { 53 . . if goexperiment.MapSplitGroup { 54 . . return g.elem(typ, i), true 55 . . } else { 56 20ms 20ms return unsafe.Pointer(uintptr(slotKey) + 8), true 57 . . } 58 . . } 59 10ms 10ms slotKey = unsafe.Pointer(uintptr(slotKey) + keyStride) 60 10ms 10ms full = full.shiftOutLowest() 61 . . i++ 62 . . } 63 . . return unsafe.Pointer(&zeroVal[0]), false 64 . . } 65 . .
runtime.mapaccess2_fast64
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 50ms 50ms (flat, cum) 0.07% 83 . . g := t.groups.group(typ, seq.offset) 84 . . 85 . . match := g.ctrls().matchH2(h2Hash) 86 . . 87 . . for match != 0 { 88 20ms 20ms i := match.first() 89 . . 90 10ms 10ms slotKey := g.key(typ, i) offset := typ.KeysOff + i*typ.KeyStride group.go:299 91 20ms 20ms if key == *(*uint64)(slotKey) { 92 . . if goexperiment.MapSplitGroup { 93 . . return g.elem(typ, i), true 94 . . } else { 95 . . return unsafe.Pointer(uintptr(slotKey) + 8), true 96 . . }
internal/runtime/maps.(*Map).putSlotSmallFastPtr
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 10ms 40ms (flat, cum) 0.056% 321 . . m.writing ^= 1 322 . . 323 . . return slotElem 324 . . } 325 . . 326 10ms 40ms func (m *Map) putSlotSmallFastPtr(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { 327 . . g := groupReference{ 328 . . data: m.dirPtr, 329 . . } 330 . . 331 . . match := g.ctrls().matchH2(h2(hash))
internal/runtime/maps.(*Map).putSlotSmallFastPtr
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 20ms 20ms (flat, cum) 0.028% 354 . . i := match.first() 355 . . 356 . . slotKey := g.key(typ, i) 357 . . *(*unsafe.Pointer)(slotKey) = key 358 . . 359 20ms 20ms slotElem := g.elem(typ, i) return unsafe.Pointer(uintptr(g.data) + offset) group.go:308 360 . . 361 . . g.ctrls().set(i, ctrl(h2(hash))) 362 . . m.used++ 363 . . 364 . . return slotElem
runtime.mapassign_fast64ptr
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 60ms 190ms (flat, cum) 0.27% 396 . . 397 . . // Key is a 64-bit pointer (only called on 64-bit GOARCH). 398 . . // 399 . . //go:linkname runtime_mapassign_fast64ptr runtime.mapassign_fast64ptr 400 . . func runtime_mapassign_fast64ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { 401 10ms 10ms if m == nil { 402 . . panic(errNilAssign) 403 . . } 404 . . if race.Enabled { 405 . . callerpc := sys.GetCallerPC() 406 . . pc := abi.FuncPCABIInternal(runtime_mapassign_fast64ptr) 407 . . race.WritePC(unsafe.Pointer(m), callerpc, pc) 408 . . } 409 10ms 10ms if m.writing != 0 { 410 . . fatal("concurrent map writes") 411 . . } 412 . . 413 . . var hash uintptr 414 . . // See the related comment in runtime_mapaccess2_fast32 415 10ms 10ms if memHashAESImplemented && UseAeshash { 416 . . hash = memHash64AES(uint64((uintptr)(key)), m.seed) 417 . . } else { 418 . . hash = memHash64Fallback(uint64((uintptr)(key)), m.seed) 419 . . } 420 . . 421 . . // Set writing after calling Hasher, since Hasher may panic, in which 422 . . // case we have not actually done a write. 423 20ms 20ms m.writing ^= 1 // toggle, see comment on writing 424 . . 425 10ms 10ms if m.dirPtr == nil { 426 . 70ms m.growToSmall(typ) 427 . . } 428 . . 429 . . if m.dirLen == 0 { 430 . 60ms elem := m.putSlotSmallFastPtr(typ, hash, key) 431 . . if elem == nil { 432 . . // Can't fit another entry, grow to full size map. 433 . . tab := m.growToTable(typ) 434 . . 435 . . elem = tab.uncheckedPutSlotForAssignFastPtr(typ, hash, key)
runtime.mapassign_fast64ptr
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 10ms 10ms (flat, cum) 0.014% 480 . . match = match.removeFirst() 481 . . } 482 . . 483 . . // No existing slot for this key in this group. Is this the end 484 . . // of the probe sequence? 485 10ms 10ms match = g.ctrls().matchEmptyOrDeleted() 486 . . if match == 0 { 487 . . continue // nothing but filled slots. Keep probing. 488 . . } 489 . . i := match.first() 490 . . if g.ctrls().get(i) == ctrlDeleted {
runtime.mapassign_fast64ptr
/usr/lib/go/src/internal/runtime/maps/runtime_fast64.go
Total: 10ms 10ms (flat, cum) 0.014% 510 . . if t.growthLeft == 0 { 511 . . t.pruneTombstones(typ, m) 512 . . } 513 . . 514 . . // If there is room left to grow, just insert the new entry. 515 10ms 10ms if t.growthLeft > 0 { 516 . . slotKey := g.key(typ, i) 517 . . *(*unsafe.Pointer)(slotKey) = key 518 . . 519 . . slotElem = g.elem(typ, i) 520 . .
runtime.roundupsize
/usr/lib/go/src/runtime/msize.go
Total: 300ms 300ms (flat, cum) 0.42% 13 . . 14 . . // Returns size of the memory block that mallocgc will allocate if you ask for the size, 15 . . // minus any inline space for metadata. 16 . . func roundupsize(size uintptr, noscan bool) (reqSize uintptr) { 17 . . reqSize = size 18 20ms 20ms if reqSize <= maxSmallSize-gc.MallocHeaderSize { 19 . . // Small object. 20 20ms 20ms if !noscan && reqSize > gc.MinSizeForMallocHeader { // !noscan && !heapBitsInSpan(reqSize) 21 . . reqSize += gc.MallocHeaderSize 22 . . } 23 . . // (reqSize - size) is either mallocHeaderSize or 0. We need to subtract mallocHeaderSize 24 . . // from the result if we have one, since mallocgc will add it back in. 25 20ms 20ms if reqSize <= gc.SmallSizeMax-8 { 26 200ms 200ms return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) ⋮ return (n + a - 1) / a stubs.go:400 ⋮ ⋮ ⋮ ⋮ return (n + a - 1) / a stubs.go:400 ⋮ ⋮ ⋮ ⋮ ⋮ 27 . . } 28 40ms 40ms return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass128[divRoundUp(reqSize-gc.SmallSizeMax, gc.LargeSizeDiv)]]) - (reqSize - size) ⋮ return (n + a - 1) / a stubs.go:400 ⋮ 29 . . } 30 . . // Large object. Align reqSize up to the next page. Check for overflow. 31 . . reqSize += pageSize - 1 32 . . if reqSize < size { 33 . . return size
runtime.(*spanInlineMarkBits).init
/usr/lib/go/src/runtime/mgcmark_greenteagc.go
Total: 40ms 40ms (flat, cum) 0.056% 134 . . if needzero { 135 . . // Use memclrNoHeapPointers to avoid having the compiler make a worse 136 . . // decision. We know that imb is both aligned and a nice power-of-two 137 . . // size that works well for wider SIMD instructions. The compiler likely 138 . . // has no idea that imb is aligned to 128 bytes. 139 30ms 30ms memclrNoHeapPointers(unsafe.Pointer(imb), unsafe.Sizeof(spanInlineMarkBits{})) 140 . . } 141 10ms 10ms imb.class = class 142 . . } 143 . . 144 . . // tryAcquire attempts to acquire the span for scanning. On success, the caller 145 . . // must queue the span for scanning or scan the span immediately. 146 . . func (imb *spanInlineMarkBits) tryAcquire() bool {
runtime.(*mspan).initInlineMarkBits
/usr/lib/go/src/runtime/mgcmark_greenteagc.go
Total: 10ms 40ms (flat, cum) 0.056% 185 . . func (s *mspan) initInlineMarkBits() { 186 . . if doubleCheckGreenTea && !gcUsesSpanInlineMarkBits(s.elemsize) { 187 . . throw("expected span with inline mark bits") 188 . . } 189 . . // Zeroing is only necessary if this span wasn't just freshly allocated from the OS. 190 10ms 40ms s.inlineMarkBits().init(s.spanclass, s.needzero != 0) 191 . . } 192 . . 193 . . // moveInlineMarks merges the span's inline mark bits into dst and clears them. 194 . . // 195 . . // gcUsesSpanInlineMarkBits(s.elemsize) must be true.
runtime.(*mspan).moveInlineMarks
/usr/lib/go/src/runtime/mgcmark_greenteagc.go
Total: 230ms 240ms (flat, cum) 0.34% 196 . . func (s *mspan) moveInlineMarks(dst *gcBits) { 197 . . if doubleCheckGreenTea && !gcUsesSpanInlineMarkBits(s.elemsize) { 198 . . throw("expected span with inline mark bits") 199 . . } 200 . . bytes := divRoundUp(uintptr(s.nelems), 8) 201 210ms 210ms imb := s.inlineMarkBits() 202 . . imbMarks := (*gc.ObjMask)(unsafe.Pointer(&imb.marks)) 203 . . for i := uintptr(0); i < bytes; i += goarch.PtrSize { 204 10ms 10ms marks := bswapIfBigEndian(imbMarks[i/goarch.PtrSize]) 205 . . if i/goarch.PtrSize == uintptr(len(imb.marks)+1)/goarch.PtrSize-1 { 206 10ms 10ms marks &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out class 207 . . } 208 . . *(*uintptr)(unsafe.Pointer(dst.bytep(i))) |= bswapIfBigEndian(marks) 209 . . } 210 . . if doubleCheckGreenTea && !s.spanclass.noscan() && imb.marks != imb.scans { 211 . . throw("marks don't match scans for span with pointer") 212 . . } 213 . . 214 . . // Reset the inline mark bits. 215 . 10ms imb.init(s.spanclass, true /* We know these bits are always dirty now. */) 216 . . } 217 . . 218 . . // inlineMarkBits returns the inline mark bits for the span. 219 . . // 220 . . // gcUsesSpanInlineMarkBits(s.elemsize) must be true.
runtime.scanObject
/usr/lib/go/src/runtime/mgcmark_greenteagc.go
Total: 10ms 10ms (flat, cum) 0.014% 1245 . . } 1246 . . 1247 . . // Keep track of farthest pointer we found, so we can 1248 . . // update heapScanWork. TODO: is there a better metric, 1249 . . // now that we can skip scalar portions pretty efficiently? 1250 10ms 10ms scanSize = addr - b + goarch.PtrSize 1251 . . 1252 . . // Work here is duplicated in scanblock and above. 1253 . . // If you make changes here, make changes there too. 1254 . . obj := *(*uintptr)(unsafe.Pointer(addr)) 1255 . .
runtime.(*gcControllerState).releaseNextGCMarkWorker
/usr/lib/go/src/runtime/mgcpacer.go
Total: 30ms 30ms (flat, cum) 0.042% 894 . . // 895 . . // This function is allowed to have write barriers because it is called from 896 . . // the portion of schedule. 897 . . // 898 . . //go:yeswritebarrierrec 899 20ms 20ms func (c *gcControllerState) releaseNextGCMarkWorker(pp *p) { 900 . . node := pp.nextGCMarkWorker 901 10ms 10ms if node == nil { 902 . . return 903 . . } 904 . . 905 . . c.markWorkerStop(pp.gcMarkWorkerMode, 0) 906 . . gcBgMarkWorkerPool.push(&node.node)
runtime.(*gcControllerState).update
/usr/lib/go/src/runtime/mgcpacer.go
Total: 120ms 120ms (flat, cum) 0.17% 949 . . } 950 . . } 951 . . 952 . . func (c *gcControllerState) update(dHeapLive, dHeapScan int64) { 953 . . if dHeapLive != 0 { 954 10ms 10ms trace := traceAcquire() if !traceEnabled() { traceruntime.go:189 return trace.enabled traceruntime.go:152 955 100ms 100ms live := gcController.heapLive.Add(dHeapLive) return Xadd64(&u.value, delta) types.go:344 956 . . if trace.ok() { 957 . . // gcController.heapLive changed. 958 . . trace.HeapAlloc(live) 959 . . traceRelease(trace) 960 . . } 961 . . } 962 . . if gcBlackenEnabled == 0 { 963 . . // Update heapScan when we're not in a current GC. It is fixed 964 . . // at the beginning of a cycle. 965 . . if dHeapScan != 0 { 966 10ms 10ms gcController.heapScan.Add(dHeapScan) return Xadd64(&u.value, delta) types.go:344 967 . . } 968 . . } else { 969 . . // gcController.heapLive changed. 970 . . c.revise() 971 . . }
runtime.(*gcControllerState).addScannableStack
/usr/lib/go/src/runtime/mgcpacer.go
Total: 70ms 70ms (flat, cum) 0.098% 974 . . func (c *gcControllerState) addScannableStack(pp *p, amount int64) { 975 . . if pp == nil { 976 . . c.maxStackScan.Add(amount) 977 . . return 978 . . } 979 10ms 10ms pp.maxStackScanDelta += amount 980 . . if pp.maxStackScanDelta >= maxStackScanSlack || pp.maxStackScanDelta <= -maxStackScanSlack { 981 60ms 60ms c.maxStackScan.Add(pp.maxStackScanDelta) return Xadd64(&u.value, delta) types.go:344 ⋮ 982 . . pp.maxStackScanDelta = 0 983 . . } 984 . . } 985 . . 986 . . func (c *gcControllerState) addGlobals(amount int64) {
runtime.(*gcControllerState).heapGoalInternal
/usr/lib/go/src/runtime/mgcpacer.go
Total: 20ms 60ms (flat, cum) 0.084% 995 . . 996 . . // heapGoalInternal is the implementation of heapGoal which returns additional 997 . . // information that is necessary for computing the trigger. 998 . . // 999 . . // The returned minTrigger is always <= goal. 1000 10ms 10ms func (c *gcControllerState) heapGoalInternal() (goal, minTrigger uint64) { 1001 . . // Start with the goal calculated for gcPercent. 1002 . . goal = c.gcPercentHeapGoal.Load() 1003 . . 1004 . . // Check if the memory-limit-based goal is smaller, and if so, pick that. 1005 10ms 50ms if newGoal := c.memoryLimitHeapGoal(); newGoal < goal { 1006 . . goal = newGoal 1007 . . } else { 1008 . . // We're not limited by the memory limit goal, so perform a series of 1009 . . // adjustments that might move the goal forward in a variety of circumstances. 1010 . .
runtime.(*gcControllerState).memoryLimitHeapGoal
/usr/lib/go/src/runtime/mgcpacer.go
Total: 40ms 40ms (flat, cum) 0.056% 1149 . . goal = headroom 1150 . . } else { 1151 . . goal = goal - headroom 1152 . . } 1153 . . // Don't let us go below the live heap. A heap goal below the live heap doesn't make sense. 1154 40ms 40ms if goal < c.heapMarked { 1155 . . goal = c.heapMarked 1156 . . } 1157 . . return goal 1158 . . } 1159 . .
runtime.(*gcControllerState).trigger
/usr/lib/go/src/runtime/mgcpacer.go
Total: 10ms 70ms (flat, cum) 0.098% 1184 . . // The returned value may be compared against heapLive to determine whether 1185 . . // the GC should trigger. Thus, the GC trigger condition should be (but may 1186 . . // not be, in the case of small movements for efficiency) checked whenever 1187 . . // the heap goal may change. 1188 . . func (c *gcControllerState) trigger() (uint64, uint64) { 1189 10ms 70ms goal, minTrigger := c.heapGoalInternal() 1190 . . 1191 . . // Invariant: the trigger must always be less than the heap goal. 1192 . . // 1193 . . // Note that the memory limit sets a hard maximum on our heap goal, 1194 . . // but the live heap may grow beyond it.
git.urbach.dev/cli/q/src/core.(*Function).define
/home/user/q/src/core/define.go
Total: 290ms 4.32s (flat, cum) 6.04% 7 . . "git.urbach.dev/cli/q/src/token" 8 . . "git.urbach.dev/cli/q/src/types" 9 . . ) 10 . . 11 . . // define gives a value an identifier. 12 20ms 20ms func (f *Function) define(left *expression.Expression, right *expression.Expression, isAssign bool) error { 13 . 3.24s rightValue, err := f.evaluateRight(right) 14 . . 15 10ms 10ms if err != nil { 16 . . return err 17 . . } 18 . . 19 . 90ms name := left.String(f.File.Bytes) expr.write(&builder, source) Expression.go:161 20 . . 21 . . if name == "_" { 22 . . return nil 23 . . } 24 . . 25 10ms 10ms root := left.Parent 26 . . 27 10ms 10ms if isAssign && root.Token.Kind != token.Assign { 28 . . operator := removeAssign(root.Token.Kind) 29 . 10ms leftValueDeref, err := f.evaluateRight(left) 30 . . 31 . . if err != nil { 32 . . return err 33 . . } 34 . . 35 . 40ms rightValue = f.Append(&ssa.BinaryOp{ 36 . . Op: operator, 37 . . Left: leftValueDeref, 38 . . Right: rightValue, 39 . 10ms Source: root.Source(), 40 . . }) 41 . . } 42 . . 43 . 130ms leftValue, err := f.validateLeft(left, right, name, rightValue.Type(), isAssign) 44 . . 45 . . if err != nil { 46 . . return err 47 . . } 48 . . 49 . . call, isCall := rightValue.(*ssa.Call) 50 . . 51 . . if isCall && len(call.Func.Typ.Output) != 1 { 52 . . return errors.New(&DefinitionCountMismatch{Function: call.Func.String(), Count: 1, ExpectedCount: len(call.Func.Typ.Output)}, f.File, left.Source()) 53 . . } 54 . . 55 . . // If the value we got was a value that is stored in a variable, 56 . . // it must have been returned from the optimizer as a cached value. 57 . . // We want to assure that every named variable creates a copy of 58 . . // another named variable instead of using the cached value itself 59 . . // because it could lead to incorrect optimizations. 60 220ms 540ms if f.IsIdentified(rightValue) { if block.IsIdentified(value) { IR.go:70 for _, existing := range i.After { Identifiers.go:52 ⋮ ⋮ for _, block := range ir.Blocks { IR.go:69 ⋮ if block.IsIdentified(value) { IR.go:70 if existing == value { Identifiers.go:53 ⋮ ⋮ ⋮ for _, block := range ir.Blocks { IR.go:69 ⋮ if block.IsIdentified(value) { IR.go:70 if existing == value { Identifiers.go:53 ⋮ for _, block := range ir.Blocks { IR.go:69 ⋮ if block.IsIdentified(value) { IR.go:70 for _, existing := range i.After { Identifiers.go:52 ⋮ ⋮ ⋮ 61 . . _, isResource := rightValue.Type().(*types.Resource) 62 . . 63 . . if isResource { 64 . . f.Block().Unidentify(rightValue) 65 . . } else { 66 . 10ms rightValue = f.copy(rightValue, left.Source()) 67 . . } 68 . . } 69 . . 70 10ms 10ms global, isGlobal := leftValue.(*ssa.Global) 71 . . 72 . . if isGlobal { 73 . . zero := f.Append(&ssa.Int{Int: 0}) 74 . . 75 . . memory := &ssa.Memory{ 76 . . Typ: global.Typ.(*types.Pointer).To, 77 . . Address: global, 78 . . Index: zero, 79 . . Source: global.Source, 80 . . } 81 . . 82 . 70ms return f.store(memory, rightValue) 83 . . } 84 . . 85 . . if !isCall { 86 . . structure, isStructType := rightValue.(*ssa.Struct) 87 . . 88 . . if isStructType { 89 10ms 10ms for i, field := range types.Unwrap(structure.Typ).(*types.Struct).Fields { 90 . 40ms f.Block().Identify(name+"."+field.Name, structure.Arguments[i]) i.After[name] = value Identifiers.go:47 ⋮ 91 . . } 92 . . } 93 . . 94 . 70ms f.Block().Identify(name, rightValue) i.After[name] = value Identifiers.go:47 95 . . return nil 96 . . } 97 . . 98 . . structure, isStructType := types.Unwrap(rightValue.Type()).(*types.Struct) 99 . .