diff options
Diffstat (limited to 'src/runtime/asm_amd64.s')
-rw-r--r-- | src/runtime/asm_amd64.s | 147 |
1 files changed, 70 insertions, 77 deletions
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 39d7c78f2..6e3f5ff6c 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "zasm_GOOS_GOARCH.h" +#include "go_asm.h" +#include "go_tls.h" #include "funcdata.h" #include "textflag.h" @@ -47,7 +48,7 @@ nocpuinfo: // update stackguard after _cgo_init MOVQ $runtime·g0(SB), CX MOVQ (g_stack+stack_lo)(CX), AX - ADDQ $const_StackGuard, AX + ADDQ $const__StackGuard, AX MOVQ AX, g_stackguard0(CX) MOVQ AX, g_stackguard1(CX) @@ -189,55 +190,41 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8 JMP AX RET -// switchtoM is a dummy routine that onM leaves at the bottom +// systemstack_switch is a dummy routine that systemstack leaves at the bottom // of the G stack. We need to distinguish the routine that // lives at the bottom of the G stack from the one that lives -// at the top of the M stack because the one at the top of -// the M stack terminates the stack walk (see topofstack()). -TEXT runtime·switchtoM(SB), NOSPLIT, $0-0 +// at the top of the system stack because the one at the top of +// the system stack terminates the stack walk (see topofstack()). +TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 RET -// func onM_signalok(fn func()) -TEXT runtime·onM_signalok(SB), NOSPLIT, $0-8 +// func systemstack(fn func()) +TEXT runtime·systemstack(SB), NOSPLIT, $0-8 + MOVQ fn+0(FP), DI // DI = fn get_tls(CX) MOVQ g(CX), AX // AX = g MOVQ g_m(AX), BX // BX = m + MOVQ m_gsignal(BX), DX // DX = gsignal CMPQ AX, DX - JEQ ongsignal - JMP runtime·onM(SB) - -ongsignal: - MOVQ fn+0(FP), DI // DI = fn - MOVQ DI, DX - MOVQ 0(DI), DI - CALL DI - RET - -// func onM(fn func()) -TEXT runtime·onM(SB), NOSPLIT, $0-8 - MOVQ fn+0(FP), DI // DI = fn - get_tls(CX) - MOVQ g(CX), AX // AX = g - MOVQ g_m(AX), BX // BX = m + JEQ noswitch MOVQ m_g0(BX), DX // DX = g0 CMPQ AX, DX - JEQ onm + JEQ noswitch MOVQ m_curg(BX), BP CMPQ AX, BP - JEQ oncurg + JEQ switch - // Not g0, not curg. Must be gsignal, but that's not allowed. - // Hide call from linker nosplit analysis. - MOVQ $runtime·badonm(SB), AX + // Bad: g is not gsignal, not g0, not curg. What is it? + MOVQ $runtime·badsystemstack(SB), AX CALL AX -oncurg: +switch: // save our state in g->sched. Pretend to - // be switchtoM if the G stack is scanned. - MOVQ $runtime·switchtoM(SB), BP + // be systemstack_switch if the G stack is scanned. + MOVQ $runtime·systemstack_switch(SB), BP MOVQ BP, (g_sched+gobuf_pc)(AX) MOVQ SP, (g_sched+gobuf_sp)(AX) MOVQ AX, (g_sched+gobuf_g)(AX) @@ -245,7 +232,7 @@ oncurg: // switch to g0 MOVQ DX, g(CX) MOVQ (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called onM on g0, to stop traceback + // make it look like mstart called systemstack on g0, to stop traceback SUBQ $8, BX MOVQ $runtime·mstart(SB), DX MOVQ DX, 0(BX) @@ -266,7 +253,7 @@ oncurg: MOVQ $0, (g_sched+gobuf_sp)(AX) RET -onm: +noswitch: // already on m stack, just call directly MOVQ DI, DX MOVQ 0(DI), DI @@ -461,11 +448,11 @@ TEXT runtime·cas64(SB), NOSPLIT, $0-25 MOVQ new+16(FP), CX LOCK CMPXCHGQ CX, 0(BX) - JNZ cas64_fail + JNZ fail MOVL $1, AX MOVB AX, ret+24(FP) RET -cas64_fail: +fail: MOVL $0, AX MOVB AX, ret+24(FP) RET @@ -489,7 +476,7 @@ TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16 // return 1; // } else // return 0; -TEXT runtime·casp(SB), NOSPLIT, $0-25 +TEXT runtime·casp1(SB), NOSPLIT, $0-25 MOVQ ptr+0(FP), BX MOVQ old+8(FP), AX MOVQ new+16(FP), CX @@ -541,7 +528,7 @@ TEXT runtime·xchg64(SB), NOSPLIT, $0-24 MOVQ AX, ret+16(FP) RET -TEXT runtime·xchgp(SB), NOSPLIT, $0-24 +TEXT runtime·xchgp1(SB), NOSPLIT, $0-24 MOVQ ptr+0(FP), BX MOVQ new+8(FP), AX XCHGQ AX, 0(BX) @@ -559,7 +546,7 @@ again: JNZ again RET -TEXT runtime·atomicstorep(SB), NOSPLIT, $0-16 +TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-16 MOVQ ptr+0(FP), BX MOVQ val+8(FP), AX XCHGQ AX, 0(BX) @@ -726,7 +713,7 @@ needm: // the same SP back to m->sched.sp. That seems redundant, // but if an unrecovered panic happens, unwindm will // restore the g->sched.sp from the stack location - // and then onM will try to use it. If we don't set it here, + // and then systemstack will try to use it. If we don't set it here, // that restored SP will be uninitialized (typically 0) and // will not be usable. MOVQ m_g0(BP), SI @@ -890,24 +877,24 @@ TEXT runtime·aeshashbody(SB),NOSPLIT,$0-32 MOVO runtime·aeskeysched+0(SB), X2 MOVO runtime·aeskeysched+16(SB), X3 CMPQ CX, $16 - JB aessmall -aesloop: + JB small +loop: CMPQ CX, $16 - JBE aesloopend + JBE loopend MOVOU (AX), X1 AESENC X2, X0 AESENC X1, X0 SUBQ $16, CX ADDQ $16, AX - JMP aesloop + JMP loop // 1-16 bytes remaining -aesloopend: +loopend: // This load may overlap with the previous load above. // We'll hash some bytes twice, but that's ok. MOVOU -16(AX)(CX*1), X1 JMP partial // 0-15 bytes -aessmall: +small: TESTQ CX, CX JE finalize // 0 bytes @@ -1050,18 +1037,18 @@ TEXT runtime·eqstring(SB),NOSPLIT,$0-33 MOVQ s1len+8(FP), AX MOVQ s2len+24(FP), BX CMPQ AX, BX - JNE different + JNE noteq MOVQ s1str+0(FP), SI MOVQ s2str+16(FP), DI CMPQ SI, DI - JEQ same + JEQ eq CALL runtime·memeqbody(SB) MOVB AX, v+32(FP) RET -same: +eq: MOVB $1, v+32(FP) RET -different: +noteq: MOVB $0, v+32(FP) RET @@ -1184,29 +1171,29 @@ TEXT runtime·cmpbytes(SB),NOSPLIT,$0-56 // AX = 1/0/-1 TEXT runtime·cmpbody(SB),NOSPLIT,$0-0 CMPQ SI, DI - JEQ cmp_allsame + JEQ allsame CMPQ BX, DX MOVQ DX, BP CMOVQLT BX, BP // BP = min(alen, blen) = # of bytes to compare CMPQ BP, $8 - JB cmp_small + JB small -cmp_loop: +loop: CMPQ BP, $16 - JBE cmp_0through16 + JBE _0through16 MOVOU (SI), X0 MOVOU (DI), X1 PCMPEQB X0, X1 PMOVMSKB X1, AX XORQ $0xffff, AX // convert EQ to NE - JNE cmp_diff16 // branch if at least one byte is not equal + JNE diff16 // branch if at least one byte is not equal ADDQ $16, SI ADDQ $16, DI SUBQ $16, BP - JMP cmp_loop + JMP loop // AX = bit mask of differences -cmp_diff16: +diff16: BSFQ AX, BX // index of first byte that differs XORQ AX, AX MOVB (SI)(BX*1), CX @@ -1216,21 +1203,21 @@ cmp_diff16: RET // 0 through 16 bytes left, alen>=8, blen>=8 -cmp_0through16: +_0through16: CMPQ BP, $8 - JBE cmp_0through8 + JBE _0through8 MOVQ (SI), AX MOVQ (DI), CX CMPQ AX, CX - JNE cmp_diff8 -cmp_0through8: + JNE diff8 +_0through8: MOVQ -8(SI)(BP*1), AX MOVQ -8(DI)(BP*1), CX CMPQ AX, CX - JEQ cmp_allsame + JEQ allsame // AX and CX contain parts of a and b that differ. -cmp_diff8: +diff8: BSWAPQ AX // reverse order of bytes BSWAPQ CX XORQ AX, CX @@ -1241,44 +1228,44 @@ cmp_diff8: RET // 0-7 bytes in common -cmp_small: +small: LEAQ (BP*8), CX // bytes left -> bits left NEGQ CX // - bits lift (== 64 - bits left mod 64) - JEQ cmp_allsame + JEQ allsame // load bytes of a into high bytes of AX CMPB SI, $0xf8 - JA cmp_si_high + JA si_high MOVQ (SI), SI - JMP cmp_si_finish -cmp_si_high: + JMP si_finish +si_high: MOVQ -8(SI)(BP*1), SI SHRQ CX, SI -cmp_si_finish: +si_finish: SHLQ CX, SI // load bytes of b in to high bytes of BX CMPB DI, $0xf8 - JA cmp_di_high + JA di_high MOVQ (DI), DI - JMP cmp_di_finish -cmp_di_high: + JMP di_finish +di_high: MOVQ -8(DI)(BP*1), DI SHRQ CX, DI -cmp_di_finish: +di_finish: SHLQ CX, DI BSWAPQ SI // reverse order of bytes BSWAPQ DI XORQ SI, DI // find bit differences - JEQ cmp_allsame + JEQ allsame BSRQ DI, CX // index of highest bit difference SHRQ CX, SI // move a's bit to bottom ANDQ $1, SI // mask bit LEAQ -1(SI*2), AX // 1/0 => +1/-1 RET -cmp_allsame: +allsame: XORQ AX, AX XORQ CX, CX CMPQ BX, DX @@ -1313,7 +1300,7 @@ TEXT runtime·indexbytebody(SB),NOSPLIT,$0 MOVQ SI, DI CMPQ BX, $16 - JLT indexbyte_small + JLT small // round up to first 16-byte boundary TESTQ $15, SI @@ -1371,7 +1358,7 @@ failure: RET // handle for lengths < 16 -indexbyte_small: +small: MOVQ BX, CX REPN; SCASB JZ success @@ -2235,3 +2222,9 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return + +TEXT runtime·getg(SB),NOSPLIT,$0-8 + get_tls(CX) + MOVQ g(CX), AX + MOVQ AX, ret+0(FP) + RET |