From cc319dc33cc73b0ec368864a4220604170acb674 Mon Sep 17 00:00:00 2001 From: weidai Date: Tue, 10 Mar 2009 02:56:19 +0000 Subject: add x86/x64 assembly for SHA-256, add DEFAULT_CHANNEL and AAD_CHANNEL, fix macChannel for AuthenticatedEncryptionFilter git-svn-id: svn://svn.code.sf.net/p/cryptopp/code/trunk/c5@440 57ff6487-cd31-0410-9ec3-f628ee90f5f0 --- sha.cpp | 385 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 356 insertions(+), 29 deletions(-) (limited to 'sha.cpp') diff --git a/sha.cpp b/sha.cpp index 7322543..905d12d 100644 --- a/sha.cpp +++ b/sha.cpp @@ -3,14 +3,21 @@ // Steve Reid implemented SHA-1. Wei Dai implemented SHA-2. // Both are in the public domain. +// use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM sha.cpp" to generate MASM code + #include "pch.h" #ifndef CRYPTOPP_IMPORTS +#ifndef CRYPTOPP_GENERATE_X64_MASM #include "sha.h" #include "misc.h" #include "cpu.h" +#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#include +#endif + NAMESPACE_BEGIN(CryptoPP) // start of Steve Reid's code @@ -93,7 +100,7 @@ void SHA256::InitState(HashWordType *state) memcpy(state, s, sizeof(s)); } -static const word32 SHA256_K[64] = { +extern const word32 SHA256_K[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, @@ -112,10 +119,333 @@ static const word32 SHA256_K[64] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 }; +#endif // #ifndef CRYPTOPP_GENERATE_X64_MASM + +#if defined(CRYPTOPP_X86_ASM_AVAILABLE) || defined(CRYPTOPP_GENERATE_X64_MASM) + +#pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code + +static void CRYPTOPP_FASTCALL X86_SHA256_HashBlocks(word32 *state, const word32 *data, size_t len) +{ + #define LOCALS_SIZE 8*4 + 16*4 + 4*WORD_SZ + #define H(i) [BASE+ASM_MOD(1024+7-(i),8)*4] + #define G(i) H(i+1) + #define F(i) H(i+2) + #define E(i) H(i+3) + #define D(i) H(i+4) + #define C(i) H(i+5) + #define B(i) H(i+6) + #define A(i) H(i+7) + #define Wt(i) BASE+8*4+ASM_MOD(1024+15-(i),16)*4 + #define Wt_2(i) Wt((i)-2) + #define Wt_15(i) Wt((i)-15) + #define Wt_7(i) Wt((i)-7) + #define K_END [BASE+8*4+16*4+0*WORD_SZ] + #define STATE_SAVE [BASE+8*4+16*4+1*WORD_SZ] + #define DATA_SAVE [BASE+8*4+16*4+2*WORD_SZ] + #define DATA_END [BASE+8*4+16*4+3*WORD_SZ] + #define Kt(i) WORD_REG(si)+(i)*4 +#if CRYPTOPP_BOOL_X86 + #define BASE esp+4 +#elif defined(__GNUC__) + #define BASE r8 +#else + #define BASE rsp +#endif + +#define RA0(i, edx, edi) \ + AS2( add edx, [Kt(i)] )\ + AS2( add edx, [Wt(i)] )\ + AS2( add edx, H(i) )\ + +#define RA1(i, edx, edi) + +#define RB0(i, edx, edi) + +#define RB1(i, edx, edi) \ + AS2( mov AS_REG_7d, [Wt_2(i)] )\ + AS2( mov edi, [Wt_15(i)])\ + AS2( mov ebx, AS_REG_7d )\ + AS2( shr AS_REG_7d, 10 )\ + AS2( ror ebx, 17 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( ror ebx, 2 )\ + AS2( xor ebx, AS_REG_7d )/* s1(W_t-2) */\ + AS2( add ebx, [Wt_7(i)])\ + AS2( mov AS_REG_7d, edi )\ + AS2( shr AS_REG_7d, 3 )\ + AS2( ror edi, 7 )\ + AS2( add ebx, [Wt(i)])/* s1(W_t-2) + W_t-7 + W_t-16 */\ + AS2( xor AS_REG_7d, edi )\ + AS2( add edx, [Kt(i)])\ + AS2( ror edi, 11 )\ + AS2( add edx, H(i) )\ + AS2( xor AS_REG_7d, edi )/* s0(W_t-15) */\ + AS2( add AS_REG_7d, ebx )/* W_t = s1(W_t-2) + W_t-7 + s0(W_t-15) W_t-16*/\ + AS2( mov [Wt(i)], AS_REG_7d)\ + AS2( add edx, AS_REG_7d )\ + +#define ROUND(i, r, eax, ecx, edi, edx)\ + /* in: edi = E */\ + /* unused: eax, ecx, temp: ebx, AS_REG_7d, out: edx = T1 */\ + AS2( mov edx, F(i) )\ + AS2( xor edx, G(i) )\ + AS2( and edx, edi )\ + AS2( xor edx, G(i) )/* Ch(E,F,G) = (G^(E&(F^G))) */\ + AS2( mov AS_REG_7d, edi )\ + AS2( ror edi, 6 )\ + AS2( ror AS_REG_7d, 25 )\ + RA##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\ + AS2( xor AS_REG_7d, edi )\ + AS2( ror edi, 5 )\ + AS2( xor AS_REG_7d, edi )/* S1(E) */\ + AS2( add edx, AS_REG_7d )/* T1 = S1(E) + Ch(E,F,G) + H + Wt + Kt */\ + RB##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\ + /* in: ecx = A, eax = B^C, edx = T1 */\ + /* unused: edx, temp: ebx, AS_REG_7d, out: eax = A, ecx = B^C, edx = E */\ + AS2( mov ebx, ecx )\ + AS2( xor ecx, B(i) )/* A^B */\ + AS2( and eax, ecx )\ + AS2( xor eax, B(i) )/* Maj(A,B,C) = B^((A^B)&(B^C) */\ + AS2( mov AS_REG_7d, ebx )\ + AS2( ror ebx, 2 )\ + AS2( add eax, edx )/* T1 + Maj(A,B,C) */\ + AS2( add edx, D(i) )\ + AS2( mov D(i), edx )\ + AS2( ror AS_REG_7d, 22 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( ror ebx, 11 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( add eax, AS_REG_7d )/* T1 + S0(A) + Maj(A,B,C) */\ + AS2( mov H(i), eax )\ + +#define SWAP_COPY(i) \ + AS2( mov WORD_REG(bx), [WORD_REG(dx)+i*WORD_SZ])\ + AS1( bswap WORD_REG(bx))\ + AS2( mov [Wt(i*(1+CRYPTOPP_BOOL_X64)+CRYPTOPP_BOOL_X64)], WORD_REG(bx)) + +#if defined(__GNUC__) + #if CRYPTOPP_BOOL_X64 + __m128i workspace[(LOCALS_SIZE+15)/16]; + #endif + __asm__ __volatile__ + ( + #if CRYPTOPP_BOOL_X64 + "movq %4, %%r8;" + #endif + ".intel_syntax noprefix;" +#elif defined(CRYPTOPP_GENERATE_X64_MASM) + ALIGN 8 + X86_SHA256_HashBlocks PROC FRAME + rex_push_reg rsi + push_reg rdi + push_reg rbx + push_reg rbp + alloc_stack(LOCALS_SIZE+8) + .endprolog + mov rdi, r8 + lea rsi, [?SHA256_K@CryptoPP@@3QBIB + 48*4] +#endif + +#if CRYPTOPP_BOOL_X86 + #ifndef __GNUC__ + AS2( mov edi, [len]) + AS2( lea WORD_REG(si), [SHA256_K+48*4]) + #endif + #if !defined(_MSC_VER) || (_MSC_VER < 1300) + AS_PUSH_IF86(bx) + #endif + + AS_PUSH_IF86(bp) + AS2( mov ebx, esp) + AS2( and esp, -16) + AS2( sub WORD_REG(sp), LOCALS_SIZE) + AS_PUSH_IF86(bx) +#endif + AS2( mov STATE_SAVE, WORD_REG(cx)) + AS2( mov DATA_SAVE, WORD_REG(dx)) + AS2( add WORD_REG(di), WORD_REG(dx)) + AS2( mov DATA_END, WORD_REG(di)) + AS2( mov K_END, WORD_REG(si)) + +#if CRYPTOPP_BOOL_X86 + AS2( test edi, 1) + ASJ( jnz, 2, f) +#endif + + AS2( movdqa xmm0, XMMWORD_PTR [WORD_REG(cx)+0*16]) + AS2( movdqa xmm1, XMMWORD_PTR [WORD_REG(cx)+1*16]) + +#if CRYPTOPP_BOOL_X86 + ASJ( jmp, 0, f) + ASL(2) // non-SSE2 + AS2( mov esi, ecx) + AS2( lea edi, A(0)) + AS2( mov ecx, 8) + AS1( rep movsd) + AS2( mov esi, K_END) + ASJ( jmp, 3, f) +#endif + + ASL(0) + AS2( movdqa E(0), xmm1) + AS2( movdqa A(0), xmm0) +#if CRYPTOPP_BOOL_X86 + ASL(3) +#endif + AS2( sub WORD_REG(si), 48*4) + SWAP_COPY(0) SWAP_COPY(1) SWAP_COPY(2) SWAP_COPY(3) + SWAP_COPY(4) SWAP_COPY(5) SWAP_COPY(6) SWAP_COPY(7) +#if CRYPTOPP_BOOL_X86 + SWAP_COPY(8) SWAP_COPY(9) SWAP_COPY(10) SWAP_COPY(11) + SWAP_COPY(12) SWAP_COPY(13) SWAP_COPY(14) SWAP_COPY(15) +#endif + AS2( mov edi, E(0)) // E + AS2( mov eax, B(0)) // B + AS2( xor eax, C(0)) // B^C + AS2( mov ecx, A(0)) // A + + ROUND(0, 0, eax, ecx, edi, edx) + ROUND(1, 0, ecx, eax, edx, edi) + ROUND(2, 0, eax, ecx, edi, edx) + ROUND(3, 0, ecx, eax, edx, edi) + ROUND(4, 0, eax, ecx, edi, edx) + ROUND(5, 0, ecx, eax, edx, edi) + ROUND(6, 0, eax, ecx, edi, edx) + ROUND(7, 0, ecx, eax, edx, edi) + ROUND(8, 0, eax, ecx, edi, edx) + ROUND(9, 0, ecx, eax, edx, edi) + ROUND(10, 0, eax, ecx, edi, edx) + ROUND(11, 0, ecx, eax, edx, edi) + ROUND(12, 0, eax, ecx, edi, edx) + ROUND(13, 0, ecx, eax, edx, edi) + ROUND(14, 0, eax, ecx, edi, edx) + ROUND(15, 0, ecx, eax, edx, edi) + + ASL(1) + AS2(add WORD_REG(si), 4*16) + ROUND(0, 1, eax, ecx, edi, edx) + ROUND(1, 1, ecx, eax, edx, edi) + ROUND(2, 1, eax, ecx, edi, edx) + ROUND(3, 1, ecx, eax, edx, edi) + ROUND(4, 1, eax, ecx, edi, edx) + ROUND(5, 1, ecx, eax, edx, edi) + ROUND(6, 1, eax, ecx, edi, edx) + ROUND(7, 1, ecx, eax, edx, edi) + ROUND(8, 1, eax, ecx, edi, edx) + ROUND(9, 1, ecx, eax, edx, edi) + ROUND(10, 1, eax, ecx, edi, edx) + ROUND(11, 1, ecx, eax, edx, edi) + ROUND(12, 1, eax, ecx, edi, edx) + ROUND(13, 1, ecx, eax, edx, edi) + ROUND(14, 1, eax, ecx, edi, edx) + ROUND(15, 1, ecx, eax, edx, edi) + AS2( cmp WORD_REG(si), K_END) + ASJ( jne, 1, b) + + AS2( mov WORD_REG(dx), DATA_SAVE) + AS2( add WORD_REG(dx), 64) + AS2( mov AS_REG_7, STATE_SAVE) + AS2( mov DATA_SAVE, WORD_REG(dx)) + +#if CRYPTOPP_BOOL_X86 + AS2( test DWORD PTR DATA_END, 1) + ASJ( jnz, 4, f) +#endif + + AS2( movdqa xmm1, XMMWORD_PTR [AS_REG_7+1*16]) + AS2( movdqa xmm0, XMMWORD_PTR [AS_REG_7+0*16]) + AS2( paddd xmm1, E(0)) + AS2( paddd xmm0, A(0)) + AS2( movdqa [AS_REG_7+1*16], xmm1) + AS2( movdqa [AS_REG_7+0*16], xmm0) + AS2( cmp WORD_REG(dx), DATA_END) + ASJ( jl, 0, b) + +#if CRYPTOPP_BOOL_X86 + ASJ( jmp, 5, f) + ASL(4) // non-SSE2 + AS2( add [AS_REG_7+0*4], ecx) // A + AS2( add [AS_REG_7+4*4], edi) // E + AS2( mov eax, B(0)) + AS2( mov ebx, C(0)) + AS2( mov ecx, D(0)) + AS2( add [AS_REG_7+1*4], eax) + AS2( add [AS_REG_7+2*4], ebx) + AS2( add [AS_REG_7+3*4], ecx) + AS2( mov eax, F(0)) + AS2( mov ebx, G(0)) + AS2( mov ecx, H(0)) + AS2( add [AS_REG_7+5*4], eax) + AS2( add [AS_REG_7+6*4], ebx) + AS2( add [AS_REG_7+7*4], ecx) + AS2( mov ecx, AS_REG_7d) + AS2( cmp WORD_REG(dx), DATA_END) + ASJ( jl, 2, b) + ASL(5) +#endif + + AS_POP_IF86(sp) + AS_POP_IF86(bp) + #if !defined(_MSC_VER) || (_MSC_VER < 1300) + AS_POP_IF86(bx) + #endif + +#ifdef CRYPTOPP_GENERATE_X64_MASM + add rsp, LOCALS_SIZE+8 + pop rbp + pop rbx + pop rdi + pop rsi + ret + X86_SHA256_HashBlocks ENDP +#endif + +#ifdef __GNUC__ + ".att_syntax prefix;" + : + : "c" (state), "d" (data), "S" (SHA256_K+48), "D" (len) + #if CRYPTOPP_BOOL_X64 + , "r" (workspace) + #endif + : "memory", "cc", "%eax" + #if CRYPTOPP_BOOL_X64 + , "%rbx", "%r8" + #endif + ); +#endif +} + +#endif // #if defined(CRYPTOPP_X86_ASM_AVAILABLE) || defined(CRYPTOPP_GENERATE_X64_MASM) + +#ifndef CRYPTOPP_GENERATE_X64_MASM + +#ifdef CRYPTOPP_X64_MASM_AVAILABLE +extern "C" { +void CRYPTOPP_FASTCALL X86_SHA256_HashBlocks(word32 *state, const word32 *data, size_t len); +} +#endif + +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) + +size_t SHA256::HashMultipleBlocks(const word32 *input, size_t length) +{ + X86_SHA256_HashBlocks(m_state, input, (length&(size_t(0)-BLOCKSIZE)) - !HasSSE2()); + return length % BLOCKSIZE; +} + +size_t SHA224::HashMultipleBlocks(const word32 *input, size_t length) +{ + X86_SHA256_HashBlocks(m_state, input, (length&(size_t(0)-BLOCKSIZE)) - !HasSSE2()); + return length % BLOCKSIZE; +} + +#endif + #define blk2(i) (W[i&15]+=s1(W[(i-2)&15])+W[(i-7)&15]+s0(W[(i-15)&15])) #define Ch(x,y,z) (z^(x&(y^z))) -#define Maj(x,y,z) ((x&y)|(z&(x|y))) +#define Maj(x,y,z) (y^((x^y)&(y^z))) #define a(i) T[(0-i)&7] #define b(i) T[(1-i)&7] @@ -138,6 +468,11 @@ static const word32 SHA256_K[64] = { void SHA256::Transform(word32 *state, const word32 *data) { word32 W[16]; +#if defined(CRYPTOPP_X86_ASM_AVAILABLE) || defined(CRYPTOPP_X64_MASM_AVAILABLE) + // this byte reverse is a waste of time, but this function is only called by MDC + ByteReverse(W, data, BLOCKSIZE); + X86_SHA256_HashBlocks(state, W, BLOCKSIZE - !HasSSE2()); +#else word32 T[8]; /* Copy context->state[] to working vars */ memcpy(T, state, sizeof(T)); @@ -158,11 +493,12 @@ void SHA256::Transform(word32 *state, const word32 *data) state[5] += f(0); state[6] += g(0); state[7] += h(0); +#endif } /* // smaller but slower -void SHA256_Transform(word32 *state, const word32 *data) +void SHA256::Transform(word32 *state, const word32 *data) { word32 T[20]; word32 W[32]; @@ -176,7 +512,7 @@ void SHA256_Transform(word32 *state, const word32 *data) { word32 w = data[j]; W[j] = w; - w += K[j]; + w += SHA256_K[j]; w += t[7]; w += S1(e); w += Ch(e, t[5], t[6]); @@ -196,7 +532,7 @@ void SHA256_Transform(word32 *state, const word32 *data) i = j&0xf; word32 w = s1(W[i+16-2]) + s0(W[i+16-15]) + W[i] + W[i+16-7]; W[i+16] = W[i] = w; - w += K[j]; + w += SHA256_K[j]; w += t[7]; w += S1(e); w += Ch(e, t[5], t[6]); @@ -208,7 +544,7 @@ void SHA256_Transform(word32 *state, const word32 *data) w = s1(W[(i+1)+16-2]) + s0(W[(i+1)+16-15]) + W[(i+1)] + W[(i+1)+16-7]; W[(i+1)+16] = W[(i+1)] = w; - w += K[j+1]; + w += SHA256_K[j+1]; w += (t-1)[7]; w += S1(e); w += Ch(e, (t-1)[5], (t-1)[6]); @@ -335,22 +671,16 @@ CRYPTOPP_NAKED static void CRYPTOPP_FASTCALL SHA512_SSE2_Transform(word64 *state AS2( lea edi, [esp+4+8*8]) // start at middle of state buffer. will decrement pointer each round to avoid copying AS2( lea esi, [esp+4+20*8+8]) // 16-byte alignment, then add 8 - AS2( movq mm4, [ecx+0*8]) - AS2( movq [edi+0*8], mm4) - AS2( movq mm0, [ecx+1*8]) - AS2( movq [edi+1*8], mm0) - AS2( movq mm0, [ecx+2*8]) - AS2( movq [edi+2*8], mm0) - AS2( movq mm0, [ecx+3*8]) - AS2( movq [edi+3*8], mm0) - AS2( movq mm5, [ecx+4*8]) - AS2( movq [edi+4*8], mm5) - AS2( movq mm0, [ecx+5*8]) - AS2( movq [edi+5*8], mm0) - AS2( movq mm0, [ecx+6*8]) - AS2( movq [edi+6*8], mm0) - AS2( movq mm0, [ecx+7*8]) - AS2( movq [edi+7*8], mm0) + AS2( movdqa xmm0, [ecx+0*16]) + AS2( movdq2q mm4, xmm0) + AS2( movdqa [edi+0*16], xmm0) + AS2( movdqa xmm0, [ecx+1*16]) + AS2( movdqa [edi+1*16], xmm0) + AS2( movdqa xmm0, [ecx+2*16]) + AS2( movdq2q mm5, xmm0) + AS2( movdqa [edi+2*16], xmm0) + AS2( movdqa xmm0, [ecx+3*16]) + AS2( movdqa [edi+3*16], xmm0) ASJ( jmp, 0, f) #define SSE2_S0_S1(r, a, b, c) \ @@ -475,18 +805,14 @@ CRYPTOPP_NAKED static void CRYPTOPP_FASTCALL SHA512_SSE2_Transform(word64 *state ASJ( jne, 1, b) #define SSE2_CombineState(i) \ - AS2( movq mm0, [edi+i*8])\ - AS2( paddq mm0, [ecx+i*8])\ - AS2( movq [ecx+i*8], mm0) + AS2( movdqa xmm0, [edi+i*16])\ + AS2( paddq xmm0, [ecx+i*16])\ + AS2( movdqa [ecx+i*16], xmm0) SSE2_CombineState(0) SSE2_CombineState(1) SSE2_CombineState(2) SSE2_CombineState(3) - SSE2_CombineState(4) - SSE2_CombineState(5) - SSE2_CombineState(6) - SSE2_CombineState(7) AS1( pop esp) AS1( emms) @@ -550,4 +876,5 @@ void SHA512::Transform(word64 *state, const word64 *data) NAMESPACE_END +#endif // #ifndef CRYPTOPP_GENERATE_X64_MASM #endif // #ifndef CRYPTOPP_IMPORTS -- cgit v1.2.1