Bug#14211271 ISSUES WITH SSL ON DEBIAN WHEEZY I386 AND KFREEBSD-I386

Problem:
It was reported that on Debian and KFreeBSD platforms, i386 architecture 
machines certain SSL tests are failing. main.ssl_connect  rpl.rpl_heartbeat_ssl
rpl.rpl_ssl1 rpl.rpl_ssl main.ssl_cipher, main.func_encrypt were the tests that
 were reportedly failing (crashing). The reason for the crashes are said to be
due to the assembly code of yaSSL.

Solution:
There was initially a workaround suggested i.e., to enable 
-DTAOCRYPT_DISABLE_X86ASM flag which would prevent the crash, but at an expense
 of 4X reduction of speed. Since this was unacceptable, the fix was the 
functions using assembly, now input variables from the function call using 
extended inline assembly on GCC instead of relying on direct assembly code.

1 files changed, 54 insertions, 28 deletions

diff --git a/extra/yassl/taocrypt/src/aes.cpp b/extra/yassl/taocrypt/src/aes.cpp
index 9b04e7b0dbc..d04ef69c19f 100644
--- a/extra/yassl/taocrypt/src/aes.cpp
+++ b/extra/yassl/taocrypt/src/aes.cpp
@@ -52,7 +52,7 @@ void AES::Process(byte* out, const byte* in, word32 sz)
             in  += BLOCK_SIZE;
         }
     else if (mode_ == CBC) {
-        if (dir_ == ENCRYPTION)
+        if (dir_ == ENCRYPTION) {
             while (blocks--) {
                 r_[0] ^= *(word32*)in;
                 r_[1] ^= *(word32*)(in +  4);
@@ -65,7 +65,8 @@ void AES::Process(byte* out, const byte* in, word32 sz)
                 out += BLOCK_SIZE;
                 in  += BLOCK_SIZE;
             }
-        else
+        }
+    else {
             while (blocks--) {
                 AsmDecrypt(in, out, (void*)Td0);
                 
@@ -78,6 +79,7 @@ void AES::Process(byte* out, const byte* in, word32 sz)
                 out += BLOCK_SIZE;
                 in  += BLOCK_SIZE;
             }
+       }
    }
 }
 
@@ -452,27 +454,31 @@ void AES::decrypt(const byte* inBlock, const byte* xorBlock,
 
 #if defined(DO_AES_ASM)
     #ifdef __GNUC__
-        #define AS1(x)    asm(#x);
-        #define AS2(x, y) asm(#x ", " #y);
+        #define AS1(x)    #x ";"
+        #define AS2(x, y) #x ", " #y ";"
 
         #define PROLOG()  \
-            asm(".intel_syntax noprefix"); \
-            AS2(    movd  mm3, edi                      )   \
-            AS2(    movd  mm4, ebx                      )   \
-            AS2(    sub   esp, 4                        )   \
-            AS2(    movd  mm7, ebp                      )   \
-            AS2(    mov   [ebp - 4], esi                )   \
-            AS2(    mov   ecx, DWORD PTR [ebp +  8]     )   \
-            AS2(    mov   esi, DWORD PTR [ebp + 12]     )   \
-            AS2(    mov   ebp, DWORD PTR [ebp + 20]     )
+        __asm__ __volatile__ \
+        ( \
+            ".intel_syntax noprefix;" \
+            "push ebx;" \
+            "push ebp;" \
+            "movd mm7, ebp;" \
+            "movd mm4, eax;" \
+            "mov  ebp, edx;"  \
+            "sub  esp, 4;"
 
         #define EPILOG()  \
-            AS2(    mov  esi, [ebp - 4]             )   \
-            AS2(    mov  esp, ebp                   )   \
-            AS2(    movd ebx, mm4                   )   \
-            AS2(    movd edi, mm3                   )   \
-            AS1(    emms                            )   \
-            asm(".att_syntax");
+            "add esp, 4;" \
+            "pop ebp;" \
+            "pop ebx;" \
+                   "emms;" \
+                   ".att_syntax;" \
+                : \
+                : "c" (this), "S" (inBlock), "d" (boxes), "a" (outBlock) \
+                : "%edi", "memory", "cc" \
+        );
+
     #else
         #define AS1(x)    __asm x
         #define AS2(x, y) __asm x, y
@@ -504,6 +510,8 @@ void AES::decrypt(const byte* inBlock, const byte* xorBlock,
 
 #ifdef _MSC_VER
     __declspec(naked) 
+#else
+    __attribute__ ((noinline))
 #endif
 void AES::AsmEncrypt(const byte* inBlock, byte* outBlock, void* boxes) const
 {
@@ -537,7 +545,11 @@ void AES::AsmEncrypt(const byte* inBlock, byte* outBlock, void* boxes) const
     AS2(    xor   ecx, DWORD PTR [edi +  8]          )   // s2
     AS2(    xor   edx, DWORD PTR [edi + 12]          )   // s3
 
-    AS1(loop1:                                                          )
+#ifdef _MSC_VER
+    AS1( loop1: )  // loop1
+#else
+    AS1(1:  )      // loop1
+#endif
             /* Put0 (mm0) =  
                 Te0[get0,rs 24] ^
                 Te1[get1,rs 16] ^
@@ -652,7 +664,11 @@ void AES::AsmEncrypt(const byte* inBlock, byte* outBlock, void* boxes) const
     AS1(    dec   edi                                                   )
     AS2(    movd  mm5, edi                                              )
 
-    AS1(    jnz   loop1                                                 )
+#ifdef _MSC_VER
+    AS1(    jnz   loop1)  // loop1
+#else
+    AS1(    jnz   1b )    // loop1
+#endif
 
             // last round
             /*
@@ -799,9 +815,9 @@ void AES::AsmEncrypt(const byte* inBlock, byte* outBlock, void* boxes) const
 
             // store
     #ifdef __GNUC__
-        AS2(    mov esi, DWORD PTR [ebp + 16]       )   //  outBlock
+        AS2(    movd esi, mm4                       )   //  outBlock
     #else
-        AS2(    mov esi, DWORD PTR [ebp + 12]       )   //  outBlock
+        AS2(    mov  esi, DWORD PTR [ebp + 12]      )   //  outBlock
     #endif
 
     AS1(    bswap ecx                                                   )
@@ -818,7 +834,9 @@ void AES::AsmEncrypt(const byte* inBlock, byte* outBlock, void* boxes) const
 
 
 #ifdef _MSC_VER
-    __declspec(naked) 
+    __declspec(naked)
+#else
+    __attribute__ ((noinline)) 
 #endif
 void AES::AsmDecrypt(const byte* inBlock, byte* outBlock, void* boxes) const
 {
@@ -853,7 +871,11 @@ void AES::AsmDecrypt(const byte* inBlock, byte* outBlock, void* boxes) const
     AS2(    xor   edx, DWORD PTR [edi + 12]          )   // s3
 
 
-    AS1(loop2:                                                          )
+#ifdef _MSC_VER
+    AS1( loop2: )  // loop2
+#else
+    AS1(2:  )      // loop2
+#endif
        /*   Put0 (mm0) =
             Td0[GETBYTE(get0, rs24)] ^
             Td1[GETBYTE(get3, rs16)] ^
@@ -964,7 +986,11 @@ void AES::AsmDecrypt(const byte* inBlock, byte* outBlock, void* boxes) const
     AS1(    dec   edi                                                   )
     AS2(    movd  mm5, edi                                              )
 
-    AS1(    jnz   loop2                                                 )
+#ifdef _MSC_VER
+    AS1(    jnz   loop2)  // loop2
+#else
+    AS1(    jnz   2b )    // loop2
+#endif
 
             // last round
             /*
@@ -1114,9 +1140,9 @@ void AES::AsmDecrypt(const byte* inBlock, byte* outBlock, void* boxes) const
 
             // store
     #ifdef __GNUC__
-        AS2(    mov esi, DWORD PTR [ebp + 16]       )   //  outBlock
+        AS2(    movd esi, mm4                        )   //  outBlock
     #else
-        AS2(    mov esi, DWORD PTR [ebp + 12]       )   //  outBlock
+        AS2(    mov esi,  DWORD PTR [ebp + 12]       )   //  outBlock
     #endif
     AS2(    mov DWORD PTR [esi],      eax                               )
     AS2(    mov DWORD PTR [esi +  4], ebx                               )