Updated asm sources

1 files changed, 466 insertions, 415 deletions

diff --git a/lib/accelerated/x86/coff/aesni-x86.s b/lib/accelerated/x86/coff/aesni-x86.s
index 502be77883..1c9b935f34 100644
--- a/lib/accelerated/x86/coff/aesni-x86.s
+++ b/lib/accelerated/x86/coff/aesni-x86.s
@@ -85,29 +85,82 @@ _aesni_decrypt:
 .byte	102,15,56,223,209
 	movups	%xmm2,(%eax)
 	ret
+.def	__aesni_encrypt2;	.scl	3;	.type	32;	.endef
+.align	16
+__aesni_encrypt2:
+	movups	(%edx),%xmm0
+	shll	$4,%ecx
+	movups	16(%edx),%xmm1
+	xorps	%xmm0,%xmm2
+	pxor	%xmm0,%xmm3
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+	addl	$16,%ecx
+.L002enc2_loop:
+.byte	102,15,56,220,209
+.byte	102,15,56,220,217
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
+.byte	102,15,56,220,208
+.byte	102,15,56,220,216
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L002enc2_loop
+.byte	102,15,56,220,209
+.byte	102,15,56,220,217
+.byte	102,15,56,221,208
+.byte	102,15,56,221,216
+	ret
+.def	__aesni_decrypt2;	.scl	3;	.type	32;	.endef
+.align	16
+__aesni_decrypt2:
+	movups	(%edx),%xmm0
+	shll	$4,%ecx
+	movups	16(%edx),%xmm1
+	xorps	%xmm0,%xmm2
+	pxor	%xmm0,%xmm3
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+	addl	$16,%ecx
+.L003dec2_loop:
+.byte	102,15,56,222,209
+.byte	102,15,56,222,217
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
+.byte	102,15,56,222,208
+.byte	102,15,56,222,216
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L003dec2_loop
+.byte	102,15,56,222,209
+.byte	102,15,56,222,217
+.byte	102,15,56,223,208
+.byte	102,15,56,223,216
+	ret
 .def	__aesni_encrypt3;	.scl	3;	.type	32;	.endef
 .align	16
 __aesni_encrypt3:
 	movups	(%edx),%xmm0
-	shrl	$1,%ecx
+	shll	$4,%ecx
 	movups	16(%edx),%xmm1
-	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
 	pxor	%xmm0,%xmm4
-	movups	(%edx),%xmm0
-.L002enc3_loop:
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+	addl	$16,%ecx
+.L004enc3_loop:
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
-	decl	%ecx
 .byte	102,15,56,220,225
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,220,208
 .byte	102,15,56,220,216
-	leal	32(%edx),%edx
 .byte	102,15,56,220,224
-	movups	(%edx),%xmm0
-	jnz	.L002enc3_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L004enc3_loop
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
 .byte	102,15,56,220,225
@@ -119,25 +172,26 @@ __aesni_encrypt3:
 .align	16
 __aesni_decrypt3:
 	movups	(%edx),%xmm0
-	shrl	$1,%ecx
+	shll	$4,%ecx
 	movups	16(%edx),%xmm1
-	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
 	pxor	%xmm0,%xmm4
-	movups	(%edx),%xmm0
-.L003dec3_loop:
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+	addl	$16,%ecx
+.L005dec3_loop:
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
-	decl	%ecx
 .byte	102,15,56,222,225
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,222,208
 .byte	102,15,56,222,216
-	leal	32(%edx),%edx
 .byte	102,15,56,222,224
-	movups	(%edx),%xmm0
-	jnz	.L003dec3_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L005dec3_loop
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
 .byte	102,15,56,222,225
@@ -150,27 +204,29 @@ __aesni_decrypt3:
 __aesni_encrypt4:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
-	shrl	$1,%ecx
-	leal	32(%edx),%edx
+	shll	$4,%ecx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
 	pxor	%xmm0,%xmm4
 	pxor	%xmm0,%xmm5
-	movups	(%edx),%xmm0
-.L004enc4_loop:
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+.byte	15,31,64,0
+	addl	$16,%ecx
+.L006enc4_loop:
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
-	decl	%ecx
 .byte	102,15,56,220,225
 .byte	102,15,56,220,233
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,220,208
 .byte	102,15,56,220,216
-	leal	32(%edx),%edx
 .byte	102,15,56,220,224
 .byte	102,15,56,220,232
-	movups	(%edx),%xmm0
-	jnz	.L004enc4_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L006enc4_loop
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
 .byte	102,15,56,220,225
@@ -185,27 +241,29 @@ __aesni_encrypt4:
 __aesni_decrypt4:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
-	shrl	$1,%ecx
-	leal	32(%edx),%edx
+	shll	$4,%ecx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
 	pxor	%xmm0,%xmm4
 	pxor	%xmm0,%xmm5
-	movups	(%edx),%xmm0
-.L005dec4_loop:
+	movups	32(%edx),%xmm0
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+.byte	15,31,64,0
+	addl	$16,%ecx
+.L007dec4_loop:
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
-	decl	%ecx
 .byte	102,15,56,222,225
 .byte	102,15,56,222,233
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,222,208
 .byte	102,15,56,222,216
-	leal	32(%edx),%edx
 .byte	102,15,56,222,224
 .byte	102,15,56,222,232
-	movups	(%edx),%xmm0
-	jnz	.L005dec4_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L007dec4_loop
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
 .byte	102,15,56,222,225
@@ -219,45 +277,44 @@ __aesni_decrypt4:
 .align	16
 __aesni_encrypt6:
 	movups	(%edx),%xmm0
-	shrl	$1,%ecx
+	shll	$4,%ecx
 	movups	16(%edx),%xmm1
-	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
-.byte	102,15,56,220,209
 	pxor	%xmm0,%xmm4
-.byte	102,15,56,220,217
+.byte	102,15,56,220,209
 	pxor	%xmm0,%xmm5
-	decl	%ecx
-.byte	102,15,56,220,225
 	pxor	%xmm0,%xmm6
-.byte	102,15,56,220,233
+.byte	102,15,56,220,217
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+.byte	102,15,56,220,225
 	pxor	%xmm0,%xmm7
+	addl	$16,%ecx
+.byte	102,15,56,220,233
 .byte	102,15,56,220,241
-	movups	(%edx),%xmm0
 .byte	102,15,56,220,249
+	movups	-16(%edx,%ecx,1),%xmm0
 	jmp	.L_aesni_encrypt6_enter
 .align	16
-.L006enc6_loop:
+.L008enc6_loop:
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
-	decl	%ecx
 .byte	102,15,56,220,225
 .byte	102,15,56,220,233
 .byte	102,15,56,220,241
 .byte	102,15,56,220,249
-.align	16
 .L_aesni_encrypt6_enter:
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,220,208
 .byte	102,15,56,220,216
-	leal	32(%edx),%edx
 .byte	102,15,56,220,224
 .byte	102,15,56,220,232
 .byte	102,15,56,220,240
 .byte	102,15,56,220,248
-	movups	(%edx),%xmm0
-	jnz	.L006enc6_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L008enc6_loop
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
 .byte	102,15,56,220,225
@@ -275,45 +332,44 @@ __aesni_encrypt6:
 .align	16
 __aesni_decrypt6:
 	movups	(%edx),%xmm0
-	shrl	$1,%ecx
+	shll	$4,%ecx
 	movups	16(%edx),%xmm1
-	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
 	pxor	%xmm0,%xmm3
-.byte	102,15,56,222,209
 	pxor	%xmm0,%xmm4
-.byte	102,15,56,222,217
+.byte	102,15,56,222,209
 	pxor	%xmm0,%xmm5
-	decl	%ecx
-.byte	102,15,56,222,225
 	pxor	%xmm0,%xmm6
-.byte	102,15,56,222,233
+.byte	102,15,56,222,217
+	leal	32(%edx,%ecx,1),%edx
+	negl	%ecx
+.byte	102,15,56,222,225
 	pxor	%xmm0,%xmm7
+	addl	$16,%ecx
+.byte	102,15,56,222,233
 .byte	102,15,56,222,241
-	movups	(%edx),%xmm0
 .byte	102,15,56,222,249
+	movups	-16(%edx,%ecx,1),%xmm0
 	jmp	.L_aesni_decrypt6_enter
 .align	16
-.L007dec6_loop:
+.L009dec6_loop:
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
-	decl	%ecx
 .byte	102,15,56,222,225
 .byte	102,15,56,222,233
 .byte	102,15,56,222,241
 .byte	102,15,56,222,249
-.align	16
 .L_aesni_decrypt6_enter:
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,222,208
 .byte	102,15,56,222,216
-	leal	32(%edx),%edx
 .byte	102,15,56,222,224
 .byte	102,15,56,222,232
 .byte	102,15,56,222,240
 .byte	102,15,56,222,248
-	movups	(%edx),%xmm0
-	jnz	.L007dec6_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L009dec6_loop
 .byte	102,15,56,222,209
 .byte	102,15,56,222,217
 .byte	102,15,56,222,225
@@ -342,14 +398,14 @@ _aesni_ecb_encrypt:
 	movl	32(%esp),%edx
 	movl	36(%esp),%ebx
 	andl	$-16,%eax
-	jz	.L008ecb_ret
+	jz	.L010ecb_ret
 	movl	240(%edx),%ecx
 	testl	%ebx,%ebx
-	jz	.L009ecb_decrypt
+	jz	.L011ecb_decrypt
 	movl	%edx,%ebp
 	movl	%ecx,%ebx
 	cmpl	$96,%eax
-	jb	.L010ecb_enc_tail
+	jb	.L012ecb_enc_tail
 	movdqu	(%esi),%xmm2
 	movdqu	16(%esi),%xmm3
 	movdqu	32(%esi),%xmm4
@@ -358,9 +414,9 @@ _aesni_ecb_encrypt:
 	movdqu	80(%esi),%xmm7
 	leal	96(%esi),%esi
 	subl	$96,%eax
-	jmp	.L011ecb_enc_loop6_enter
+	jmp	.L013ecb_enc_loop6_enter
 .align	16
-.L012ecb_enc_loop6:
+.L014ecb_enc_loop6:
 	movups	%xmm2,(%edi)
 	movdqu	(%esi),%xmm2
 	movups	%xmm3,16(%edi)
@@ -375,12 +431,12 @@ _aesni_ecb_encrypt:
 	leal	96(%edi),%edi
 	movdqu	80(%esi),%xmm7
 	leal	96(%esi),%esi
-.L011ecb_enc_loop6_enter:
+.L013ecb_enc_loop6_enter:
 	call	__aesni_encrypt6
 	movl	%ebp,%edx
 	movl	%ebx,%ecx
 	subl	$96,%eax
-	jnc	.L012ecb_enc_loop6
+	jnc	.L014ecb_enc_loop6
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
@@ -389,18 +445,18 @@ _aesni_ecb_encrypt:
 	movups	%xmm7,80(%edi)
 	leal	96(%edi),%edi
 	addl	$96,%eax
-	jz	.L008ecb_ret
-.L010ecb_enc_tail:
+	jz	.L010ecb_ret
+.L012ecb_enc_tail:
 	movups	(%esi),%xmm2
 	cmpl	$32,%eax
-	jb	.L013ecb_enc_one
+	jb	.L015ecb_enc_one
 	movups	16(%esi),%xmm3
-	je	.L014ecb_enc_two
+	je	.L016ecb_enc_two
 	movups	32(%esi),%xmm4
 	cmpl	$64,%eax
-	jb	.L015ecb_enc_three
+	jb	.L017ecb_enc_three
 	movups	48(%esi),%xmm5
-	je	.L016ecb_enc_four
+	je	.L018ecb_enc_four
 	movups	64(%esi),%xmm6
 	xorps	%xmm7,%xmm7
 	call	__aesni_encrypt6
@@ -409,50 +465,49 @@ _aesni_ecb_encrypt:
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
 	movups	%xmm6,64(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L013ecb_enc_one:
+.L015ecb_enc_one:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L017enc1_loop_3:
+.L019enc1_loop_3:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L017enc1_loop_3
+	jnz	.L019enc1_loop_3
 .byte	102,15,56,221,209
 	movups	%xmm2,(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L014ecb_enc_two:
-	xorps	%xmm4,%xmm4
-	call	__aesni_encrypt3
+.L016ecb_enc_two:
+	call	__aesni_encrypt2
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L015ecb_enc_three:
+.L017ecb_enc_three:
 	call	__aesni_encrypt3
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L016ecb_enc_four:
+.L018ecb_enc_four:
 	call	__aesni_encrypt4
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L009ecb_decrypt:
+.L011ecb_decrypt:
 	movl	%edx,%ebp
 	movl	%ecx,%ebx
 	cmpl	$96,%eax
-	jb	.L018ecb_dec_tail
+	jb	.L020ecb_dec_tail
 	movdqu	(%esi),%xmm2
 	movdqu	16(%esi),%xmm3
 	movdqu	32(%esi),%xmm4
@@ -461,9 +516,9 @@ _aesni_ecb_encrypt:
 	movdqu	80(%esi),%xmm7
 	leal	96(%esi),%esi
 	subl	$96,%eax
-	jmp	.L019ecb_dec_loop6_enter
+	jmp	.L021ecb_dec_loop6_enter
 .align	16
-.L020ecb_dec_loop6:
+.L022ecb_dec_loop6:
 	movups	%xmm2,(%edi)
 	movdqu	(%esi),%xmm2
 	movups	%xmm3,16(%edi)
@@ -478,12 +533,12 @@ _aesni_ecb_encrypt:
 	leal	96(%edi),%edi
 	movdqu	80(%esi),%xmm7
 	leal	96(%esi),%esi
-.L019ecb_dec_loop6_enter:
+.L021ecb_dec_loop6_enter:
 	call	__aesni_decrypt6
 	movl	%ebp,%edx
 	movl	%ebx,%ecx
 	subl	$96,%eax
-	jnc	.L020ecb_dec_loop6
+	jnc	.L022ecb_dec_loop6
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
@@ -492,18 +547,18 @@ _aesni_ecb_encrypt:
 	movups	%xmm7,80(%edi)
 	leal	96(%edi),%edi
 	addl	$96,%eax
-	jz	.L008ecb_ret
-.L018ecb_dec_tail:
+	jz	.L010ecb_ret
+.L020ecb_dec_tail:
 	movups	(%esi),%xmm2
 	cmpl	$32,%eax
-	jb	.L021ecb_dec_one
+	jb	.L023ecb_dec_one
 	movups	16(%esi),%xmm3
-	je	.L022ecb_dec_two
+	je	.L024ecb_dec_two
 	movups	32(%esi),%xmm4
 	cmpl	$64,%eax
-	jb	.L023ecb_dec_three
+	jb	.L025ecb_dec_three
 	movups	48(%esi),%xmm5
-	je	.L024ecb_dec_four
+	je	.L026ecb_dec_four
 	movups	64(%esi),%xmm6
 	xorps	%xmm7,%xmm7
 	call	__aesni_decrypt6
@@ -512,44 +567,43 @@ _aesni_ecb_encrypt:
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
 	movups	%xmm6,64(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L021ecb_dec_one:
+.L023ecb_dec_one:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L025dec1_loop_4:
+.L027dec1_loop_4:
 .byte	102,15,56,222,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L025dec1_loop_4
+	jnz	.L027dec1_loop_4
 .byte	102,15,56,223,209
 	movups	%xmm2,(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L022ecb_dec_two:
-	xorps	%xmm4,%xmm4
-	call	__aesni_decrypt3
+.L024ecb_dec_two:
+	call	__aesni_decrypt2
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L023ecb_dec_three:
+.L025ecb_dec_three:
 	call	__aesni_decrypt3
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
-	jmp	.L008ecb_ret
+	jmp	.L010ecb_ret
 .align	16
-.L024ecb_dec_four:
+.L026ecb_dec_four:
 	call	__aesni_decrypt4
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
-.L008ecb_ret:
+.L010ecb_ret:
 	popl	%edi
 	popl	%esi
 	popl	%ebx
@@ -587,45 +641,45 @@ _aesni_ccm64_encrypt_blocks:
 	movl	%ebp,20(%esp)
 	movl	%ebp,24(%esp)
 	movl	%ebp,28(%esp)
-	shrl	$1,%ecx
+	shll	$4,%ecx
+	movl	$16,%ebx
 	leal	(%edx),%ebp
 	movdqa	(%esp),%xmm5
 	movdqa	%xmm7,%xmm2
-	movl	%ecx,%ebx
+	leal	32(%edx,%ecx,1),%edx
+	subl	%ecx,%ebx
 .byte	102,15,56,0,253
-.L026ccm64_enc_outer:
+.L028ccm64_enc_outer:
 	movups	(%ebp),%xmm0
 	movl	%ebx,%ecx
 	movups	(%esi),%xmm6
 	xorps	%xmm0,%xmm2
 	movups	16(%ebp),%xmm1
 	xorps	%xmm6,%xmm0
-	leal	32(%ebp),%edx
 	xorps	%xmm0,%xmm3
-	movups	(%edx),%xmm0
-.L027ccm64_enc2_loop:
+	movups	32(%ebp),%xmm0
+.L029ccm64_enc2_loop:
 .byte	102,15,56,220,209
-	decl	%ecx
 .byte	102,15,56,220,217
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,220,208
-	leal	32(%edx),%edx
 .byte	102,15,56,220,216
-	movups	(%edx),%xmm0
-	jnz	.L027ccm64_enc2_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L029ccm64_enc2_loop
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
 	paddq	16(%esp),%xmm7
+	decl	%eax
 .byte	102,15,56,221,208
 .byte	102,15,56,221,216
-	decl	%eax
 	leal	16(%esi),%esi
 	xorps	%xmm2,%xmm6
 	movdqa	%xmm7,%xmm2
 	movups	%xmm6,(%edi)
-	leal	16(%edi),%edi
 .byte	102,15,56,0,213
-	jnz	.L026ccm64_enc_outer
+	leal	16(%edi),%edi
+	jnz	.L028ccm64_enc_outer
 	movl	48(%esp),%esp
 	movl	40(%esp),%edi
 	movups	%xmm3,(%edi)
@@ -675,67 +729,70 @@ _aesni_ccm64_decrypt_blocks:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L028enc1_loop_5:
+.L030enc1_loop_5:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L028enc1_loop_5
+	jnz	.L030enc1_loop_5
 .byte	102,15,56,221,209
+	shll	$4,%ebx
+	movl	$16,%ecx
 	movups	(%esi),%xmm6
 	paddq	16(%esp),%xmm7
 	leal	16(%esi),%esi
-	jmp	.L029ccm64_dec_outer
+	subl	%ebx,%ecx
+	leal	32(%ebp,%ebx,1),%edx
+	movl	%ecx,%ebx
+	jmp	.L031ccm64_dec_outer
 .align	16
-.L029ccm64_dec_outer:
+.L031ccm64_dec_outer:
 	xorps	%xmm2,%xmm6
 	movdqa	%xmm7,%xmm2
-	movl	%ebx,%ecx
 	movups	%xmm6,(%edi)
 	leal	16(%edi),%edi
 .byte	102,15,56,0,213
 	subl	$1,%eax
-	jz	.L030ccm64_dec_break
+	jz	.L032ccm64_dec_break
 	movups	(%ebp),%xmm0
-	shrl	$1,%ecx
+	movl	%ebx,%ecx
 	movups	16(%ebp),%xmm1
 	xorps	%xmm0,%xmm6
-	leal	32(%ebp),%edx
 	xorps	%xmm0,%xmm2
 	xorps	%xmm6,%xmm3
-	movups	(%edx),%xmm0
-.L031ccm64_dec2_loop:
+	movups	32(%ebp),%xmm0
+.L033ccm64_dec2_loop:
 .byte	102,15,56,220,209
-	decl	%ecx
 .byte	102,15,56,220,217
-	movups	16(%edx),%xmm1
+	movups	(%edx,%ecx,1),%xmm1
+	addl	$32,%ecx
 .byte	102,15,56,220,208
-	leal	32(%edx),%edx
 .byte	102,15,56,220,216
-	movups	(%edx),%xmm0
-	jnz	.L031ccm64_dec2_loop
+	movups	-16(%edx,%ecx,1),%xmm0
+	jnz	.L033ccm64_dec2_loop
 	movups	(%esi),%xmm6
 	paddq	16(%esp),%xmm7
 .byte	102,15,56,220,209
 .byte	102,15,56,220,217
-	leal	16(%esi),%esi
 .byte	102,15,56,221,208
 .byte	102,15,56,221,216
-	jmp	.L029ccm64_dec_outer
+	leal	16(%esi),%esi
+	jmp	.L031ccm64_dec_outer
 .align	16
-.L030ccm64_dec_break:
+.L032ccm64_dec_break:
+	movl	240(%ebp),%ecx
 	movl	%ebp,%edx
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
 	xorps	%xmm0,%xmm6
 	leal	32(%edx),%edx
 	xorps	%xmm6,%xmm3
-.L032enc1_loop_6:
+.L034enc1_loop_6:
 .byte	102,15,56,220,217
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L032enc1_loop_6
+	jnz	.L034enc1_loop_6
 .byte	102,15,56,221,217
 	movl	48(%esp),%esp
 	movl	40(%esp),%edi
@@ -764,7 +821,7 @@ _aesni_ctr32_encrypt_blocks:
 	andl	$-16,%esp
 	movl	%ebp,80(%esp)
 	cmpl	$1,%eax
-	je	.L033ctr32_one_shortcut
+	je	.L035ctr32_one_shortcut
 	movdqu	(%ebx),%xmm7
 	movl	$202182159,(%esp)
 	movl	$134810123,4(%esp)
@@ -780,63 +837,59 @@ _aesni_ctr32_encrypt_blocks:
 .byte	102,15,58,34,253,3
 	movl	240(%edx),%ecx
 	bswap	%ebx
-	pxor	%xmm1,%xmm1
 	pxor	%xmm0,%xmm0
+	pxor	%xmm1,%xmm1
 	movdqa	(%esp),%xmm2
-.byte	102,15,58,34,203,0
+.byte	102,15,58,34,195,0
 	leal	3(%ebx),%ebp
-.byte	102,15,58,34,197,0
+.byte	102,15,58,34,205,0
 	incl	%ebx
-.byte	102,15,58,34,203,1
+.byte	102,15,58,34,195,1
 	incl	%ebp
-.byte	102,15,58,34,197,1
+.byte	102,15,58,34,205,1
 	incl	%ebx
-.byte	102,15,58,34,203,2
+.byte	102,15,58,34,195,2
 	incl	%ebp
-.byte	102,15,58,34,197,2
-	movdqa	%xmm1,48(%esp)
-.byte	102,15,56,0,202
-	movdqa	%xmm0,64(%esp)
+.byte	102,15,58,34,205,2
+	movdqa	%xmm0,48(%esp)
 .byte	102,15,56,0,194
-	pshufd	$192,%xmm1,%xmm2
-	pshufd	$128,%xmm1,%xmm3
+	movdqu	(%edx),%xmm6
+	movdqa	%xmm1,64(%esp)
+.byte	102,15,56,0,202
+	pshufd	$192,%xmm0,%xmm2
+	pshufd	$128,%xmm0,%xmm3
 	cmpl	$6,%eax
-	jb	.L034ctr32_tail
+	jb	.L036ctr32_tail
+	pxor	%xmm6,%xmm7
+	shll	$4,%ecx
+	movl	$16,%ebx
 	movdqa	%xmm7,32(%esp)
-	shrl	$1,%ecx
 	movl	%edx,%ebp
-	movl	%ecx,%ebx
+	subl	%ecx,%ebx
+	leal	32(%edx,%ecx,1),%edx
 	subl	$6,%eax
-	jmp	.L035ctr32_loop6
-.align	16
-.L035ctr32_loop6:
-	pshufd	$64,%xmm1,%xmm4
-	movdqa	32(%esp),%xmm1
-	pshufd	$192,%xmm0,%xmm5
-	por	%xmm1,%xmm2
-	pshufd	$128,%xmm0,%xmm6
-	por	%xmm1,%xmm3
-	pshufd	$64,%xmm0,%xmm7
-	por	%xmm1,%xmm4
-	por	%xmm1,%xmm5
-	por	%xmm1,%xmm6
-	por	%xmm1,%xmm7
-	movups	(%ebp),%xmm0
-	movups	16(%ebp),%xmm1
-	leal	32(%ebp),%edx
-	decl	%ecx
+	jmp	.L037ctr32_loop6
+.align	16
+.L037ctr32_loop6:
+	pshufd	$64,%xmm0,%xmm4
+	movdqa	32(%esp),%xmm0
+	pshufd	$192,%xmm1,%xmm5
 	pxor	%xmm0,%xmm2
+	pshufd	$128,%xmm1,%xmm6
 	pxor	%xmm0,%xmm3
-.byte	102,15,56,220,209
+	pshufd	$64,%xmm1,%xmm7
+	movups	16(%ebp),%xmm1
 	pxor	%xmm0,%xmm4
-.byte	102,15,56,220,217
 	pxor	%xmm0,%xmm5
-.byte	102,15,56,220,225
+.byte	102,15,56,220,209
 	pxor	%xmm0,%xmm6
-.byte	102,15,56,220,233
 	pxor	%xmm0,%xmm7
+.byte	102,15,56,220,217
+	movups	32(%ebp),%xmm0
+	movl	%ebx,%ecx
+.byte	102,15,56,220,225
+.byte	102,15,56,220,233
 .byte	102,15,56,220,241
-	movups	(%edx),%xmm0
 .byte	102,15,56,220,249
 	call	.L_aesni_encrypt6_enter
 	movups	(%esi),%xmm1
@@ -847,51 +900,51 @@ _aesni_ctr32_encrypt_blocks:
 	movups	%xmm2,(%edi)
 	movdqa	16(%esp),%xmm0
 	xorps	%xmm1,%xmm4
-	movdqa	48(%esp),%xmm1
+	movdqa	64(%esp),%xmm1
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
 	paddd	%xmm0,%xmm1
-	paddd	64(%esp),%xmm0
+	paddd	48(%esp),%xmm0
 	movdqa	(%esp),%xmm2
 	movups	48(%esi),%xmm3
 	movups	64(%esi),%xmm4
 	xorps	%xmm3,%xmm5
 	movups	80(%esi),%xmm3
 	leal	96(%esi),%esi
-	movdqa	%xmm1,48(%esp)
-.byte	102,15,56,0,202
+	movdqa	%xmm0,48(%esp)
+.byte	102,15,56,0,194
 	xorps	%xmm4,%xmm6
 	movups	%xmm5,48(%edi)
 	xorps	%xmm3,%xmm7
-	movdqa	%xmm0,64(%esp)
-.byte	102,15,56,0,194
+	movdqa	%xmm1,64(%esp)
+.byte	102,15,56,0,202
 	movups	%xmm6,64(%edi)
-	pshufd	$192,%xmm1,%xmm2
+	pshufd	$192,%xmm0,%xmm2
 	movups	%xmm7,80(%edi)
 	leal	96(%edi),%edi
-	movl	%ebx,%ecx
-	pshufd	$128,%xmm1,%xmm3
+	pshufd	$128,%xmm0,%xmm3
 	subl	$6,%eax
-	jnc	.L035ctr32_loop6
+	jnc	.L037ctr32_loop6
 	addl	$6,%eax
-	jz	.L036ctr32_ret
+	jz	.L038ctr32_ret
+	movdqu	(%ebp),%xmm7
 	movl	%ebp,%edx
-	leal	1(,%ecx,2),%ecx
-	movdqa	32(%esp),%xmm7
-.L034ctr32_tail:
+	pxor	32(%esp),%xmm7
+	movl	240(%ebp),%ecx
+.L036ctr32_tail:
 	por	%xmm7,%xmm2
 	cmpl	$2,%eax
-	jb	.L037ctr32_one
-	pshufd	$64,%xmm1,%xmm4
+	jb	.L039ctr32_one
+	pshufd	$64,%xmm0,%xmm4
 	por	%xmm7,%xmm3
-	je	.L038ctr32_two
-	pshufd	$192,%xmm0,%xmm5
+	je	.L040ctr32_two
+	pshufd	$192,%xmm1,%xmm5
 	por	%xmm7,%xmm4
 	cmpl	$4,%eax
-	jb	.L039ctr32_three
-	pshufd	$128,%xmm0,%xmm6
+	jb	.L041ctr32_three
+	pshufd	$128,%xmm1,%xmm6
 	por	%xmm7,%xmm5
-	je	.L040ctr32_four
+	je	.L042ctr32_four
 	por	%xmm7,%xmm6
 	call	__aesni_encrypt6
 	movups	(%esi),%xmm1
@@ -909,39 +962,39 @@ _aesni_ctr32_encrypt_blocks:
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
 	movups	%xmm6,64(%edi)
-	jmp	.L036ctr32_ret
+	jmp	.L038ctr32_ret
 .align	16
-.L033ctr32_one_shortcut:
+.L035ctr32_one_shortcut:
 	movups	(%ebx),%xmm2
 	movl	240(%edx),%ecx
-.L037ctr32_one:
+.L039ctr32_one:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L041enc1_loop_7:
+.L043enc1_loop_7:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L041enc1_loop_7
+	jnz	.L043enc1_loop_7
 .byte	102,15,56,221,209
 	movups	(%esi),%xmm6
 	xorps	%xmm2,%xmm6
 	movups	%xmm6,(%edi)
-	jmp	.L036ctr32_ret
+	jmp	.L038ctr32_ret
 .align	16
-.L038ctr32_two:
-	call	__aesni_encrypt3
+.L040ctr32_two:
+	call	__aesni_encrypt2
 	movups	(%esi),%xmm5
 	movups	16(%esi),%xmm6
 	xorps	%xmm5,%xmm2
 	xorps	%xmm6,%xmm3
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
-	jmp	.L036ctr32_ret
+	jmp	.L038ctr32_ret
 .align	16
-.L039ctr32_three:
+.L041ctr32_three:
 	call	__aesni_encrypt3
 	movups	(%esi),%xmm5
 	movups	16(%esi),%xmm6
@@ -952,9 +1005,9 @@ _aesni_ctr32_encrypt_blocks:
 	xorps	%xmm7,%xmm4
 	movups	%xmm3,16(%edi)
 	movups	%xmm4,32(%edi)
-	jmp	.L036ctr32_ret
+	jmp	.L038ctr32_ret
 .align	16
-.L040ctr32_four:
+.L042ctr32_four:
 	call	__aesni_encrypt4
 	movups	(%esi),%xmm6
 	movups	16(%esi),%xmm7
@@ -968,7 +1021,7 @@ _aesni_ctr32_encrypt_blocks:
 	xorps	%xmm0,%xmm5
 	movups	%xmm4,32(%edi)
 	movups	%xmm5,48(%edi)
-.L036ctr32_ret:
+.L038ctr32_ret:
 	movl	80(%esp),%esp
 	popl	%edi
 	popl	%esi
@@ -992,12 +1045,12 @@ _aesni_xts_encrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L042enc1_loop_8:
+.L044enc1_loop_8:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L042enc1_loop_8
+	jnz	.L044enc1_loop_8
 .byte	102,15,56,221,209
 	movl	20(%esp),%esi
 	movl	24(%esp),%edi
@@ -1021,12 +1074,14 @@ _aesni_xts_encrypt:
 	movl	%edx,%ebp
 	movl	%ecx,%ebx
 	subl	$96,%eax
-	jc	.L043xts_enc_short
-	shrl	$1,%ecx
-	movl	%ecx,%ebx
-	jmp	.L044xts_enc_loop6
+	jc	.L045xts_enc_short
+	shll	$4,%ecx
+	movl	$16,%ebx
+	subl	%ecx,%ebx
+	leal	32(%edx,%ecx,1),%edx
+	jmp	.L046xts_enc_loop6
 .align	16
-.L044xts_enc_loop6:
+.L046xts_enc_loop6:
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,(%esp)
@@ -1062,6 +1117,7 @@ _aesni_xts_encrypt:
 	pand	%xmm3,%xmm7
 	movups	(%esi),%xmm2
 	pxor	%xmm1,%xmm7
+	movl	%ebx,%ecx
 	movdqu	16(%esi),%xmm3
 	xorps	%xmm0,%xmm2
 	movdqu	32(%esi),%xmm4
@@ -1077,19 +1133,17 @@ _aesni_xts_encrypt:
 	movdqa	%xmm7,80(%esp)
 	pxor	%xmm1,%xmm7
 	movups	16(%ebp),%xmm1
-	leal	32(%ebp),%edx
 	pxor	16(%esp),%xmm3
-.byte	102,15,56,220,209
 	pxor	32(%esp),%xmm4
-.byte	102,15,56,220,217
+.byte	102,15,56,220,209
 	pxor	48(%esp),%xmm5
-	decl	%ecx
-.byte	102,15,56,220,225
 	pxor	64(%esp),%xmm6
-.byte	102,15,56,220,233
+.byte	102,15,56,220,217
 	pxor	%xmm0,%xmm7
+	movups	32(%ebp),%xmm0
+.byte	102,15,56,220,225
+.byte	102,15,56,220,233
 .byte	102,15,56,220,241
-	movups	(%edx),%xmm0
 .byte	102,15,56,220,249
 	call	.L_aesni_encrypt6_enter
 	movdqa	80(%esp),%xmm1
@@ -1114,26 +1168,25 @@ _aesni_xts_encrypt:
 	paddq	%xmm1,%xmm1
 	pand	%xmm3,%xmm2
 	pcmpgtd	%xmm1,%xmm0
-	movl	%ebx,%ecx
 	pxor	%xmm2,%xmm1
 	subl	$96,%eax
-	jnc	.L044xts_enc_loop6
-	leal	1(,%ecx,2),%ecx
+	jnc	.L046xts_enc_loop6
+	movl	240(%ebp),%ecx
 	movl	%ebp,%edx
 	movl	%ecx,%ebx
-.L043xts_enc_short:
+.L045xts_enc_short:
 	addl	$96,%eax
-	jz	.L045xts_enc_done6x
+	jz	.L047xts_enc_done6x
 	movdqa	%xmm1,%xmm5
 	cmpl	$32,%eax
-	jb	.L046xts_enc_one
+	jb	.L048xts_enc_one
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	paddq	%xmm1,%xmm1
 	pand	%xmm3,%xmm2
 	pcmpgtd	%xmm1,%xmm0
 	pxor	%xmm2,%xmm1
-	je	.L047xts_enc_two
+	je	.L049xts_enc_two
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,%xmm6
@@ -1142,7 +1195,7 @@ _aesni_xts_encrypt:
 	pcmpgtd	%xmm1,%xmm0
 	pxor	%xmm2,%xmm1
 	cmpl	$64,%eax
-	jb	.L048xts_enc_three
+	jb	.L050xts_enc_three
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,%xmm7
@@ -1152,7 +1205,7 @@ _aesni_xts_encrypt:
 	pxor	%xmm2,%xmm1
 	movdqa	%xmm5,(%esp)
 	movdqa	%xmm6,16(%esp)
-	je	.L049xts_enc_four
+	je	.L051xts_enc_four
 	movdqa	%xmm7,32(%esp)
 	pshufd	$19,%xmm0,%xmm7
 	movdqa	%xmm1,48(%esp)
@@ -1184,9 +1237,9 @@ _aesni_xts_encrypt:
 	movups	%xmm5,48(%edi)
 	movups	%xmm6,64(%edi)
 	leal	80(%edi),%edi
-	jmp	.L050xts_enc_done
+	jmp	.L052xts_enc_done
 .align	16
-.L046xts_enc_one:
+.L048xts_enc_one:
 	movups	(%esi),%xmm2
 	leal	16(%esi),%esi
 	xorps	%xmm5,%xmm2
@@ -1194,37 +1247,36 @@ _aesni_xts_encrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L051enc1_loop_9:
+.L053enc1_loop_9:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L051enc1_loop_9
+	jnz	.L053enc1_loop_9
 .byte	102,15,56,221,209
 	xorps	%xmm5,%xmm2
 	movups	%xmm2,(%edi)
 	leal	16(%edi),%edi
 	movdqa	%xmm5,%xmm1
-	jmp	.L050xts_enc_done
+	jmp	.L052xts_enc_done
 .align	16
-.L047xts_enc_two:
+.L049xts_enc_two:
 	movaps	%xmm1,%xmm6
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
 	leal	32(%esi),%esi
 	xorps	%xmm5,%xmm2
 	xorps	%xmm6,%xmm3
-	xorps	%xmm4,%xmm4
-	call	__aesni_encrypt3
+	call	__aesni_encrypt2
 	xorps	%xmm5,%xmm2
 	xorps	%xmm6,%xmm3
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	leal	32(%edi),%edi
 	movdqa	%xmm6,%xmm1
-	jmp	.L050xts_enc_done
+	jmp	.L052xts_enc_done
 .align	16
-.L048xts_enc_three:
+.L050xts_enc_three:
 	movaps	%xmm1,%xmm7
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
@@ -1242,9 +1294,9 @@ _aesni_xts_encrypt:
 	movups	%xmm4,32(%edi)
 	leal	48(%edi),%edi
 	movdqa	%xmm7,%xmm1
-	jmp	.L050xts_enc_done
+	jmp	.L052xts_enc_done
 .align	16
-.L049xts_enc_four:
+.L051xts_enc_four:
 	movaps	%xmm1,%xmm6
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
@@ -1266,28 +1318,28 @@ _aesni_xts_encrypt:
 	movups	%xmm5,48(%edi)
 	leal	64(%edi),%edi
 	movdqa	%xmm6,%xmm1
-	jmp	.L050xts_enc_done
+	jmp	.L052xts_enc_done
 .align	16
-.L045xts_enc_done6x:
+.L047xts_enc_done6x:
 	movl	112(%esp),%eax
 	andl	$15,%eax
-	jz	.L052xts_enc_ret
+	jz	.L054xts_enc_ret
 	movdqa	%xmm1,%xmm5
 	movl	%eax,112(%esp)
-	jmp	.L053xts_enc_steal
+	jmp	.L055xts_enc_steal
 .align	16
-.L050xts_enc_done:
+.L052xts_enc_done:
 	movl	112(%esp),%eax
 	pxor	%xmm0,%xmm0
 	andl	$15,%eax
-	jz	.L052xts_enc_ret
+	jz	.L054xts_enc_ret
 	pcmpgtd	%xmm1,%xmm0
 	movl	%eax,112(%esp)
 	pshufd	$19,%xmm0,%xmm5
 	paddq	%xmm1,%xmm1
 	pand	96(%esp),%xmm5
 	pxor	%xmm1,%xmm5
-.L053xts_enc_steal:
+.L055xts_enc_steal:
 	movzbl	(%esi),%ecx
 	movzbl	-16(%edi),%edx
 	leal	1(%esi),%esi
@@ -1295,7 +1347,7 @@ _aesni_xts_encrypt:
 	movb	%dl,(%edi)
 	leal	1(%edi),%edi
 	subl	$1,%eax
-	jnz	.L053xts_enc_steal
+	jnz	.L055xts_enc_steal
 	subl	112(%esp),%edi
 	movl	%ebp,%edx
 	movl	%ebx,%ecx
@@ -1305,16 +1357,16 @@ _aesni_xts_encrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L054enc1_loop_10:
+.L056enc1_loop_10:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L054enc1_loop_10
+	jnz	.L056enc1_loop_10
 .byte	102,15,56,221,209
 	xorps	%xmm5,%xmm2
 	movups	%xmm2,-16(%edi)
-.L052xts_enc_ret:
+.L054xts_enc_ret:
 	movl	116(%esp),%esp
 	popl	%edi
 	popl	%esi
@@ -1338,12 +1390,12 @@ _aesni_xts_decrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L055enc1_loop_11:
+.L057enc1_loop_11:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L055enc1_loop_11
+	jnz	.L057enc1_loop_11
 .byte	102,15,56,221,209
 	movl	20(%esp),%esi
 	movl	24(%esp),%edi
@@ -1372,12 +1424,14 @@ _aesni_xts_decrypt:
 	pcmpgtd	%xmm1,%xmm0
 	andl	$-16,%eax
 	subl	$96,%eax
-	jc	.L056xts_dec_short
-	shrl	$1,%ecx
-	movl	%ecx,%ebx
-	jmp	.L057xts_dec_loop6
+	jc	.L058xts_dec_short
+	shll	$4,%ecx
+	movl	$16,%ebx
+	subl	%ecx,%ebx
+	leal	32(%edx,%ecx,1),%edx
+	jmp	.L059xts_dec_loop6
 .align	16
-.L057xts_dec_loop6:
+.L059xts_dec_loop6:
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,(%esp)
@@ -1413,6 +1467,7 @@ _aesni_xts_decrypt:
 	pand	%xmm3,%xmm7
 	movups	(%esi),%xmm2
 	pxor	%xmm1,%xmm7
+	movl	%ebx,%ecx
 	movdqu	16(%esi),%xmm3
 	xorps	%xmm0,%xmm2
 	movdqu	32(%esi),%xmm4
@@ -1428,19 +1483,17 @@ _aesni_xts_decrypt:
 	movdqa	%xmm7,80(%esp)
 	pxor	%xmm1,%xmm7
 	movups	16(%ebp),%xmm1
-	leal	32(%ebp),%edx
 	pxor	16(%esp),%xmm3
-.byte	102,15,56,222,209
 	pxor	32(%esp),%xmm4
-.byte	102,15,56,222,217
+.byte	102,15,56,222,209
 	pxor	48(%esp),%xmm5
-	decl	%ecx
-.byte	102,15,56,222,225
 	pxor	64(%esp),%xmm6
-.byte	102,15,56,222,233
+.byte	102,15,56,222,217
 	pxor	%xmm0,%xmm7
+	movups	32(%ebp),%xmm0
+.byte	102,15,56,222,225
+.byte	102,15,56,222,233
 .byte	102,15,56,222,241
-	movups	(%edx),%xmm0
 .byte	102,15,56,222,249
 	call	.L_aesni_decrypt6_enter
 	movdqa	80(%esp),%xmm1
@@ -1465,26 +1518,25 @@ _aesni_xts_decrypt:
 	paddq	%xmm1,%xmm1
 	pand	%xmm3,%xmm2
 	pcmpgtd	%xmm1,%xmm0
-	movl	%ebx,%ecx
 	pxor	%xmm2,%xmm1
 	subl	$96,%eax
-	jnc	.L057xts_dec_loop6
-	leal	1(,%ecx,2),%ecx
+	jnc	.L059xts_dec_loop6
+	movl	240(%ebp),%ecx
 	movl	%ebp,%edx
 	movl	%ecx,%ebx
-.L056xts_dec_short:
+.L058xts_dec_short:
 	addl	$96,%eax
-	jz	.L058xts_dec_done6x
+	jz	.L060xts_dec_done6x
 	movdqa	%xmm1,%xmm5
 	cmpl	$32,%eax
-	jb	.L059xts_dec_one
+	jb	.L061xts_dec_one
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	paddq	%xmm1,%xmm1
 	pand	%xmm3,%xmm2
 	pcmpgtd	%xmm1,%xmm0
 	pxor	%xmm2,%xmm1
-	je	.L060xts_dec_two
+	je	.L062xts_dec_two
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,%xmm6
@@ -1493,7 +1545,7 @@ _aesni_xts_decrypt:
 	pcmpgtd	%xmm1,%xmm0
 	pxor	%xmm2,%xmm1
 	cmpl	$64,%eax
-	jb	.L061xts_dec_three
+	jb	.L063xts_dec_three
 	pshufd	$19,%xmm0,%xmm2
 	pxor	%xmm0,%xmm0
 	movdqa	%xmm1,%xmm7
@@ -1503,7 +1555,7 @@ _aesni_xts_decrypt:
 	pxor	%xmm2,%xmm1
 	movdqa	%xmm5,(%esp)
 	movdqa	%xmm6,16(%esp)
-	je	.L062xts_dec_four
+	je	.L064xts_dec_four
 	movdqa	%xmm7,32(%esp)
 	pshufd	$19,%xmm0,%xmm7
 	movdqa	%xmm1,48(%esp)
@@ -1535,9 +1587,9 @@ _aesni_xts_decrypt:
 	movups	%xmm5,48(%edi)
 	movups	%xmm6,64(%edi)
 	leal	80(%edi),%edi
-	jmp	.L063xts_dec_done
+	jmp	.L065xts_dec_done
 .align	16
-.L059xts_dec_one:
+.L061xts_dec_one:
 	movups	(%esi),%xmm2
 	leal	16(%esi),%esi
 	xorps	%xmm5,%xmm2
@@ -1545,36 +1597,36 @@ _aesni_xts_decrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L064dec1_loop_12:
+.L066dec1_loop_12:
 .byte	102,15,56,222,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L064dec1_loop_12
+	jnz	.L066dec1_loop_12
 .byte	102,15,56,223,209
 	xorps	%xmm5,%xmm2
 	movups	%xmm2,(%edi)
 	leal	16(%edi),%edi
 	movdqa	%xmm5,%xmm1
-	jmp	.L063xts_dec_done
+	jmp	.L065xts_dec_done
 .align	16
-.L060xts_dec_two:
+.L062xts_dec_two:
 	movaps	%xmm1,%xmm6
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
 	leal	32(%esi),%esi
 	xorps	%xmm5,%xmm2
 	xorps	%xmm6,%xmm3
-	call	__aesni_decrypt3
+	call	__aesni_decrypt2
 	xorps	%xmm5,%xmm2
 	xorps	%xmm6,%xmm3
 	movups	%xmm2,(%edi)
 	movups	%xmm3,16(%edi)
 	leal	32(%edi),%edi
 	movdqa	%xmm6,%xmm1
-	jmp	.L063xts_dec_done
+	jmp	.L065xts_dec_done
 .align	16
-.L061xts_dec_three:
+.L063xts_dec_three:
 	movaps	%xmm1,%xmm7
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
@@ -1592,9 +1644,9 @@ _aesni_xts_decrypt:
 	movups	%xmm4,32(%edi)
 	leal	48(%edi),%edi
 	movdqa	%xmm7,%xmm1
-	jmp	.L063xts_dec_done
+	jmp	.L065xts_dec_done
 .align	16
-.L062xts_dec_four:
+.L064xts_dec_four:
 	movaps	%xmm1,%xmm6
 	movups	(%esi),%xmm2
 	movups	16(%esi),%xmm3
@@ -1616,20 +1668,20 @@ _aesni_xts_decrypt:
 	movups	%xmm5,48(%edi)
 	leal	64(%edi),%edi
 	movdqa	%xmm6,%xmm1
-	jmp	.L063xts_dec_done
+	jmp	.L065xts_dec_done
 .align	16
-.L058xts_dec_done6x:
+.L060xts_dec_done6x:
 	movl	112(%esp),%eax
 	andl	$15,%eax
-	jz	.L065xts_dec_ret
+	jz	.L067xts_dec_ret
 	movl	%eax,112(%esp)
-	jmp	.L066xts_dec_only_one_more
+	jmp	.L068xts_dec_only_one_more
 .align	16
-.L063xts_dec_done:
+.L065xts_dec_done:
 	movl	112(%esp),%eax
 	pxor	%xmm0,%xmm0
 	andl	$15,%eax
-	jz	.L065xts_dec_ret
+	jz	.L067xts_dec_ret
 	pcmpgtd	%xmm1,%xmm0
 	movl	%eax,112(%esp)
 	pshufd	$19,%xmm0,%xmm2
@@ -1639,7 +1691,7 @@ _aesni_xts_decrypt:
 	pand	%xmm3,%xmm2
 	pcmpgtd	%xmm1,%xmm0
 	pxor	%xmm2,%xmm1
-.L066xts_dec_only_one_more:
+.L068xts_dec_only_one_more:
 	pshufd	$19,%xmm0,%xmm5
 	movdqa	%xmm1,%xmm6
 	paddq	%xmm1,%xmm1
@@ -1653,16 +1705,16 @@ _aesni_xts_decrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L067dec1_loop_13:
+.L069dec1_loop_13:
 .byte	102,15,56,222,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L067dec1_loop_13
+	jnz	.L069dec1_loop_13
 .byte	102,15,56,223,209
 	xorps	%xmm5,%xmm2
 	movups	%xmm2,(%edi)
-.L068xts_dec_steal:
+.L070xts_dec_steal:
 	movzbl	16(%esi),%ecx
 	movzbl	(%edi),%edx
 	leal	1(%esi),%esi
@@ -1670,7 +1722,7 @@ _aesni_xts_decrypt:
 	movb	%dl,16(%edi)
 	leal	1(%edi),%edi
 	subl	$1,%eax
-	jnz	.L068xts_dec_steal
+	jnz	.L070xts_dec_steal
 	subl	112(%esp),%edi
 	movl	%ebp,%edx
 	movl	%ebx,%ecx
@@ -1680,16 +1732,16 @@ _aesni_xts_decrypt:
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L069dec1_loop_14:
+.L071dec1_loop_14:
 .byte	102,15,56,222,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L069dec1_loop_14
+	jnz	.L071dec1_loop_14
 .byte	102,15,56,223,209
 	xorps	%xmm6,%xmm2
 	movups	%xmm2,(%edi)
-.L065xts_dec_ret:
+.L067xts_dec_ret:
 	movl	116(%esp),%esp
 	popl	%edi
 	popl	%esi
@@ -1714,7 +1766,7 @@ _aesni_cbc_encrypt:
 	movl	32(%esp),%edx
 	movl	36(%esp),%ebp
 	testl	%eax,%eax
-	jz	.L070cbc_abort
+	jz	.L072cbc_abort
 	cmpl	$0,40(%esp)
 	xchgl	%esp,%ebx
 	movups	(%ebp),%xmm7
@@ -1722,14 +1774,14 @@ _aesni_cbc_encrypt:
 	movl	%edx,%ebp
 	movl	%ebx,16(%esp)
 	movl	%ecx,%ebx
-	je	.L071cbc_decrypt
+	je	.L073cbc_decrypt
 	movaps	%xmm7,%xmm2
 	cmpl	$16,%eax
-	jb	.L072cbc_enc_tail
+	jb	.L074cbc_enc_tail
 	subl	$16,%eax
-	jmp	.L073cbc_enc_loop
+	jmp	.L075cbc_enc_loop
 .align	16
-.L073cbc_enc_loop:
+.L075cbc_enc_loop:
 	movups	(%esi),%xmm7
 	leal	16(%esi),%esi
 	movups	(%edx),%xmm0
@@ -1737,24 +1789,24 @@ _aesni_cbc_encrypt:
 	xorps	%xmm0,%xmm7
 	leal	32(%edx),%edx
 	xorps	%xmm7,%xmm2
-.L074enc1_loop_15:
+.L076enc1_loop_15:
 .byte	102,15,56,220,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L074enc1_loop_15
+	jnz	.L076enc1_loop_15
 .byte	102,15,56,221,209
 	movl	%ebx,%ecx
 	movl	%ebp,%edx
 	movups	%xmm2,(%edi)
 	leal	16(%edi),%edi
 	subl	$16,%eax
-	jnc	.L073cbc_enc_loop
+	jnc	.L075cbc_enc_loop
 	addl	$16,%eax
-	jnz	.L072cbc_enc_tail
+	jnz	.L074cbc_enc_tail
 	movaps	%xmm2,%xmm7
-	jmp	.L075cbc_ret
-.L072cbc_enc_tail:
+	jmp	.L077cbc_ret
+.L074cbc_enc_tail:
 	movl	%eax,%ecx
 .long	2767451785
 	movl	$16,%ecx
@@ -1765,20 +1817,20 @@ _aesni_cbc_encrypt:
 	movl	%ebx,%ecx
 	movl	%edi,%esi
 	movl	%ebp,%edx
-	jmp	.L073cbc_enc_loop
+	jmp	.L075cbc_enc_loop
 .align	16
-.L071cbc_decrypt:
+.L073cbc_decrypt:
 	cmpl	$80,%eax
-	jbe	.L076cbc_dec_tail
+	jbe	.L078cbc_dec_tail
 	movaps	%xmm7,(%esp)
 	subl	$80,%eax
-	jmp	.L077cbc_dec_loop6_enter
+	jmp	.L079cbc_dec_loop6_enter
 .align	16
-.L078cbc_dec_loop6:
+.L080cbc_dec_loop6:
 	movaps	%xmm0,(%esp)
 	movups	%xmm7,(%edi)
 	leal	16(%edi),%edi
-.L077cbc_dec_loop6_enter:
+.L079cbc_dec_loop6_enter:
 	movdqu	(%esi),%xmm2
 	movdqu	16(%esi),%xmm3
 	movdqu	32(%esi),%xmm4
@@ -1808,28 +1860,28 @@ _aesni_cbc_encrypt:
 	movups	%xmm6,64(%edi)
 	leal	80(%edi),%edi
 	subl	$96,%eax
-	ja	.L078cbc_dec_loop6
+	ja	.L080cbc_dec_loop6
 	movaps	%xmm7,%xmm2
 	movaps	%xmm0,%xmm7
 	addl	$80,%eax
-	jle	.L079cbc_dec_tail_collected
+	jle	.L081cbc_dec_tail_collected
 	movups	%xmm2,(%edi)
 	leal	16(%edi),%edi
-.L076cbc_dec_tail:
+.L078cbc_dec_tail:
 	movups	(%esi),%xmm2
 	movaps	%xmm2,%xmm6
 	cmpl	$16,%eax
-	jbe	.L080cbc_dec_one
+	jbe	.L082cbc_dec_one
 	movups	16(%esi),%xmm3
 	movaps	%xmm3,%xmm5
 	cmpl	$32,%eax
-	jbe	.L081cbc_dec_two
+	jbe	.L083cbc_dec_two
 	movups	32(%esi),%xmm4
 	cmpl	$48,%eax
-	jbe	.L082cbc_dec_three
+	jbe	.L084cbc_dec_three
 	movups	48(%esi),%xmm5
 	cmpl	$64,%eax
-	jbe	.L083cbc_dec_four
+	jbe	.L085cbc_dec_four
 	movups	64(%esi),%xmm6
 	movaps	%xmm7,(%esp)
 	movups	(%esi),%xmm2
@@ -1852,28 +1904,27 @@ _aesni_cbc_encrypt:
 	leal	64(%edi),%edi
 	movaps	%xmm6,%xmm2
 	subl	$80,%eax
-	jmp	.L079cbc_dec_tail_collected
+	jmp	.L081cbc_dec_tail_collected
 .align	16
-.L080cbc_dec_one:
+.L082cbc_dec_one:
 	movups	(%edx),%xmm0
 	movups	16(%edx),%xmm1
 	leal	32(%edx),%edx
 	xorps	%xmm0,%xmm2
-.L084dec1_loop_16:
+.L086dec1_loop_16:
 .byte	102,15,56,222,209
 	decl	%ecx
 	movups	(%edx),%xmm1
 	leal	16(%edx),%edx
-	jnz	.L084dec1_loop_16
+	jnz	.L086dec1_loop_16
 .byte	102,15,56,223,209
 	xorps	%xmm7,%xmm2
 	movaps	%xmm6,%xmm7
 	subl	$16,%eax
-	jmp	.L079cbc_dec_tail_collected
+	jmp	.L081cbc_dec_tail_collected
 .align	16
-.L081cbc_dec_two:
-	xorps	%xmm4,%xmm4
-	call	__aesni_decrypt3
+.L083cbc_dec_two:
+	call	__aesni_decrypt2
 	xorps	%xmm7,%xmm2
 	xorps	%xmm6,%xmm3
 	movups	%xmm2,(%edi)
@@ -1881,9 +1932,9 @@ _aesni_cbc_encrypt:
 	leal	16(%edi),%edi
 	movaps	%xmm5,%xmm7
 	subl	$32,%eax
-	jmp	.L079cbc_dec_tail_collected
+	jmp	.L081cbc_dec_tail_collected
 .align	16
-.L082cbc_dec_three:
+.L084cbc_dec_three:
 	call	__aesni_decrypt3
 	xorps	%xmm7,%xmm2
 	xorps	%xmm6,%xmm3
@@ -1894,9 +1945,9 @@ _aesni_cbc_encrypt:
 	leal	32(%edi),%edi
 	movups	32(%esi),%xmm7
 	subl	$48,%eax
-	jmp	.L079cbc_dec_tail_collected
+	jmp	.L081cbc_dec_tail_collected
 .align	16
-.L083cbc_dec_four:
+.L085cbc_dec_four:
 	call	__aesni_decrypt4
 	movups	16(%esi),%xmm1
 	movups	32(%esi),%xmm0
@@ -1911,23 +1962,23 @@ _aesni_cbc_encrypt:
 	leal	48(%edi),%edi
 	movaps	%xmm5,%xmm2
 	subl	$64,%eax
-.L079cbc_dec_tail_collected:
+.L081cbc_dec_tail_collected:
 	andl	$15,%eax
-	jnz	.L085cbc_dec_tail_partial
+	jnz	.L087cbc_dec_tail_partial
 	movups	%xmm2,(%edi)
-	jmp	.L075cbc_ret
+	jmp	.L077cbc_ret
 .align	16
-.L085cbc_dec_tail_partial:
+.L087cbc_dec_tail_partial:
 	movaps	%xmm2,(%esp)
 	movl	$16,%ecx
 	movl	%esp,%esi
 	subl	%eax,%ecx
 .long	2767451785
-.L075cbc_ret:
+.L077cbc_ret:
 	movl	16(%esp),%esp
 	movl	36(%esp),%ebp
 	movups	%xmm7,(%ebp)
-.L070cbc_abort:
+.L072cbc_abort:
 	popl	%edi
 	popl	%esi
 	popl	%ebx
@@ -1937,51 +1988,51 @@ _aesni_cbc_encrypt:
 .align	16
 __aesni_set_encrypt_key:
 	testl	%eax,%eax
-	jz	.L086bad_pointer
+	jz	.L088bad_pointer
 	testl	%edx,%edx
-	jz	.L086bad_pointer
+	jz	.L088bad_pointer
 	movups	(%eax),%xmm0
 	xorps	%xmm4,%xmm4
 	leal	16(%edx),%edx
 	cmpl	$256,%ecx
-	je	.L08714rounds
+	je	.L08914rounds
 	cmpl	$192,%ecx
-	je	.L08812rounds
+	je	.L09012rounds
 	cmpl	$128,%ecx
-	jne	.L089bad_keybits
+	jne	.L091bad_keybits
 .align	16
-.L09010rounds:
+.L09210rounds:
 	movl	$9,%ecx
 	movups	%xmm0,-16(%edx)
 .byte	102,15,58,223,200,1
-	call	.L091key_128_cold
+	call	.L093key_128_cold
 .byte	102,15,58,223,200,2
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,4
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,8
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,16
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,32
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,64
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,128
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,27
-	call	.L092key_128
+	call	.L094key_128
 .byte	102,15,58,223,200,54
-	call	.L092key_128
+	call	.L094key_128
 	movups	%xmm0,(%edx)
 	movl	%ecx,80(%edx)
 	xorl	%eax,%eax
 	ret
 .align	16
-.L092key_128:
+.L094key_128:
 	movups	%xmm0,(%edx)
 	leal	16(%edx),%edx
-.L091key_128_cold:
+.L093key_128_cold:
 	shufps	$16,%xmm0,%xmm4
 	xorps	%xmm4,%xmm0
 	shufps	$140,%xmm0,%xmm4
@@ -1990,38 +2041,38 @@ __aesni_set_encrypt_key:
 	xorps	%xmm1,%xmm0
 	ret
 .align	16
-.L08812rounds:
+.L09012rounds:
 	movq	16(%eax),%xmm2
 	movl	$11,%ecx
 	movups	%xmm0,-16(%edx)
 .byte	102,15,58,223,202,1
-	call	.L093key_192a_cold
+	call	.L095key_192a_cold
 .byte	102,15,58,223,202,2
-	call	.L094key_192b
+	call	.L096key_192b
 .byte	102,15,58,223,202,4
-	call	.L095key_192a
+	call	.L097key_192a
 .byte	102,15,58,223,202,8
-	call	.L094key_192b
+	call	.L096key_192b
 .byte	102,15,58,223,202,16
-	call	.L095key_192a
+	call	.L097key_192a
 .byte	102,15,58,223,202,32
-	call	.L094key_192b
+	call	.L096key_192b
 .byte	102,15,58,223,202,64
-	call	.L095key_192a
+	call	.L097key_192a
 .byte	102,15,58,223,202,128
-	call	.L094key_192b
+	call	.L096key_192b
 	movups	%xmm0,(%edx)
 	movl	%ecx,48(%edx)
 	xorl	%eax,%eax
 	ret
 .align	16
-.L095key_192a:
+.L097key_192a:
 	movups	%xmm0,(%edx)
 	leal	16(%edx),%edx
 .align	16
-.L093key_192a_cold:
+.L095key_192a_cold:
 	movaps	%xmm2,%xmm5
-.L096key_192b_warm:
+.L098key_192b_warm:
 	shufps	$16,%xmm0,%xmm4
 	movdqa	%xmm2,%xmm3
 	xorps	%xmm4,%xmm0
@@ -2035,56 +2086,56 @@ __aesni_set_encrypt_key:
 	pxor	%xmm3,%xmm2
 	ret
 .align	16
-.L094key_192b:
+.L096key_192b:
 	movaps	%xmm0,%xmm3
 	shufps	$68,%xmm0,%xmm5
 	movups	%xmm5,(%edx)
 	shufps	$78,%xmm2,%xmm3
 	movups	%xmm3,16(%edx)
 	leal	32(%edx),%edx
-	jmp	.L096key_192b_warm
+	jmp	.L098key_192b_warm
 .align	16
-.L08714rounds:
+.L08914rounds:
 	movups	16(%eax),%xmm2
 	movl	$13,%ecx
 	leal	16(%edx),%edx
 	movups	%xmm0,-32(%edx)
 	movups	%xmm2,-16(%edx)
 .byte	102,15,58,223,202,1
-	call	.L097key_256a_cold
+	call	.L099key_256a_cold
 .byte	102,15,58,223,200,1
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,2
-	call	.L099key_256a
+	call	.L101key_256a
 .byte	102,15,58,223,200,2
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,4
-	call	.L099key_256a
+	call	.L101key_256a
 .byte	102,15,58,223,200,4
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,8
-	call	.L099key_256a
+	call	.L101key_256a
 .byte	102,15,58,223,200,8
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,16
-	call	.L099key_256a
+	call	.L101key_256a
 .byte	102,15,58,223,200,16
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,32
-	call	.L099key_256a
+	call	.L101key_256a
 .byte	102,15,58,223,200,32
-	call	.L098key_256b
+	call	.L100key_256b
 .byte	102,15,58,223,202,64
-	call	.L099key_256a
+	call	.L101key_256a
 	movups	%xmm0,(%edx)
 	movl	%ecx,16(%edx)
 	xorl	%eax,%eax
 	ret
 .align	16
-.L099key_256a:
+.L101key_256a:
 	movups	%xmm2,(%edx)
 	leal	16(%edx),%edx
-.L097key_256a_cold:
+.L099key_256a_cold:
 	shufps	$16,%xmm0,%xmm4
 	xorps	%xmm4,%xmm0
 	shufps	$140,%xmm0,%xmm4
@@ -2093,7 +2144,7 @@ __aesni_set_encrypt_key:
 	xorps	%xmm1,%xmm0
 	ret
 .align	16
-.L098key_256b:
+.L100key_256b:
 	movups	%xmm0,(%edx)
 	leal	16(%edx),%edx
 	shufps	$16,%xmm2,%xmm4
@@ -2104,11 +2155,11 @@ __aesni_set_encrypt_key:
 	xorps	%xmm1,%xmm2
 	ret
 .align	4
-.L086bad_pointer:
+.L088bad_pointer:
 	movl	$-1,%eax
 	ret
 .align	4
-.L089bad_keybits:
+.L091bad_keybits:
 	movl	$-2,%eax
 	ret
 .globl	_aesni_set_encrypt_key
@@ -2133,7 +2184,7 @@ _aesni_set_decrypt_key:
 	movl	12(%esp),%edx
 	shll	$4,%ecx
 	testl	%eax,%eax
-	jnz	.L100dec_key_ret
+	jnz	.L102dec_key_ret
 	leal	16(%edx,%ecx,1),%eax
 	movups	(%edx),%xmm0
 	movups	(%eax),%xmm1
@@ -2141,7 +2192,7 @@ _aesni_set_decrypt_key:
 	movups	%xmm1,(%edx)
 	leal	16(%edx),%edx
 	leal	-16(%eax),%eax
-.L101dec_key_inverse:
+.L103dec_key_inverse:
 	movups	(%edx),%xmm0
 	movups	(%eax),%xmm1
 .byte	102,15,56,219,192
@@ -2151,12 +2202,12 @@ _aesni_set_decrypt_key:
 	movups	%xmm0,16(%eax)
 	movups	%xmm1,-16(%edx)
 	cmpl	%edx,%eax
-	ja	.L101dec_key_inverse
+	ja	.L103dec_key_inverse
 	movups	(%edx),%xmm0
 .byte	102,15,56,219,192
 	movups	%xmm0,(%edx)
 	xorl	%eax,%eax
-.L100dec_key_ret:
+.L102dec_key_ret:
 	ret
 .byte	65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69
 .byte	83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83