path: root/cipher/chacha20-amd64-avx512.S

/* chacha20-amd64-avx512.S  -  AVX512 implementation of ChaCha20 cipher
 *
 * Copyright (C) 2022 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This file is part of Libgcrypt.
 *
 * Libgcrypt is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * Libgcrypt is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Based on D. J. Bernstein reference implementation at
 * http://cr.yp.to/chacha.html:
 *
 * chacha-regs.c version 20080118
 * D. J. Bernstein
 * Public domain.
 */

#ifdef __x86_64
#include <config.h>
#if defined(HAVE_GCC_INLINE_ASM_AVX512) && \
   (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
    defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS))

#include "asm-common-amd64.h"

/* register macros */
#define INPUT %rdi
#define DST   %rsi
#define SRC   %rdx
#define NBLKS %rcx
#define ROUND %eax

/* vector registers */
#define X0 %zmm0
#define X1 %zmm1
#define X2 %zmm2
#define X3 %zmm3
#define X4 %zmm4
#define X5 %zmm5
#define X6 %zmm6
#define X7 %zmm7
#define X8 %zmm8
#define X9 %zmm9
#define X10 %zmm10
#define X11 %zmm11
#define X12 %zmm12
#define X13 %zmm13
#define X14 %zmm14
#define X15 %zmm15
#define X0y %ymm0
#define X1y %ymm1
#define X2y %ymm2
#define X3y %ymm3
#define X4y %ymm4
#define X5y %ymm5
#define X6y %ymm6
#define X7y %ymm7
#define X8y %ymm8
#define X9y %ymm9
#define X10y %ymm10
#define X11y %ymm11
#define X12y %ymm12
#define X13y %ymm13
#define X14y %ymm14
#define X15y %ymm15
#define X0x %xmm0
#define X1x %xmm1
#define X2x %xmm2
#define X3x %xmm3
#define X4x %xmm4
#define X5x %xmm5
#define X6x %xmm6
#define X7x %xmm7
#define X8x %xmm8
#define X9x %xmm9
#define X10x %xmm10
#define X11x %xmm11
#define X12x %xmm12
#define X13x %xmm13
#define X14x %xmm14
#define X15x %xmm15

#define TMP0 %zmm16
#define TMP1 %zmm17
#define TMP0y %ymm16
#define TMP1y %ymm17
#define TMP0x %xmm16
#define TMP1x %xmm17

#define COUNTER_ADD %zmm18
#define COUNTER_ADDy %ymm18
#define COUNTER_ADDx %xmm18

#define X12_SAVE %zmm19
#define X12_SAVEy %ymm19
#define X12_SAVEx %xmm19
#define X13_SAVE %zmm20
#define X13_SAVEy %ymm20
#define X13_SAVEx %xmm20

#define S0 %zmm21
#define S1 %zmm22
#define S2 %zmm23
#define S3 %zmm24
#define S4 %zmm25
#define S5 %zmm26
#define S6 %zmm27
#define S7 %zmm28
#define S8 %zmm29
#define S14 %zmm30
#define S15 %zmm31
#define S0y %ymm21
#define S1y %ymm22
#define S2y %ymm23
#define S3y %ymm24
#define S4y %ymm25
#define S5y %ymm26
#define S6y %ymm27
#define S7y %ymm28
#define S8y %ymm29
#define S14y %ymm30
#define S15y %ymm31
#define S0x %xmm21
#define S1x %xmm22
#define S2x %xmm23
#define S3x %xmm24
#define S4x %xmm25
#define S5x %xmm26
#define S6x %xmm27
#define S7x %xmm28
#define S8x %xmm29
#define S14x %xmm30
#define S15x %xmm31

/**********************************************************************
  helper macros
 **********************************************************************/

/* 4x4 32-bit integer matrix transpose */
#define transpose_4x4(x0,x1,x2,x3,t1,t2) \
	vpunpckhdq x1, x0, t2; \
	vpunpckldq x1, x0, x0; \
	\
	vpunpckldq x3, x2, t1; \
	vpunpckhdq x3, x2, x2; \
	\
	vpunpckhqdq t1, x0, x1; \
	vpunpcklqdq t1, x0, x0; \
	\
	vpunpckhqdq x2, t2, x3; \
	vpunpcklqdq x2, t2, x2;

/* 4x4 128-bit matrix transpose */
#define transpose_16byte_4x4(x0,x1,x2,x3,t1,t2) \
	vshufi32x4 $0xee, x1, x0, t2; \
	vshufi32x4 $0x44, x1, x0, x0; \
	\
	vshufi32x4 $0x44, x3, x2, t1; \
	vshufi32x4 $0xee, x3, x2, x2; \
	\
	vshufi32x4 $0xdd, t1, x0, x1; \
	vshufi32x4 $0x88, t1, x0, x0; \
	\
	vshufi32x4 $0xdd, x2, t2, x3; \
	vshufi32x4 $0x88, x2, t2, x2;

/* 2x2 128-bit matrix transpose */
#define transpose_16byte_2x2(x0,x1,t1) \
	vmovdqa32  x0, t1; \
	vshufi32x4 $0x0, x1, x0, x0; \
	vshufi32x4 $0x3, x1, t1, x1;

#define xor_src_dst_4x4(dst, src, offset, add, x0, x4, x8, x12) \
	vpxord (offset + 0 * (add))(src), x0, x0; \
	vpxord (offset + 1 * (add))(src), x4, x4; \
	vpxord (offset + 2 * (add))(src), x8, x8; \
	vpxord (offset + 3 * (add))(src), x12, x12; \
	vmovdqu32 x0, (offset + 0 * (add))(dst); \
	vmovdqu32 x4, (offset + 1 * (add))(dst); \
	vmovdqu32 x8, (offset + 2 * (add))(dst); \
	vmovdqu32 x12, (offset + 3 * (add))(dst);

#define xor_src_dst(dst, src, offset, xreg) \
	vpxord offset(src), xreg, xreg; \
	vmovdqu32 xreg, offset(dst);

#define clear_vec4(v0,v1,v2,v3) \
	vpxord v0, v0, v0; \
	vpxord v1, v1, v1; \
	vpxord v2, v2, v2; \
	vpxord v3, v3, v3;

#define clear_zmm16_zmm31() \
	clear_vec4(%ymm16, %ymm20, %ymm24, %ymm28); \
	clear_vec4(%ymm17, %ymm21, %ymm25, %ymm29); \
	clear_vec4(%ymm18, %ymm22, %ymm26, %ymm30); \
	clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31);

/**********************************************************************
  16-way (zmm), 8-way (ymm), 4-way (xmm) chacha20
 **********************************************************************/

#define ROTATE2(v1,v2,c)	\
	vprold $(c), v1, v1;	\
	vprold $(c), v2, v2;

#define XOR(ds,s) \
	vpxord s, ds, ds;

#define PLUS(ds,s) \
	vpaddd s, ds, ds;

#define QUARTERROUND2V(a1,b1,c1,d1,a2,b2,c2,d2)			\
	PLUS(a1,b1); PLUS(a2,b2); XOR(d1,a1); XOR(d2,a2);	\
	    ROTATE2(d1, d2, 16);				\
	PLUS(c1,d1); PLUS(c2,d2); XOR(b1,c1); XOR(b2,c2);	\
	    ROTATE2(b1, b2, 12);				\
	PLUS(a1,b1); PLUS(a2,b2); XOR(d1,a1); XOR(d2,a2);	\
	    ROTATE2(d1, d2, 8);					\
	PLUS(c1,d1); PLUS(c2,d2); XOR(b1,c1); XOR(b2,c2);	\
	    ROTATE2(b1, b2, 7);

/**********************************************************************
  1-way/2-way (xmm) chacha20
 **********************************************************************/

#define ROTATE(v1,c)			\
	vprold $(c), v1, v1;		\

#define WORD_SHUF(v1,shuf)		\
	vpshufd $shuf, v1, v1;

#define QUARTERROUND1H(x0,x1,x2,x3,shuf_x1,shuf_x2,shuf_x3) \
	PLUS(x0, x1); XOR(x3, x0); ROTATE(x3, 16); \
	PLUS(x2, x3); XOR(x1, x2); ROTATE(x1, 12); \
	PLUS(x0, x1); XOR(x3, x0); ROTATE(x3, 8); \
	PLUS(x2, x3); \
	  WORD_SHUF(x3, shuf_x3); \
		      XOR(x1, x2); \
	  WORD_SHUF(x2, shuf_x2); \
				   ROTATE(x1, 7); \
	  WORD_SHUF(x1, shuf_x1);

#define QUARTERROUND2H(x0,x1,x2,x3,y0,y1,y2,y3,shuf_x1,shuf_x2,shuf_x3) \
	PLUS(x0, x1); PLUS(y0, y1); XOR(x3, x0); XOR(y3, y0); \
	  ROTATE(x3, 16); ROTATE(y3, 16); \
	PLUS(x2, x3); PLUS(y2, y3); XOR(x1, x2); XOR(y1, y2); \
	  ROTATE(x1, 12); ROTATE(y1, 12); \
	PLUS(x0, x1); PLUS(y0, y1); XOR(x3, x0); XOR(y3, y0); \
	  ROTATE(x3, 8); ROTATE(y3, 8); \
	PLUS(x2, x3); PLUS(y2, y3); \
	  WORD_SHUF(x3, shuf_x3); WORD_SHUF(y3, shuf_x3); \
		      XOR(x1, x2); XOR(y1, y2); \
	  WORD_SHUF(x2, shuf_x2); WORD_SHUF(y2, shuf_x2); \
				   ROTATE(x1, 7); ROTATE(y1, 7); \
	  WORD_SHUF(x1, shuf_x1); WORD_SHUF(y1, shuf_x1);

SECTION_RODATA

.align 64
ELF(.type _gcry_chacha20_amd64_avx512_data,@object;)
_gcry_chacha20_amd64_avx512_data:
.Lcounter_0_1_2_3:
.Lcounter_0_1:
	.long 0,0,0,0
.Lone:
	.long 1,0,0,0
.Lcounter_2_3:
.Ltwo:
	.long 2,0,0,0
.Lthree:
	.long 3,0,0,0
.Linc_counter:
	.byte 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
ELF(.size _gcry_chacha20_amd64_avx512_data,.-_gcry_chacha20_amd64_avx512_data)

.text

.align 16
.globl _gcry_chacha20_amd64_avx512_blocks
ELF(.type _gcry_chacha20_amd64_avx512_blocks,@function;)
_gcry_chacha20_amd64_avx512_blocks:
	/* input:
	 *	%rdi: input
	 *	%rsi: dst
	 *	%rdx: src
	 *	%rcx: nblks
	 */
	CFI_STARTPROC();

	spec_stop_avx512;

	cmpq $4, NBLKS;
	jb .Lskip_vertical_handling;

	/* Load constants */
	vpmovzxbd .Linc_counter rRIP, COUNTER_ADD;
	kxnorq %k1, %k1, %k1;

	cmpq $16, NBLKS;
	jae .Lprocess_16v;

	/* Preload state to YMM registers */
	vpbroadcastd (0 * 4)(INPUT), S0y;
	vpbroadcastd (1 * 4)(INPUT), S1y;
	vpbroadcastd (2 * 4)(INPUT), S2y;
	vpbroadcastd (3 * 4)(INPUT), S3y;
	vpbroadcastd (4 * 4)(INPUT), S4y;
	vpbroadcastd (5 * 4)(INPUT), S5y;
	vpbroadcastd (6 * 4)(INPUT), S6y;
	vpbroadcastd (7 * 4)(INPUT), S7y;
	vpbroadcastd (8 * 4)(INPUT), S8y;
	vpbroadcastd (14 * 4)(INPUT), S14y;
	vpbroadcastd (15 * 4)(INPUT), S15y;
	jmp .Lskip16v;

.align 16
.Lprocess_16v:
	/* Process 16 ChaCha20 blocks */

	/* Preload state to ZMM registers */
	vpbroadcastd (0 * 4)(INPUT), S0;
	vpbroadcastd (1 * 4)(INPUT), S1;
	vpbroadcastd (2 * 4)(INPUT), S2;
	vpbroadcastd (3 * 4)(INPUT), S3;
	vpbroadcastd (4 * 4)(INPUT), S4;
	vpbroadcastd (5 * 4)(INPUT), S5;
	vpbroadcastd (6 * 4)(INPUT), S6;
	vpbroadcastd (7 * 4)(INPUT), S7;
	vpbroadcastd (8 * 4)(INPUT), S8;
	vpbroadcastd (14 * 4)(INPUT), S14;
	vpbroadcastd (15 * 4)(INPUT), S15;

	movl $20, ROUND;
	subq $16, NBLKS;

	/* Construct counter vectors X12 and X13 */
	vpmovm2d %k1, X9;
	vpaddd (12 * 4)(INPUT){1to16}, COUNTER_ADD, X12;
	vpbroadcastd (13 * 4)(INPUT), X13;
	vpcmpud $6, X12, COUNTER_ADD, %k2;
	vpsubd X9, X13, X13{%k2};
	vmovdqa32 X12, X12_SAVE;
	vmovdqa32 X13, X13_SAVE;

	/* Load vectors */
	vmovdqa32 S0, X0;
	vmovdqa32 S4, X4;
	vmovdqa32 S8, X8;
	vmovdqa32 S1, X1;
	vmovdqa32 S5, X5;
	vpbroadcastd (9 * 4)(INPUT), X9;
	QUARTERROUND2V(X0, X4,  X8, X12,   X1, X5,  X9, X13)
	vmovdqa32 S2, X2;
	vmovdqa32 S6, X6;
	vpbroadcastd (10 * 4)(INPUT), X10;
	vmovdqa32 S14, X14;
	vmovdqa32 S3, X3;
	vmovdqa32 S7, X7;
	vpbroadcastd (11 * 4)(INPUT), X11;
	vmovdqa32 S15, X15;

	/* Update counter */
	addq $16, (12 * 4)(INPUT);
	jmp .Lround2_entry_16v;

.align 16
.Loop16v:
	movl $20, ROUND;
	subq $16, NBLKS;

	vmovdqa32 S0, X0;
	vmovdqa32 S4, X4;
	vmovdqa32 S8, X8;
	transpose_16byte_4x4(X1, X5, X9, X13, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 1), 256, X1, X5, X9, X13);
	vpmovm2d %k1, X9;
	vpaddd (12 * 4)(INPUT){1to16}, COUNTER_ADD, X12;
	vpbroadcastd (13 * 4)(INPUT), X13;
	vpcmpud $6, X12, COUNTER_ADD, %k2;
	vpsubd X9, X13, X13{%k2};
	vmovdqa32 S1, X1;
	vmovdqa32 S5, X5;
	vpbroadcastd (9 * 4)(INPUT), X9;
	vmovdqa32 X12, X12_SAVE;
	vmovdqa32 X13, X13_SAVE;
	QUARTERROUND2V(X0, X4,  X8, X12,   X1, X5,  X9, X13)
	transpose_16byte_4x4(X2, X6, X10, X14, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 2), 256, X2, X6, X10, X14);
	vmovdqa32 S2, X2;
	vmovdqa32 S6, X6;
	vpbroadcastd (10 * 4)(INPUT), X10;
	vmovdqa32 S14, X14;
	transpose_16byte_4x4(X3, X7, X11, X15, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 3), 256, X3, X7, X11, X15);
	leaq (16 * 64)(SRC), SRC;
	leaq (16 * 64)(DST), DST;
	vmovdqa32 S3, X3;
	vmovdqa32 S7, X7;
	vpbroadcastd (11 * 4)(INPUT), X11;
	vmovdqa32 S15, X15;

	/* Update counter */
	addq $16, (12 * 4)(INPUT);
	jmp .Lround2_entry_16v;

.align 16
.Lround2_16v:
	QUARTERROUND2V(X2, X7,  X8, X13,   X3, X4,  X9, X14)
	QUARTERROUND2V(X0, X4,  X8, X12,   X1, X5,  X9, X13)
.align 16
.Lround2_entry_16v:
	QUARTERROUND2V(X2, X6, X10, X14,   X3, X7, X11, X15)
	QUARTERROUND2V(X0, X5, X10, X15,   X1, X6, X11, X12)
	subl $2, ROUND;
	jnz .Lround2_16v;

	PLUS(X0, S0);
	PLUS(X1, S1);
	QUARTERROUND2V(X2, X7,  X8, X13,   X3, X4,  X9, X14)
	PLUS(X2, S2);
	PLUS(X3, S3);
	transpose_4x4(X0, X1, X2, X3, TMP0, TMP1);
	PLUS(X4, S4);
	PLUS(X5, S5);
	PLUS(X6, S6);
	PLUS(X7, S7);
	transpose_4x4(X4, X5, X6, X7, TMP0, TMP1);
	PLUS(X8, S8);
	PLUS(X9, (9 * 4)(INPUT){1to16});
	PLUS(X10, (10 * 4)(INPUT){1to16});
	PLUS(X11, (11 * 4)(INPUT){1to16});
	transpose_4x4(X8, X9, X10, X11, TMP0, TMP1);
	PLUS(X12, X12_SAVE);
	PLUS(X13, X13_SAVE);
	PLUS(X14, S14);
	PLUS(X15, S15);
	transpose_4x4(X12, X13, X14, X15, TMP0, TMP1);

	transpose_16byte_4x4(X0, X4, X8, X12, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 0), 256, X0, X4, X8, X12);

	cmpq $16, NBLKS;
	jae .Loop16v;

	transpose_16byte_4x4(X1, X5, X9, X13, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 1), 256, X1, X5, X9, X13);
	transpose_16byte_4x4(X2, X6, X10, X14, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 2), 256, X2, X6, X10, X14);
	transpose_16byte_4x4(X3, X7, X11, X15, TMP0, TMP1);
	xor_src_dst_4x4(DST, SRC, (64 * 3), 256, X3, X7, X11, X15);

	leaq (16 * 64)(SRC), SRC;
	leaq (16 * 64)(DST), DST;

.align 16
.Lskip16v:
	cmpq $8, NBLKS;
	jb .Lskip8v;

	/* Process 8 ChaCha20 blocks */

	/* Construct counter vectors X12 and X13 */
	vpmovm2d %k1, X9y;
	vpaddd (12 * 4)(INPUT){1to8}, COUNTER_ADDy, X12y;
	vpbroadcastd (13 * 4)(INPUT), X13y;
	vpcmpud $6, X12y, COUNTER_ADDy, %k2;
	vpsubd X9y, X13y, X13y{%k2};
	vmovdqa32 X12y, X12_SAVEy;
	vmovdqa32 X13y, X13_SAVEy;

	/* Load vectors */
	vmovdqa32 S0y, X0y;
	vmovdqa32 S4y, X4y;
	vmovdqa32 S8y, X8y;
	vmovdqa32 S1y, X1y;
	vmovdqa32 S5y, X5y;
	vpbroadcastd (9 * 4)(INPUT), X9y;
	vmovdqa32 S2y, X2y;
	vmovdqa32 S6y, X6y;
	vpbroadcastd (10 * 4)(INPUT), X10y;
	vmovdqa32 S14y, X14y;
	vmovdqa32 S3y, X3y;
	vmovdqa32 S7y, X7y;
	vpbroadcastd (11 * 4)(INPUT), X11y;
	vmovdqa32 S15y, X15y;

	/* Update counter */
	addq $8, (12 * 4)(INPUT);

	movl $20, ROUND;
	subq $8, NBLKS;
.align 16
.Lround2_8v:
	QUARTERROUND2V(X0y, X4y,  X8y, X12y,   X1y, X5y,  X9y, X13y)
	QUARTERROUND2V(X2y, X6y, X10y, X14y,   X3y, X7y, X11y, X15y)
	QUARTERROUND2V(X0y, X5y, X10y, X15y,   X1y, X6y, X11y, X12y)
	QUARTERROUND2V(X2y, X7y,  X8y, X13y,   X3y, X4y,  X9y, X14y)
	subl $2, ROUND;
	jnz .Lround2_8v;

	PLUS(X0y, S0y);
	PLUS(X1y, S1y);
	PLUS(X2y, S2y);
	PLUS(X3y, S3y);
	transpose_4x4(X0y, X1y, X2y, X3y, TMP0y, TMP1y);
	PLUS(X4y, S4y);
	PLUS(X5y, S5y);
	PLUS(X6y, S6y);
	PLUS(X7y, S7y);
	transpose_4x4(X4y, X5y, X6y, X7y, TMP0y, TMP1y);
	PLUS(X8y, S8y);
	transpose_16byte_2x2(X0y, X4y, TMP0y);
	PLUS(X9y, (9 * 4)(INPUT){1to8});
	transpose_16byte_2x2(X1y, X5y, TMP0y);
	PLUS(X10y, (10 * 4)(INPUT){1to8});
	transpose_16byte_2x2(X2y, X6y, TMP0y);
	PLUS(X11y, (11 * 4)(INPUT){1to8});
	transpose_16byte_2x2(X3y, X7y, TMP0y);
	xor_src_dst_4x4(DST, SRC, (16 * 0),  64, X0y, X1y, X2y, X3y);
	transpose_4x4(X8y, X9y, X10y, X11y, TMP0y, TMP1y);
	PLUS(X12y, X12_SAVEy);
	PLUS(X13y, X13_SAVEy);
	PLUS(X14y, S14y);
	PLUS(X15y, S15y);
	xor_src_dst_4x4(DST, SRC, (16 * 16), 64, X4y, X5y, X6y, X7y);
	transpose_4x4(X12y, X13y, X14y, X15y, TMP0y, TMP1y);
	transpose_16byte_2x2(X8y, X12y, TMP0y);
	transpose_16byte_2x2(X9y, X13y, TMP0y);
	transpose_16byte_2x2(X10y, X14y, TMP0y);
	transpose_16byte_2x2(X11y, X15y, TMP0y);
	xor_src_dst_4x4(DST, SRC, (16 * 2),  64, X8y, X9y, X10y, X11y);
	xor_src_dst_4x4(DST, SRC, (16 * 18), 64, X12y, X13y, X14y, X15y);

	leaq (8 * 64)(SRC), SRC;
	leaq (8 * 64)(DST), DST;

.align 16
.Lskip8v:
	cmpq $4, NBLKS;
	jb .Lskip4v;

	/* Process 4 ChaCha20 blocks */

	/* Construct counter vectors X12 and X13 */
	vpmovm2d %k1, X9x;
	vpaddd (12 * 4)(INPUT){1to4}, COUNTER_ADDx, X12x;
	vpbroadcastd (13 * 4)(INPUT), X13x;
	vpcmpud $6, X12x, COUNTER_ADDx, %k2;
	vpsubd X9x, X13x, X13x{%k2};
	vmovdqa32 X12x, X12_SAVEx;
	vmovdqa32 X13x, X13_SAVEx;

	/* Load vectors */
	vmovdqa32 S0x, X0x;
	vmovdqa32 S4x, X4x;
	vmovdqa32 S8x, X8x;
	vmovdqa32 S1x, X1x;
	vmovdqa32 S5x, X5x;
	vpbroadcastd (9 * 4)(INPUT), X9x;
	vmovdqa32 S2x, X2x;
	vmovdqa32 S6x, X6x;
	vpbroadcastd (10 * 4)(INPUT), X10x;
	vmovdqa32 S14x, X14x;
	vmovdqa32 S3x, X3x;
	vmovdqa32 S7x, X7x;
	vpbroadcastd (11 * 4)(INPUT), X11x;
	vmovdqa32 S15x, X15x;

	/* Update counter */
	addq $4, (12 * 4)(INPUT);

	movl $20, ROUND;
	subq $4, NBLKS;
.align 16
.Lround2_4v:
	QUARTERROUND2V(X0x, X4x,  X8x, X12x,   X1x, X5x,  X9x, X13x)
	QUARTERROUND2V(X2x, X6x, X10x, X14x,   X3x, X7x, X11x, X15x)
	QUARTERROUND2V(X0x, X5x, X10x, X15x,   X1x, X6x, X11x, X12x)
	QUARTERROUND2V(X2x, X7x,  X8x, X13x,   X3x, X4x,  X9x, X14x)
	subl $2, ROUND;
	jnz .Lround2_4v;

	PLUS(X0x, S0x);
	PLUS(X1x, S1x);
	PLUS(X2x, S2x);
	PLUS(X3x, S3x);
	transpose_4x4(X0x, X1x, X2x, X3x, TMP0x, TMP1x);
	PLUS(X4x, S4x);
	PLUS(X5x, S5x);
	PLUS(X6x, S6x);
	PLUS(X7x, S7x);
	xor_src_dst_4x4(DST, SRC, (16 * 0), 64, X0x, X1x, X2x, X3x);
	transpose_4x4(X4x, X5x, X6x, X7x, TMP0x, TMP1x);
	PLUS(X8x, S8x);
	PLUS(X9x, (9 * 4)(INPUT){1to4});
	PLUS(X10x, (10 * 4)(INPUT){1to4});
	PLUS(X11x, (11 * 4)(INPUT){1to4});
	xor_src_dst_4x4(DST, SRC, (16 * 1), 64, X4x, X5x, X6x, X7x);
	transpose_4x4(X8x, X9x, X10x, X11x, TMP0x, TMP1x);
	PLUS(X12x, X12_SAVEx);
	PLUS(X13x, X13_SAVEx);
	PLUS(X14x, S14x);
	PLUS(X15x, S15x);
	xor_src_dst_4x4(DST, SRC, (16 * 2), 64, X8x, X9x, X10x, X11x);
	transpose_4x4(X12x, X13x, X14x, X15x, TMP0x, TMP1x);
	xor_src_dst_4x4(DST, SRC, (16 * 3), 64, X12x, X13x, X14x, X15x);

	leaq (4 * 64)(SRC), SRC;
	leaq (4 * 64)(DST), DST;

.align 16
.Lskip4v:
	/* clear AVX512 registers */
	kxorq %k2, %k2, %k2;
	vzeroupper;
	clear_zmm16_zmm31();

.align 16
.Lskip_vertical_handling:
	cmpq $0, NBLKS;
	je .Ldone;

	/* Load state */
	vmovdqu (0 * 4)(INPUT), X10x;
	vmovdqu (4 * 4)(INPUT), X11x;
	vmovdqu (8 * 4)(INPUT), X12x;
	vmovdqu (12 * 4)(INPUT), X13x;

	/* Load constant */
	vmovdqa .Lone rRIP, X4x;

	cmpq $1, NBLKS;
	je .Lhandle1;

	/* Process two ChaCha20 blocks (XMM) */
	movl $20, ROUND;
	subq $2, NBLKS;

	vmovdqa X10x, X0x;
	vmovdqa X11x, X1x;
	vmovdqa X12x, X2x;
	vmovdqa X13x, X3x;

	vmovdqa X10x, X8x;
	vmovdqa X11x, X9x;
	vmovdqa X12x, X14x;
	vpaddq X4x, X13x, X15x;
	vmovdqa X15x, X7x;

.align 16
.Lround2_2:
	QUARTERROUND2H(X0x, X1x, X2x,  X3x,  X8x, X9x, X14x, X15x,
		       0x39, 0x4e, 0x93);
	QUARTERROUND2H(X0x, X1x, X2x,  X3x,  X8x, X9x, X14x, X15x,
		       0x93, 0x4e, 0x39);
	subl $2, ROUND;
	jnz .Lround2_2;

	PLUS(X0x, X10x);
	PLUS(X1x, X11x);
	PLUS(X2x, X12x);
	PLUS(X3x, X13x);

	vpaddq .Ltwo rRIP, X13x, X13x; /* Update counter */

	xor_src_dst_4x4(DST, SRC, 0 * 4, 4 * 4, X0x, X1x, X2x, X3x);

	PLUS(X8x, X10x);
	PLUS(X9x, X11x);
	PLUS(X14x, X12x);
	PLUS(X15x, X7x);

	xor_src_dst_4x4(DST, SRC, 16 * 4, 4 * 4, X8x, X9x, X14x, X15x);
	lea (2 * 64)(DST), DST;
	lea (2 * 64)(SRC), SRC;

	cmpq $0, NBLKS;
	je .Lskip1;

.align 16
.Lhandle1:
	/* Process one ChaCha20 block (XMM) */
	movl $20, ROUND;
	subq $1, NBLKS;

	vmovdqa X10x, X0x;
	vmovdqa X11x, X1x;
	vmovdqa X12x, X2x;
	vmovdqa X13x, X3x;

.align 16
.Lround2_1:
	QUARTERROUND1H(X0x, X1x, X2x, X3x, 0x39, 0x4e, 0x93);
	QUARTERROUND1H(X0x, X1x, X2x, X3x, 0x93, 0x4e, 0x39);
	subl $2, ROUND;
	jnz .Lround2_1;

	PLUS(X0x, X10x);
	PLUS(X1x, X11x);
	PLUS(X2x, X12x);
	PLUS(X3x, X13x);

	vpaddq X4x, X13x, X13x; /* Update counter */

	xor_src_dst_4x4(DST, SRC, 0 * 4, 4 * 4, X0x, X1x, X2x, X3x);

.align 16
.Lskip1:
	/* Store counter */
	vmovdqu X13x, (12 * 4)(INPUT);

.align 16
.Ldone:
	vzeroall; /* clears ZMM0-ZMM15 */

	xorl %eax, %eax;
	ret_spec_stop;
	CFI_ENDPROC();
ELF(.size _gcry_chacha20_amd64_avx512_blocks,
	  .-_gcry_chacha20_amd64_avx512_blocks;)

#endif /*defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS)*/
#endif /*__x86_64*/