mirror of
https://git.savannah.gnu.org/git/grub.git
synced 2026-10-24 08:09:22 +00:00
Vladimir Serbinenko
3312af6e07
We currently use an old version of libgcrypt which results in us having fewer ciphers and missing on many other improvements. Signed-off-by: Vladimir Serbinenko <phcoder@gmail.com> Reviewed-by: Daniel Kiper <daniel.kiper@oracle.com>
2805 lines
82 KiB
ArmAsm
2805 lines
82 KiB
ArmAsm
/* VAES/AVX2 i386 accelerated AES for Libgcrypt
|
|
* Copyright (C) 2023 Jussi Kivilinna <jussi.kivilinna@iki.fi>
|
|
*
|
|
* This file is part of Libgcrypt.
|
|
*
|
|
* Libgcrypt is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU Lesser General Public License as
|
|
* published by the Free Software Foundation; either version 2.1 of
|
|
* the License, or (at your option) any later version.
|
|
*
|
|
* Libgcrypt is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#if defined(__i386__)
|
|
#include <config.h>
|
|
#if (defined(HAVE_COMPATIBLE_GCC_I386_PLATFORM_AS) || \
|
|
defined(HAVE_COMPATIBLE_GCC_WIN32_PLATFORM_AS)) && \
|
|
defined(ENABLE_AESNI_SUPPORT) && defined(ENABLE_AVX2_SUPPORT) && \
|
|
defined(HAVE_GCC_INLINE_ASM_VAES_VPCLMUL)
|
|
|
|
#include "asm-common-i386.h"
|
|
|
|
.text
|
|
|
|
DECL_GET_PC_THUNK(eax);
|
|
|
|
/**********************************************************************
|
|
helper macros
|
|
**********************************************************************/
|
|
#define AES_OP4(op, key, b0, b1, b2, b3) \
|
|
op key, b0, b0; \
|
|
op key, b1, b1; \
|
|
op key, b2, b2; \
|
|
op key, b3, b3;
|
|
|
|
#define VAESENC4(key, b0, b1, b2, b3) \
|
|
AES_OP4(vaesenc, key, b0, b1, b2, b3)
|
|
|
|
#define VAESDEC4(key, b0, b1, b2, b3) \
|
|
AES_OP4(vaesdec, key, b0, b1, b2, b3)
|
|
|
|
#define XOR4(key, b0, b1, b2, b3) \
|
|
AES_OP4(vpxor, key, b0, b1, b2, b3)
|
|
|
|
#define AES_OP2(op, key, b0, b1) \
|
|
op key, b0, b0; \
|
|
op key, b1, b1;
|
|
|
|
#define VAESENC2(key, b0, b1) \
|
|
AES_OP2(vaesenc, key, b0, b1)
|
|
|
|
#define VAESDEC2(key, b0, b1) \
|
|
AES_OP2(vaesdec, key, b0, b1)
|
|
|
|
#define XOR2(key, b0, b1) \
|
|
AES_OP2(vpxor, key, b0, b1)
|
|
|
|
#define VAESENC6(key, b0, b1, b2, b3, b4, b5) \
|
|
AES_OP4(vaesenc, key, b0, b1, b2, b3); \
|
|
AES_OP2(vaesenc, key, b4, b5)
|
|
|
|
#define VAESDEC6(key, b0, b1, b2, b3, b4, b5) \
|
|
AES_OP4(vaesdec, key, b0, b1, b2, b3); \
|
|
AES_OP2(vaesdec, key, b4, b5)
|
|
|
|
#define XOR6(key, b0, b1, b2, b3, b4, b5) \
|
|
AES_OP4(vpxor, key, b0, b1, b2, b3); \
|
|
AES_OP2(vpxor, key, b4, b5)
|
|
|
|
#define CADDR(name, reg) \
|
|
(name - SYM_NAME(_gcry_vaes_consts))(reg)
|
|
|
|
/**********************************************************************
|
|
CBC-mode decryption
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): iv
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
*/
|
|
CFI_STARTPROC();
|
|
pushl %edi;
|
|
CFI_PUSH(%edi);
|
|
pushl %esi;
|
|
CFI_PUSH(%esi);
|
|
|
|
movl 8+4(%esp), %edi;
|
|
movl 8+8(%esp), %esi;
|
|
movl 8+12(%esp), %edx;
|
|
movl 8+16(%esp), %ecx;
|
|
movl 8+20(%esp), %eax;
|
|
|
|
/* Process 8 blocks per loop. */
|
|
.align 8
|
|
.Lcbc_dec_blk8:
|
|
cmpl $8, %eax;
|
|
jb .Lcbc_dec_blk4;
|
|
|
|
leal -8(%eax), %eax;
|
|
|
|
/* Load input and xor first key. Update IV. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm0;
|
|
vmovdqu (2 * 16)(%ecx), %ymm1;
|
|
vmovdqu (4 * 16)(%ecx), %ymm2;
|
|
vmovdqu (6 * 16)(%ecx), %ymm3;
|
|
vmovdqu (%esi), %xmm6; /* Load IV. */
|
|
vinserti128 $1, %xmm0, %ymm6, %ymm5;
|
|
vextracti128 $1, %ymm3, (%esi); /* Store IV. */
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vpxor %ymm4, %ymm2, %ymm2;
|
|
vpxor %ymm4, %ymm3, %ymm3;
|
|
vmovdqu (1 * 16)(%ecx), %ymm6;
|
|
vmovdqu (3 * 16)(%ecx), %ymm7;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcbc_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lcbc_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcbc_dec_blk8_last:
|
|
vpxor %ymm4, %ymm5, %ymm5;
|
|
vpxor %ymm4, %ymm6, %ymm6;
|
|
vpxor %ymm4, %ymm7, %ymm7;
|
|
vpxor (5 * 16)(%ecx), %ymm4, %ymm4;
|
|
leal (8 * 16)(%ecx), %ecx;
|
|
vaesdeclast %ymm5, %ymm0, %ymm0;
|
|
vaesdeclast %ymm6, %ymm1, %ymm1;
|
|
vaesdeclast %ymm7, %ymm2, %ymm2;
|
|
vaesdeclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
|
|
jmp .Lcbc_dec_blk8;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lcbc_dec_blk4:
|
|
cmpl $4, %eax;
|
|
jb .Lcbc_dec_blk1;
|
|
|
|
leal -4(%eax), %eax;
|
|
|
|
/* Load input and xor first key. Update IV. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm0;
|
|
vmovdqu (2 * 16)(%ecx), %ymm1;
|
|
vmovdqu (%esi), %xmm6; /* Load IV. */
|
|
vinserti128 $1, %xmm0, %ymm6, %ymm5;
|
|
vextracti128 $1, %ymm1, (%esi); /* Store IV. */
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vmovdqu (1 * 16)(%ecx), %ymm6;
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcbc_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lcbc_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcbc_dec_blk4_last:
|
|
vpxor %ymm4, %ymm5, %ymm5;
|
|
vpxor %ymm4, %ymm6, %ymm6;
|
|
vaesdeclast %ymm5, %ymm0, %ymm0;
|
|
vaesdeclast %ymm6, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lcbc_dec_blk1:
|
|
cmpl $1, %eax;
|
|
jb .Ldone_cbc_dec;
|
|
|
|
leal -1(%eax), %eax;
|
|
|
|
/* Load input. */
|
|
vmovdqu (%ecx), %xmm2;
|
|
leal 16(%ecx), %ecx;
|
|
|
|
/* Xor first key. */
|
|
vpxor (0 * 16)(%edi), %xmm2, %xmm0;
|
|
|
|
/* AES rounds. */
|
|
vaesdec (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcbc_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lcbc_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcbc_dec_blk1_last:
|
|
vpxor (%esi), %xmm1, %xmm1;
|
|
vaesdeclast %xmm1, %xmm0, %xmm0;
|
|
vmovdqu %xmm2, (%esi);
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lcbc_dec_blk1;
|
|
|
|
.align 8
|
|
.Ldone_cbc_dec:
|
|
popl %esi;
|
|
CFI_POP(%esi);
|
|
popl %edi;
|
|
CFI_POP(%edi);
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386))
|
|
|
|
/**********************************************************************
|
|
CFB-mode decryption
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): iv
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
*/
|
|
CFI_STARTPROC();
|
|
pushl %edi;
|
|
CFI_PUSH(%edi);
|
|
pushl %esi;
|
|
CFI_PUSH(%esi);
|
|
|
|
movl 8+4(%esp), %edi;
|
|
movl 8+8(%esp), %esi;
|
|
movl 8+12(%esp), %edx;
|
|
movl 8+16(%esp), %ecx;
|
|
movl 8+20(%esp), %eax;
|
|
|
|
/* Process 8 blocks per loop. */
|
|
.align 8
|
|
.Lcfb_dec_blk8:
|
|
cmpl $8, %eax;
|
|
jb .Lcfb_dec_blk4;
|
|
|
|
leal -8(%eax), %eax;
|
|
|
|
/* Load IV. */
|
|
vmovdqu (%esi), %xmm0;
|
|
|
|
/* Load input and xor first key. Update IV. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm5;
|
|
vinserti128 $1, %xmm5, %ymm0, %ymm0;
|
|
vmovdqu (1 * 16)(%ecx), %ymm1;
|
|
vmovdqu (3 * 16)(%ecx), %ymm2;
|
|
vmovdqu (5 * 16)(%ecx), %ymm3;
|
|
vmovdqu (7 * 16)(%ecx), %xmm6;
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vpxor %ymm4, %ymm2, %ymm2;
|
|
vpxor %ymm4, %ymm3, %ymm3;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
vmovdqu %xmm6, (%esi); /* Store IV. */
|
|
vmovdqu (2 * 16)(%ecx), %ymm6;
|
|
vmovdqu (4 * 16)(%ecx), %ymm7;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcfb_dec_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lcfb_dec_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcfb_dec_blk8_last:
|
|
vpxor %ymm4, %ymm5, %ymm5;
|
|
vpxor %ymm4, %ymm6, %ymm6;
|
|
vpxor %ymm4, %ymm7, %ymm7;
|
|
vpxor (6 * 16)(%ecx), %ymm4, %ymm4;
|
|
leal (8 * 16)(%ecx), %ecx;
|
|
vaesenclast %ymm5, %ymm0, %ymm0;
|
|
vaesenclast %ymm6, %ymm1, %ymm1;
|
|
vaesenclast %ymm7, %ymm2, %ymm2;
|
|
vaesenclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
|
|
jmp .Lcfb_dec_blk8;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lcfb_dec_blk4:
|
|
cmpl $4, %eax;
|
|
jb .Lcfb_dec_blk1;
|
|
|
|
leal -4(%eax), %eax;
|
|
|
|
/* Load IV. */
|
|
vmovdqu (%esi), %xmm0;
|
|
|
|
/* Load input and xor first key. Update IV. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm5;
|
|
vinserti128 $1, %xmm5, %ymm0, %ymm0;
|
|
vmovdqu (1 * 16)(%ecx), %ymm1;
|
|
vmovdqu (3 * 16)(%ecx), %xmm6;
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
vmovdqu %xmm6, (%esi); /* Store IV. */
|
|
vmovdqu (2 * 16)(%ecx), %ymm6;
|
|
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcfb_dec_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lcfb_dec_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcfb_dec_blk4_last:
|
|
vpxor %ymm4, %ymm5, %ymm5;
|
|
vpxor %ymm4, %ymm6, %ymm6;
|
|
vaesenclast %ymm5, %ymm0, %ymm0;
|
|
vaesenclast %ymm6, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lcfb_dec_blk1:
|
|
cmpl $1, %eax;
|
|
jb .Ldone_cfb_dec;
|
|
|
|
leal -1(%eax), %eax;
|
|
|
|
/* Load IV. */
|
|
vmovdqu (%esi), %xmm0;
|
|
|
|
/* Xor first key. */
|
|
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
|
|
|
|
/* Load input as next IV. */
|
|
vmovdqu (%ecx), %xmm2;
|
|
leal 16(%ecx), %ecx;
|
|
|
|
/* AES rounds. */
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
vmovdqu %xmm2, (%esi); /* Store IV. */
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lcfb_dec_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lcfb_dec_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lcfb_dec_blk1_last:
|
|
vpxor %xmm2, %xmm1, %xmm1;
|
|
vaesenclast %xmm1, %xmm0, %xmm0;
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lcfb_dec_blk1;
|
|
|
|
.align 8
|
|
.Ldone_cfb_dec:
|
|
popl %esi;
|
|
CFI_POP(%esi);
|
|
popl %edi;
|
|
CFI_POP(%edi);
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386))
|
|
|
|
/**********************************************************************
|
|
CTR-mode encryption
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): iv
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
*/
|
|
CFI_STARTPROC();
|
|
|
|
GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax);
|
|
|
|
pushl %ebp;
|
|
CFI_PUSH(%ebp);
|
|
movl %esp, %ebp;
|
|
CFI_DEF_CFA_REGISTER(%ebp);
|
|
|
|
subl $(3 * 32 + 3 * 4), %esp;
|
|
andl $-32, %esp;
|
|
|
|
movl %edi, (3 * 32 + 0 * 4)(%esp);
|
|
CFI_REG_ON_STACK(edi, 3 * 32 + 0 * 4);
|
|
movl %esi, (3 * 32 + 1 * 4)(%esp);
|
|
CFI_REG_ON_STACK(esi, 3 * 32 + 1 * 4);
|
|
movl %ebx, (3 * 32 + 2 * 4)(%esp);
|
|
CFI_REG_ON_STACK(ebx, 3 * 32 + 2 * 4);
|
|
|
|
movl %eax, %ebx;
|
|
movl 4+4(%ebp), %edi;
|
|
movl 4+8(%ebp), %esi;
|
|
movl 4+12(%ebp), %edx;
|
|
movl 4+16(%ebp), %ecx;
|
|
|
|
#define prepare_ctr_const(minus_one, minus_two) \
|
|
vpcmpeqd minus_one, minus_one, minus_one; \
|
|
vpsrldq $8, minus_one, minus_one; /* 0:-1 */ \
|
|
vpaddq minus_one, minus_one, minus_two; /* 0:-2 */
|
|
|
|
#define inc_le128(x, minus_one, tmp) \
|
|
vpcmpeqq minus_one, x, tmp; \
|
|
vpsubq minus_one, x, x; \
|
|
vpslldq $8, tmp, tmp; \
|
|
vpsubq tmp, x, x;
|
|
|
|
#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \
|
|
vpcmpeqq minus_one, x, tmp1; \
|
|
vpcmpeqq minus_two, x, tmp2; \
|
|
vpor tmp1, tmp2, tmp2; \
|
|
vpsubq minus_two, x, x; \
|
|
vpslldq $8, tmp2, tmp2; \
|
|
vpsubq tmp2, x, x;
|
|
|
|
#define handle_ctr_128bit_add(nblks) \
|
|
movl 12(%esi), %eax; \
|
|
bswapl %eax; \
|
|
addl $nblks, %eax; \
|
|
bswapl %eax; \
|
|
movl %eax, 12(%esi); \
|
|
jnc 1f; \
|
|
\
|
|
movl 8(%esi), %eax; \
|
|
bswapl %eax; \
|
|
adcl $0, %eax; \
|
|
bswapl %eax; \
|
|
movl %eax, 8(%esi); \
|
|
\
|
|
movl 4(%esi), %eax; \
|
|
bswapl %eax; \
|
|
adcl $0, %eax; \
|
|
bswapl %eax; \
|
|
movl %eax, 4(%esi); \
|
|
\
|
|
movl 0(%esi), %eax; \
|
|
bswapl %eax; \
|
|
adcl $0, %eax; \
|
|
bswapl %eax; \
|
|
movl %eax, 0(%esi); \
|
|
.align 8; \
|
|
1:;
|
|
|
|
cmpl $12, 4+20(%ebp);
|
|
jae .Lctr_enc_blk12_loop;
|
|
jmp .Lctr_enc_blk4;
|
|
|
|
/* Process 12 blocks per loop. */
|
|
.align 16
|
|
.Lctr_enc_blk12_loop:
|
|
subl $12, 4+20(%ebp);
|
|
|
|
vbroadcasti128 (%esi), %ymm6;
|
|
|
|
/* detect if carry handling is needed */
|
|
movl 12(%esi), %eax;
|
|
addl $(12 << 24), %eax;
|
|
jc .Lctr_enc_blk12_handle_carry;
|
|
movl %eax, 12(%esi);
|
|
|
|
.Lctr_enc_blk12_byte_bige_add:
|
|
/* Increment counters. */
|
|
vpaddb CADDR(.Lbige_addb_0, %ebx), %ymm6, %ymm0;
|
|
vpaddb CADDR(.Lbige_addb_2, %ebx), %ymm6, %ymm1;
|
|
vpaddb CADDR(.Lbige_addb_4, %ebx), %ymm6, %ymm2;
|
|
vpaddb CADDR(.Lbige_addb_6, %ebx), %ymm6, %ymm3;
|
|
vpaddb CADDR(.Lbige_addb_8, %ebx), %ymm6, %ymm5;
|
|
vpaddb CADDR(.Lbige_addb_10, %ebx), %ymm6, %ymm6;
|
|
|
|
.Lctr_enc_blk12_rounds:
|
|
/* AES rounds */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
XOR6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr_enc_blk12_last;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lctr_enc_blk12_last;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr_enc_blk12_last:
|
|
vpxor (0 * 16)(%ecx), %ymm4, %ymm7; /* Xor src to last round key. */
|
|
vaesenclast %ymm7, %ymm0, %ymm0;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vpxor (2 * 16)(%ecx), %ymm4, %ymm7;
|
|
vpxor (4 * 16)(%ecx), %ymm4, %ymm0;
|
|
vaesenclast %ymm7, %ymm1, %ymm1;
|
|
vaesenclast %ymm0, %ymm2, %ymm2;
|
|
vpxor (6 * 16)(%ecx), %ymm4, %ymm7;
|
|
vpxor (8 * 16)(%ecx), %ymm4, %ymm0;
|
|
vpxor (10 * 16)(%ecx), %ymm4, %ymm4;
|
|
leal (12 * 16)(%ecx), %ecx;
|
|
vaesenclast %ymm7, %ymm3, %ymm3;
|
|
vaesenclast %ymm0, %ymm5, %ymm5;
|
|
vaesenclast %ymm4, %ymm6, %ymm6;
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
vmovdqu %ymm5, (8 * 16)(%edx);
|
|
vmovdqu %ymm6, (10 * 16)(%edx);
|
|
leal (12 * 16)(%edx), %edx;
|
|
|
|
cmpl $12, 4+20(%ebp);
|
|
jae .Lctr_enc_blk12_loop;
|
|
jmp .Lctr_enc_blk4;
|
|
|
|
.align 8
|
|
.Lctr_enc_blk12_handle_only_ctr_carry:
|
|
handle_ctr_128bit_add(12);
|
|
jmp .Lctr_enc_blk12_byte_bige_add;
|
|
|
|
.align 8
|
|
.Lctr_enc_blk12_handle_carry:
|
|
jz .Lctr_enc_blk12_handle_only_ctr_carry;
|
|
/* Increment counters (handle carry). */
|
|
prepare_ctr_const(%ymm4, %ymm7);
|
|
vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2;
|
|
vpshufb %xmm2, %xmm6, %xmm1; /* be => le */
|
|
vmovdqa %xmm1, %xmm0;
|
|
inc_le128(%xmm1, %xmm4, %xmm5);
|
|
vinserti128 $1, %xmm1, %ymm0, %ymm6; /* ctr: +1:+0 */
|
|
handle_ctr_128bit_add(12);
|
|
vpshufb %ymm2, %ymm6, %ymm0;
|
|
vmovdqa %ymm0, (0 * 32)(%esp);
|
|
add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +3:+2 */
|
|
vpshufb %ymm2, %ymm6, %ymm0;
|
|
vmovdqa %ymm0, (1 * 32)(%esp);
|
|
add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +5:+4 */
|
|
vpshufb %ymm2, %ymm6, %ymm0;
|
|
vmovdqa %ymm0, (2 * 32)(%esp);
|
|
add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +7:+6 */
|
|
vpshufb %ymm2, %ymm6, %ymm3;
|
|
add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +9:+8 */
|
|
vpshufb %ymm2, %ymm6, %ymm5;
|
|
add2_le128(%ymm6, %ymm4, %ymm7, %ymm2, %ymm1); /* ctr: +11:+10 */
|
|
vmovdqa (0 * 32)(%esp), %ymm0;
|
|
vmovdqa (1 * 32)(%esp), %ymm1;
|
|
vmovdqa (2 * 32)(%esp), %ymm2;
|
|
vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm6, %ymm6;
|
|
|
|
jmp .Lctr_enc_blk12_rounds;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lctr_enc_blk4:
|
|
cmpl $4, 4+20(%ebp);
|
|
jb .Lctr_enc_blk1;
|
|
|
|
subl $4, 4+20(%ebp);
|
|
|
|
vbroadcasti128 (%esi), %ymm3;
|
|
|
|
/* detect if carry handling is needed */
|
|
movl 12(%esi), %eax;
|
|
addl $(4 << 24), %eax;
|
|
jc .Lctr_enc_blk4_handle_carry;
|
|
movl %eax, 12(%esi);
|
|
|
|
.Lctr_enc_blk4_byte_bige_add:
|
|
/* Increment counters. */
|
|
vpaddb CADDR(.Lbige_addb_0, %ebx), %ymm3, %ymm0;
|
|
vpaddb CADDR(.Lbige_addb_2, %ebx), %ymm3, %ymm1;
|
|
|
|
.Lctr_enc_blk4_rounds:
|
|
/* AES rounds */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
XOR2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lctr_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr_enc_blk4_last:
|
|
vpxor (0 * 16)(%ecx), %ymm4, %ymm5; /* Xor src to last round key. */
|
|
vpxor (2 * 16)(%ecx), %ymm4, %ymm6;
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
vaesenclast %ymm5, %ymm0, %ymm0;
|
|
vaesenclast %ymm6, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
jmp .Lctr_enc_blk1;
|
|
|
|
.align 8
|
|
.Lctr_enc_blk4_handle_only_ctr_carry:
|
|
handle_ctr_128bit_add(4);
|
|
jmp .Lctr_enc_blk4_byte_bige_add;
|
|
|
|
.align 8
|
|
.Lctr_enc_blk4_handle_carry:
|
|
jz .Lctr_enc_blk4_handle_only_ctr_carry;
|
|
/* Increment counters (handle carry). */
|
|
prepare_ctr_const(%ymm4, %ymm7);
|
|
vpshufb CADDR(.Lbswap128_mask, %ebx), %xmm3, %xmm1; /* be => le */
|
|
vmovdqa %xmm1, %xmm0;
|
|
inc_le128(%xmm1, %xmm4, %xmm5);
|
|
vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
|
|
vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm0;
|
|
handle_ctr_128bit_add(4);
|
|
add2_le128(%ymm3, %ymm4, %ymm7, %ymm5, %ymm6); /* ctr: +3:+2 */
|
|
vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm1;
|
|
|
|
jmp .Lctr_enc_blk4_rounds;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lctr_enc_blk1:
|
|
cmpl $1, 4+20(%ebp);
|
|
jb .Ldone_ctr_enc;
|
|
|
|
subl $1, 4+20(%ebp);
|
|
|
|
/* Load and increament counter. */
|
|
vmovdqu (%esi), %xmm0;
|
|
handle_ctr_128bit_add(1);
|
|
|
|
/* AES rounds. */
|
|
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lctr_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr_enc_blk1_last:
|
|
vpxor (%ecx), %xmm1, %xmm1; /* Xor src to last round key. */
|
|
leal 16(%ecx), %ecx;
|
|
vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lctr_enc_blk1;
|
|
|
|
.align 8
|
|
.Ldone_ctr_enc:
|
|
vpxor %ymm0, %ymm0, %ymm0;
|
|
movl (3 * 32 + 0 * 4)(%esp), %edi;
|
|
CFI_RESTORE(edi);
|
|
movl (3 * 32 + 1 * 4)(%esp), %esi;
|
|
CFI_RESTORE(esi);
|
|
movl (3 * 32 + 2 * 4)(%esp), %ebx;
|
|
CFI_RESTORE(ebx);
|
|
vmovdqa %ymm0, (0 * 32)(%esp);
|
|
vmovdqa %ymm0, (1 * 32)(%esp);
|
|
vmovdqa %ymm0, (2 * 32)(%esp);
|
|
leave;
|
|
CFI_LEAVE();
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386))
|
|
|
|
/**********************************************************************
|
|
Little-endian 32-bit CTR-mode encryption (GCM-SIV)
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): counter
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
*/
|
|
CFI_STARTPROC();
|
|
|
|
GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax);
|
|
|
|
pushl %ebp;
|
|
CFI_PUSH(%ebp);
|
|
movl %esp, %ebp;
|
|
CFI_DEF_CFA_REGISTER(%ebp);
|
|
|
|
subl $(3 * 4), %esp;
|
|
|
|
movl %edi, (0 * 4)(%esp);
|
|
CFI_REG_ON_STACK(edi, 0 * 4);
|
|
movl %esi, (1 * 4)(%esp);
|
|
CFI_REG_ON_STACK(esi, 1 * 4);
|
|
movl %ebx, (2 * 4)(%esp);
|
|
CFI_REG_ON_STACK(ebx, 2 * 4);
|
|
|
|
movl %eax, %ebx;
|
|
movl 4+4(%ebp), %edi;
|
|
movl 4+8(%ebp), %esi;
|
|
movl 4+12(%ebp), %edx;
|
|
movl 4+16(%ebp), %ecx;
|
|
movl 4+20(%ebp), %eax;
|
|
|
|
vbroadcasti128 (%esi), %ymm7; /* Load CTR. */
|
|
|
|
/* Process 12 blocks per loop. */
|
|
.align 8
|
|
.Lctr32le_enc_blk12:
|
|
cmpl $12, %eax;
|
|
jb .Lctr32le_enc_blk4;
|
|
|
|
leal -12(%eax), %eax;
|
|
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
|
|
/* Increment counters. */
|
|
vpaddd CADDR(.Lle_addd_0, %ebx), %ymm7, %ymm0;
|
|
vpaddd CADDR(.Lle_addd_2, %ebx), %ymm7, %ymm1;
|
|
vpaddd CADDR(.Lle_addd_4, %ebx), %ymm7, %ymm2;
|
|
vpaddd CADDR(.Lle_addd_6, %ebx), %ymm7, %ymm3;
|
|
vpaddd CADDR(.Lle_addd_8, %ebx), %ymm7, %ymm5;
|
|
vpaddd CADDR(.Lle_addd_10, %ebx), %ymm7, %ymm6;
|
|
|
|
vpaddd CADDR(.Lle_addd_12_2, %ebx), %ymm7, %ymm7;
|
|
vmovdqu %xmm7, (%esi); /* Store CTR. */
|
|
|
|
/* AES rounds */
|
|
XOR6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr32le_enc_blk8_last;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lctr32le_enc_blk8_last;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr32le_enc_blk8_last:
|
|
vpxor (0 * 16)(%ecx), %ymm4, %ymm7; /* Xor src to last round key. */
|
|
vaesenclast %ymm7, %ymm0, %ymm0;
|
|
vpxor (2 * 16)(%ecx), %ymm4, %ymm7;
|
|
vaesenclast %ymm7, %ymm1, %ymm1;
|
|
vpxor (4 * 16)(%ecx), %ymm4, %ymm7;
|
|
vaesenclast %ymm7, %ymm2, %ymm2;
|
|
vpxor (6 * 16)(%ecx), %ymm4, %ymm7;
|
|
vaesenclast %ymm7, %ymm3, %ymm3;
|
|
vpxor (8 * 16)(%ecx), %ymm4, %ymm7;
|
|
vpxor (10 * 16)(%ecx), %ymm4, %ymm4;
|
|
vaesenclast %ymm7, %ymm5, %ymm5;
|
|
vbroadcasti128 (%esi), %ymm7; /* Reload CTR. */
|
|
vaesenclast %ymm4, %ymm6, %ymm6;
|
|
leal (12 * 16)(%ecx), %ecx;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
vmovdqu %ymm5, (8 * 16)(%edx);
|
|
vmovdqu %ymm6, (10 * 16)(%edx);
|
|
leal (12 * 16)(%edx), %edx;
|
|
|
|
jmp .Lctr32le_enc_blk12;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lctr32le_enc_blk4:
|
|
cmpl $4, %eax;
|
|
jb .Lctr32le_enc_blk1;
|
|
|
|
leal -4(%eax), %eax;
|
|
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
|
|
/* Increment counters. */
|
|
vpaddd CADDR(.Lle_addd_0, %ebx), %ymm7, %ymm0;
|
|
vpaddd CADDR(.Lle_addd_2, %ebx), %ymm7, %ymm1;
|
|
|
|
vpaddd CADDR(.Lle_addd_4_2, %ebx), %ymm7, %ymm7;
|
|
|
|
/* AES rounds */
|
|
XOR2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr32le_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lctr32le_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr32le_enc_blk4_last:
|
|
vpxor (0 * 16)(%ecx), %ymm4, %ymm5; /* Xor src to last round key. */
|
|
vpxor (2 * 16)(%ecx), %ymm4, %ymm6;
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
vaesenclast %ymm5, %ymm0, %ymm0;
|
|
vaesenclast %ymm6, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lctr32le_enc_blk1:
|
|
cmpl $1, %eax;
|
|
jb .Ldone_ctr32le_enc;
|
|
|
|
leal -1(%eax), %eax;
|
|
|
|
/* Load and increament counter. */
|
|
vmovdqu %xmm7, %xmm0;
|
|
vpaddd CADDR(.Lle_addd_1, %ebx), %xmm7, %xmm7;
|
|
|
|
/* AES rounds. */
|
|
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lctr32le_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lctr32le_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lctr32le_enc_blk1_last:
|
|
vpxor (%ecx), %xmm1, %xmm1; /* Xor src to last round key. */
|
|
leal 16(%ecx), %ecx;
|
|
vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lctr32le_enc_blk1;
|
|
|
|
.align 8
|
|
.Ldone_ctr32le_enc:
|
|
vmovdqu %xmm7, (%esi); /* Store CTR. */
|
|
movl (0 * 4)(%esp), %edi;
|
|
CFI_RESTORE(edi);
|
|
movl (1 * 4)(%esp), %esi;
|
|
CFI_RESTORE(esi);
|
|
movl (2 * 4)(%esp), %ebx;
|
|
CFI_RESTORE(ebx);
|
|
leave;
|
|
CFI_LEAVE();
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386))
|
|
|
|
/**********************************************************************
|
|
OCB-mode encryption/decryption/authentication
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): dst
|
|
* (esp + 12): src
|
|
* (esp + 16): nblocks
|
|
* (esp + 20): nrounds
|
|
* (esp + 24): offset
|
|
* (esp + 28): checksum
|
|
* (esp + 32): blkn
|
|
* (esp + 36): L table
|
|
* (esp + 44): encrypt/decrypt/auth mode
|
|
*/
|
|
CFI_STARTPROC();
|
|
|
|
pushl %ebp;
|
|
CFI_PUSH(%ebp);
|
|
movl %esp, %ebp;
|
|
CFI_DEF_CFA_REGISTER(%ebp);
|
|
|
|
#define STACK_VEC_POS 0
|
|
#define STACK_TMP_Y0 (STACK_VEC_POS + 0 * 32)
|
|
#define STACK_TMP_Y1 (STACK_VEC_POS + 1 * 32)
|
|
#define STACK_TMP_Y2 (STACK_VEC_POS + 2 * 32)
|
|
#define STACK_TMP_Y3 (STACK_VEC_POS + 3 * 32)
|
|
#define STACK_TMP_Y4 (STACK_VEC_POS + 4 * 32)
|
|
#define STACK_TMP_Y5 (STACK_VEC_POS + 5 * 32)
|
|
#define STACK_FXL_KEY (STACK_VEC_POS + 6 * 32)
|
|
#define STACK_OFFSET_AND_F_KEY (STACK_VEC_POS + 7 * 32)
|
|
#define STACK_CHECKSUM (STACK_VEC_POS + 8 * 32)
|
|
#define STACK_GPR_POS (9 * 32)
|
|
#define STACK_END_POS (STACK_GPR_POS + 3 * 4)
|
|
|
|
subl $STACK_END_POS, %esp;
|
|
andl $-32, %esp;
|
|
|
|
movl %edi, (STACK_GPR_POS + 0 * 4)(%esp);
|
|
CFI_REG_ON_STACK(edi, STACK_GPR_POS + 0 * 4);
|
|
movl %esi, (STACK_GPR_POS + 1 * 4)(%esp);
|
|
CFI_REG_ON_STACK(esi, STACK_GPR_POS + 1 * 4);
|
|
movl %ebx, (STACK_GPR_POS + 2 * 4)(%esp);
|
|
CFI_REG_ON_STACK(ebx, STACK_GPR_POS + 2 * 4);
|
|
|
|
movl 4+4(%ebp), %edi;
|
|
movl 4+8(%ebp), %esi;
|
|
movl 4+12(%ebp), %edx;
|
|
movl 4+32(%ebp), %ebx;
|
|
|
|
movl 4+24(%ebp), %eax;
|
|
movl 4+20(%ebp), %ecx;
|
|
leal (, %ecx, 4), %ecx;
|
|
vmovdqu (%eax), %xmm1; /* offset */
|
|
vmovdqa (%edi), %xmm0; /* first key */
|
|
vpxor %xmm0, %xmm1, %xmm1; /* offset ^ first key */
|
|
vpxor (%edi, %ecx, 4), %xmm0, %xmm0; /* first key ^ last key */
|
|
vinserti128 $1, %xmm0, %ymm0, %ymm0;
|
|
vpxor %ymm2, %ymm2, %ymm2;
|
|
vmovdqa %xmm1, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
vmovdqa %ymm2, (STACK_CHECKSUM)(%esp);
|
|
vmovdqa %ymm0, (STACK_FXL_KEY)(%esp);
|
|
|
|
cmpl $12, 4+16(%ebp);
|
|
jae .Locb_crypt_blk12_loop;
|
|
jmp .Locb_crypt_blk4;
|
|
|
|
/* Process 12 blocks per loop. */
|
|
.align 16
|
|
.Locb_crypt_blk12_loop:
|
|
subl $12, 4+16(%ebp);
|
|
|
|
movl 4+36(%ebp), %ecx;
|
|
vmovdqa (%ecx), %xmm7; /* Preload L[0] */
|
|
|
|
testl $1, %ebx;
|
|
jz .Locb_crypt_blk12_nblk_even;
|
|
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
|
|
leal 1(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+1)
|
|
shll $4, %eax;
|
|
vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm1;
|
|
vmovdqa %ymm1, (STACK_TMP_Y0)(%esp);
|
|
|
|
leal 3(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+3)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm0, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm2;
|
|
|
|
leal 5(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+5)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm0, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm3;
|
|
|
|
leal 7(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+7)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm0, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm4;
|
|
|
|
leal 9(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+9)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm0, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm5;
|
|
|
|
leal 11(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+11)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm0, %xmm1;
|
|
|
|
leal 12(%ebx), %ebx;
|
|
vpxor %xmm7, %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm6;
|
|
|
|
cmpl $1, 4+40(%ebp);
|
|
jb .Locb_dec_blk12;
|
|
ja .Locb_auth_blk12;
|
|
jmp .Locb_enc_blk12;
|
|
|
|
.align 8
|
|
.Locb_crypt_blk12_nblk_even:
|
|
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
|
|
vpxor (STACK_OFFSET_AND_F_KEY)(%esp), %xmm7, %xmm1;
|
|
|
|
leal 2(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+2)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm1;
|
|
vmovdqa %ymm1, (STACK_TMP_Y0)(%esp);
|
|
|
|
vpxor %xmm7, %xmm0, %xmm1;
|
|
|
|
leal 4(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+4)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm2;
|
|
|
|
vpxor %xmm7, %xmm0, %xmm1;
|
|
|
|
leal 6(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+6)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm3;
|
|
|
|
vpxor %xmm7, %xmm0, %xmm1;
|
|
|
|
leal 8(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+8)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm4;
|
|
|
|
vpxor %xmm7, %xmm0, %xmm1;
|
|
|
|
leal 10(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+10)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm5;
|
|
|
|
vpxor %xmm7, %xmm0, %xmm1;
|
|
|
|
leal 12(%ebx), %ebx;
|
|
tzcntl %ebx, %eax; // ntz(blkn+12)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm0;
|
|
vinserti128 $1, %xmm0, %ymm1, %ymm6;
|
|
|
|
cmpl $1, 4+40(%ebp);
|
|
jb .Locb_dec_blk12;
|
|
ja .Locb_auth_blk12;
|
|
|
|
.align 8
|
|
.Locb_enc_blk12:
|
|
vmovdqa %ymm2, (STACK_TMP_Y1)(%esp);
|
|
vmovdqa %ymm3, (STACK_TMP_Y2)(%esp);
|
|
vmovdqa %ymm4, (STACK_TMP_Y3)(%esp);
|
|
vmovdqa %ymm5, (STACK_TMP_Y4)(%esp);
|
|
vmovdqa %ymm6, (STACK_TMP_Y5)(%esp);
|
|
vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
|
|
vmovdqu 0*16(%edx), %ymm1;
|
|
vmovdqu 2*16(%edx), %ymm2;
|
|
vmovdqu 4*16(%edx), %ymm3;
|
|
vmovdqu 6*16(%edx), %ymm4;
|
|
vmovdqu 8*16(%edx), %ymm5;
|
|
vmovdqu 10*16(%edx), %ymm6;
|
|
leal 12*16(%edx), %edx;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor %ymm1, %ymm2, %ymm0;
|
|
vpxor %ymm3, %ymm4, %ymm7;
|
|
vpxor %ymm5, %ymm0, %ymm0;
|
|
vpxor %ymm6, %ymm7, %ymm7;
|
|
vpxor %ymm0, %ymm7, %ymm7;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm7, %ymm7;
|
|
|
|
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
|
|
vpxor (STACK_TMP_Y0)(%esp), %ymm1, %ymm1;
|
|
vpxor (STACK_TMP_Y1)(%esp), %ymm2, %ymm2;
|
|
vpxor (STACK_TMP_Y2)(%esp), %ymm3, %ymm3;
|
|
vpxor (STACK_TMP_Y3)(%esp), %ymm4, %ymm4;
|
|
vpxor (STACK_TMP_Y4)(%esp), %ymm5, %ymm5;
|
|
vpxor (STACK_TMP_Y5)(%esp), %ymm6, %ymm6;
|
|
|
|
vmovdqa %ymm7, (STACK_CHECKSUM)(%esp);
|
|
|
|
/* AES rounds */
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_enc_blk12_last;
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
jz .Locb_enc_blk12_last;
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_enc_blk12_last:
|
|
vmovdqa (STACK_FXL_KEY)(%esp), %ymm0;
|
|
vpxor (STACK_TMP_Y0)(%esp), %ymm0, %ymm7;
|
|
vaesenclast %ymm7, %ymm1, %ymm1;
|
|
vpxor (STACK_TMP_Y1)(%esp), %ymm0, %ymm7;
|
|
vmovdqu %ymm1, 0*16(%esi);
|
|
vpxor (STACK_TMP_Y2)(%esp), %ymm0, %ymm1;
|
|
vaesenclast %ymm7, %ymm2, %ymm2;
|
|
vpxor (STACK_TMP_Y3)(%esp), %ymm0, %ymm7;
|
|
vaesenclast %ymm1, %ymm3, %ymm3;
|
|
vpxor (STACK_TMP_Y4)(%esp), %ymm0, %ymm1;
|
|
vaesenclast %ymm7, %ymm4, %ymm4;
|
|
vpxor (STACK_TMP_Y5)(%esp), %ymm0, %ymm7;
|
|
vaesenclast %ymm1, %ymm5, %ymm5;
|
|
vaesenclast %ymm7, %ymm6, %ymm6;
|
|
vmovdqu %ymm2, 2*16(%esi);
|
|
vmovdqu %ymm3, 4*16(%esi);
|
|
vmovdqu %ymm4, 6*16(%esi);
|
|
vmovdqu %ymm5, 8*16(%esi);
|
|
vmovdqu %ymm6, 10*16(%esi);
|
|
leal 12*16(%esi), %esi;
|
|
|
|
cmpl $12, 4+16(%ebp);
|
|
jae .Locb_crypt_blk12_loop;
|
|
jmp .Locb_crypt_blk12_cleanup;
|
|
|
|
.align 8
|
|
.Locb_auth_blk12:
|
|
vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
|
|
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
|
|
vmovdqa (STACK_TMP_Y0)(%esp), %ymm1;
|
|
vpxor 0*16(%edx), %ymm1, %ymm1;
|
|
vpxor 2*16(%edx), %ymm2, %ymm2;
|
|
vpxor 4*16(%edx), %ymm3, %ymm3;
|
|
vpxor 6*16(%edx), %ymm4, %ymm4;
|
|
vpxor 8*16(%edx), %ymm5, %ymm5;
|
|
vpxor 10*16(%edx), %ymm6, %ymm6;
|
|
leal 12*16(%edx), %edx;
|
|
|
|
/* AES rounds */
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_auth_blk12_last;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
jz .Locb_auth_blk12_last;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm0;
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_auth_blk12_last:
|
|
vaesenclast %ymm0, %ymm1, %ymm1;
|
|
vaesenclast %ymm0, %ymm2, %ymm2;
|
|
vaesenclast %ymm0, %ymm3, %ymm3;
|
|
vaesenclast %ymm0, %ymm4, %ymm4;
|
|
vaesenclast %ymm0, %ymm5, %ymm5;
|
|
vaesenclast %ymm0, %ymm6, %ymm6;
|
|
|
|
vpxor %ymm1, %ymm2, %ymm0;
|
|
vpxor %ymm3, %ymm4, %ymm4;
|
|
vpxor %ymm5, %ymm0, %ymm0;
|
|
vpxor %ymm6, %ymm4, %ymm4;
|
|
vpxor %ymm0, %ymm4, %ymm4;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm4, %ymm4;
|
|
vmovdqa %ymm4, (STACK_CHECKSUM)(%esp);
|
|
|
|
cmpl $12, 4+16(%ebp);
|
|
jae .Locb_crypt_blk12_loop;
|
|
jmp .Locb_crypt_blk12_cleanup;
|
|
|
|
.align 8
|
|
.Locb_dec_blk12:
|
|
vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
|
|
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */
|
|
vmovdqa (STACK_TMP_Y0)(%esp), %ymm1;
|
|
vmovdqu 0*16(%edx), %ymm0;
|
|
vmovdqu 2*16(%edx), %ymm7;
|
|
vpxor %ymm0, %ymm1, %ymm1;
|
|
vmovdqa %ymm2, (STACK_TMP_Y1)(%esp);
|
|
vpxor %ymm7, %ymm2, %ymm2;
|
|
vmovdqu 4*16(%edx), %ymm0;
|
|
vmovdqu 6*16(%edx), %ymm7;
|
|
vmovdqa %ymm3, (STACK_TMP_Y2)(%esp);
|
|
vmovdqa %ymm4, (STACK_TMP_Y3)(%esp);
|
|
vpxor %ymm0, %ymm3, %ymm3;
|
|
vpxor %ymm7, %ymm4, %ymm4;
|
|
vmovdqu 8*16(%edx), %ymm0;
|
|
vmovdqu 10*16(%edx), %ymm7;
|
|
leal 12*16(%edx), %edx;
|
|
vmovdqa %ymm5, (STACK_TMP_Y4)(%esp);
|
|
vmovdqa %ymm6, (STACK_TMP_Y5)(%esp);
|
|
vpxor %ymm0, %ymm5, %ymm5;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
vpxor %ymm7, %ymm6, %ymm6;
|
|
|
|
/* AES rounds */
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_dec_blk12_last;
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
jz .Locb_dec_blk12_last;
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6);
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_dec_blk12_last:
|
|
vmovdqa (STACK_FXL_KEY)(%esp), %ymm0;
|
|
vpxor (STACK_TMP_Y0)(%esp), %ymm0, %ymm7;
|
|
vaesdeclast %ymm7, %ymm1, %ymm1;
|
|
vmovdqu %ymm1, 0*16(%esi);
|
|
vpxor (STACK_TMP_Y1)(%esp), %ymm0, %ymm1;
|
|
vpxor (STACK_TMP_Y2)(%esp), %ymm0, %ymm7;
|
|
vaesdeclast %ymm1, %ymm2, %ymm2;
|
|
vpxor (STACK_TMP_Y3)(%esp), %ymm0, %ymm1;
|
|
vaesdeclast %ymm7, %ymm3, %ymm3;
|
|
vpxor (STACK_TMP_Y4)(%esp), %ymm0, %ymm7;
|
|
vaesdeclast %ymm1, %ymm4, %ymm4;
|
|
vpxor (STACK_TMP_Y5)(%esp), %ymm0, %ymm0;
|
|
vaesdeclast %ymm7, %ymm5, %ymm5;
|
|
vaesdeclast %ymm0, %ymm6, %ymm6;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor %ymm2, %ymm3, %ymm0;
|
|
vpxor %ymm4, %ymm5, %ymm7;
|
|
vpxor %ymm6, %ymm0, %ymm0;
|
|
vpxor 0*16(%esi), %ymm7, %ymm7;
|
|
vpxor %ymm0, %ymm7, %ymm7;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm7, %ymm7;
|
|
|
|
vmovdqu %ymm2, 2*16(%esi);
|
|
vmovdqu %ymm3, 4*16(%esi);
|
|
vmovdqu %ymm4, 6*16(%esi);
|
|
vmovdqu %ymm5, 8*16(%esi);
|
|
vmovdqu %ymm6, 10*16(%esi);
|
|
leal 12*16(%esi), %esi;
|
|
|
|
vmovdqa %ymm7, (STACK_CHECKSUM)(%esp);
|
|
|
|
cmpl $12, 4+16(%ebp);
|
|
jae .Locb_crypt_blk12_loop;
|
|
|
|
.align 8
|
|
.Locb_crypt_blk12_cleanup:
|
|
vpxor %ymm0, %ymm0, %ymm0;
|
|
vmovdqa %ymm0, (STACK_TMP_Y0)(%esp);
|
|
vmovdqa %ymm0, (STACK_TMP_Y1)(%esp);
|
|
vmovdqa %ymm0, (STACK_TMP_Y2)(%esp);
|
|
vmovdqa %ymm0, (STACK_TMP_Y3)(%esp);
|
|
vmovdqa %ymm0, (STACK_TMP_Y4)(%esp);
|
|
vmovdqa %ymm0, (STACK_TMP_Y5)(%esp);
|
|
|
|
/* Process trailing four blocks. */
|
|
.align 8
|
|
.Locb_crypt_blk4:
|
|
cmpl $4, 4+16(%ebp);
|
|
jb .Locb_crypt_blk1;
|
|
|
|
subl $4, 4+16(%ebp);
|
|
|
|
movl 4+36(%ebp), %ecx;
|
|
vmovdqa (%ecx), %xmm7; /* Preload L[0] */
|
|
|
|
testl $1, %ebx;
|
|
jz .Locb_crypt_blk4_nblk_even;
|
|
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
|
|
leal 1(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+1)
|
|
shll $4, %eax;
|
|
vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm1;
|
|
|
|
vpxor %xmm7, %xmm1, %xmm2;
|
|
vinserti128 $1, %xmm2, %ymm1, %ymm6;
|
|
|
|
leal 3(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+3)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm2, %xmm3;
|
|
|
|
leal 4(%ebx), %ebx;
|
|
vpxor %xmm7, %xmm3, %xmm4;
|
|
vinserti128 $1, %xmm4, %ymm3, %ymm7;
|
|
vmovdqa %xmm4, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
|
|
cmpl $1, 4+40(%ebp);
|
|
jb .Locb_dec_blk4;
|
|
ja .Locb_auth_blk4;
|
|
jmp .Locb_enc_blk4;
|
|
|
|
.align 8
|
|
.Locb_crypt_blk4_nblk_even:
|
|
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
|
|
vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1;
|
|
vpxor %xmm7, %xmm1, %xmm1;
|
|
|
|
leal 2(%ebx), %eax;
|
|
tzcntl %eax, %eax; // ntz(blkn+2)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm1, %xmm2;
|
|
vinserti128 $1, %xmm2, %ymm1, %ymm6;
|
|
|
|
vpxor %xmm7, %xmm2, %xmm3;
|
|
|
|
leal 4(%ebx), %ebx;
|
|
tzcntl %ebx, %eax; // ntz(blkn+4)
|
|
shll $4, %eax;
|
|
vpxor (%ecx, %eax), %xmm3, %xmm4;
|
|
vinserti128 $1, %xmm4, %ymm3, %ymm7;
|
|
vmovdqa %xmm4, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
|
|
cmpl $1, 4+40(%ebp);
|
|
jb .Locb_dec_blk4;
|
|
ja .Locb_auth_blk4;
|
|
|
|
.align 8
|
|
.Locb_enc_blk4:
|
|
vmovdqu 0*16(%edx), %ymm1;
|
|
vmovdqu 2*16(%edx), %ymm2;
|
|
leal 4*16(%edx), %edx;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor %ymm1, %ymm2, %ymm5;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5;
|
|
vmovdqa %ymm5, (STACK_CHECKSUM)(%esp);
|
|
|
|
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
|
|
vpxor %ymm6, %ymm1, %ymm1;
|
|
vpxor %ymm7, %ymm2, %ymm2;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_enc_blk4_last;
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
jz .Locb_enc_blk4_last;
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_enc_blk4_last:
|
|
vmovdqa (STACK_FXL_KEY)(%esp), %ymm0;
|
|
vpxor %ymm0, %ymm6, %ymm6; /* Xor offset to last round key. */
|
|
vpxor %ymm0, %ymm7, %ymm7;
|
|
vaesenclast %ymm6, %ymm1, %ymm1;
|
|
vaesenclast %ymm7, %ymm2, %ymm2;
|
|
vmovdqu %ymm1, 0*16(%esi);
|
|
vmovdqu %ymm2, 2*16(%esi);
|
|
leal 4*16(%esi), %esi;
|
|
|
|
jmp .Locb_crypt_blk1;
|
|
|
|
.align 8
|
|
.Locb_auth_blk4:
|
|
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
|
|
vpxor 0*16(%edx), %ymm6, %ymm1;
|
|
vpxor 2*16(%edx), %ymm7, %ymm2;
|
|
leal 4*16(%edx), %edx;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_auth_blk4_last;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
jz .Locb_auth_blk4_last;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESENC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm0;
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_auth_blk4_last:
|
|
vaesenclast %ymm0, %ymm1, %ymm1;
|
|
vaesenclast %ymm0, %ymm2, %ymm2;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor %ymm1, %ymm2, %ymm5;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5;
|
|
vmovdqa %ymm5, (STACK_CHECKSUM)(%esp);
|
|
|
|
jmp .Locb_crypt_blk1;
|
|
|
|
.align 8
|
|
.Locb_dec_blk4:
|
|
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */
|
|
vpxor 0*16(%edx), %ymm6, %ymm1;
|
|
vpxor 2*16(%edx), %ymm7, %ymm2;
|
|
leal 4*16(%edx), %edx;
|
|
|
|
/* AES rounds */
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_dec_blk4_last;
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
jz .Locb_dec_blk4_last;
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm0;
|
|
VAESDEC2(%ymm0, %ymm1, %ymm2);
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_dec_blk4_last:
|
|
vmovdqa (STACK_FXL_KEY)(%esp), %ymm0;
|
|
vpxor %ymm0, %ymm6, %ymm6; /* Xor offset to last round key. */
|
|
vpxor %ymm0, %ymm7, %ymm7;
|
|
vaesdeclast %ymm6, %ymm1, %ymm1;
|
|
vaesdeclast %ymm7, %ymm2, %ymm2;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor %ymm1, %ymm2, %ymm5;
|
|
vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5;
|
|
|
|
vmovdqu %ymm1, 0*16(%esi);
|
|
vmovdqu %ymm2, 2*16(%esi);
|
|
leal 4*16(%esi), %esi;
|
|
|
|
vmovdqa %ymm5, (STACK_CHECKSUM)(%esp);
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Locb_crypt_blk1:
|
|
cmpl $1, 4+16(%ebp);
|
|
jb .Locb_crypt_done;
|
|
|
|
subl $1, 4+16(%ebp);
|
|
|
|
movl 4+36(%ebp), %ecx;
|
|
leal 1(%ebx), %ebx;
|
|
tzcntl %ebx, %eax; // ntz(blkn+1)
|
|
shll $4, %eax;
|
|
vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm7;
|
|
vpxor (%ecx, %eax), %xmm7, %xmm7;
|
|
|
|
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
|
|
vmovdqa %xmm7, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
|
|
cmpl $1, 4+40(%ebp);
|
|
jb .Locb_dec_blk1;
|
|
ja .Locb_auth_blk1;
|
|
vmovdqu (%edx), %xmm0;
|
|
leal 16(%edx), %edx;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor (STACK_CHECKSUM)(%esp), %xmm0, %xmm1;
|
|
vmovdqa %xmm1, (STACK_CHECKSUM)(%esp);
|
|
|
|
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
|
|
vpxor %xmm7, %xmm0, %xmm0;
|
|
|
|
/* AES rounds. */
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_enc_blk1_last;
|
|
vaesenc (10 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
jz .Locb_enc_blk1_last;
|
|
vaesenc (12 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_enc_blk1_last:
|
|
vpxor (STACK_FXL_KEY)(%esp), %xmm7, %xmm1;
|
|
vaesenclast %xmm1, %xmm0, %xmm0;
|
|
vmovdqu %xmm0, (%esi);
|
|
leal 16(%esi), %esi;
|
|
|
|
jmp .Locb_crypt_blk1;
|
|
|
|
.align 8
|
|
.Locb_auth_blk1:
|
|
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
|
|
vpxor (%edx), %xmm7, %xmm0;
|
|
leal 16(%edx), %edx;
|
|
|
|
/* AES rounds. */
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_auth_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Locb_auth_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_auth_blk1_last:
|
|
vpxor (STACK_CHECKSUM)(%esp), %xmm1, %xmm1;
|
|
vaesenclast %xmm1, %xmm0, %xmm0;
|
|
vmovdqa %xmm0, (STACK_CHECKSUM)(%esp);
|
|
|
|
jmp .Locb_crypt_blk1;
|
|
|
|
.align 8
|
|
.Locb_dec_blk1:
|
|
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */
|
|
vpxor (%edx), %xmm7, %xmm0;
|
|
leal 16(%edx), %edx;
|
|
|
|
/* AES rounds. */
|
|
vaesdec (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (9 * 16)(%edi), %xmm0, %xmm0;
|
|
cmpl $12, 4+20(%ebp);
|
|
jb .Locb_dec_blk1_last;
|
|
vaesdec (10 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (11 * 16)(%edi), %xmm0, %xmm0;
|
|
jz .Locb_dec_blk1_last;
|
|
vaesdec (12 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (13 * 16)(%edi), %xmm0, %xmm0;
|
|
|
|
/* Last round and output handling. */
|
|
.Locb_dec_blk1_last:
|
|
vpxor (STACK_FXL_KEY)(%esp), %xmm7, %xmm1;
|
|
vaesdeclast %xmm1, %xmm0, %xmm0;
|
|
|
|
/* Checksum_i = Checksum_{i-1} xor P_i */
|
|
vpxor (STACK_CHECKSUM)(%esp), %xmm0, %xmm1;
|
|
|
|
vmovdqu %xmm0, (%esi);
|
|
leal 16(%esi), %esi;
|
|
|
|
vmovdqa %xmm1, (STACK_CHECKSUM)(%esp);
|
|
|
|
jmp .Locb_crypt_blk1;
|
|
|
|
.align 8
|
|
.Locb_crypt_done:
|
|
movl 4+24(%ebp), %ecx;
|
|
vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1;
|
|
vpxor (%edi), %xmm1, %xmm1;
|
|
vmovdqu %xmm1, (%ecx);
|
|
|
|
movl 4+28(%ebp), %eax;
|
|
vmovdqa (STACK_CHECKSUM)(%esp), %xmm2;
|
|
vpxor (STACK_CHECKSUM + 16)(%esp), %xmm2, %xmm2;
|
|
vpxor (%eax), %xmm2, %xmm2;
|
|
vmovdqu %xmm2, (%eax);
|
|
|
|
movl (STACK_GPR_POS + 0 * 4)(%esp), %edi;
|
|
CFI_RESTORE(edi);
|
|
movl (STACK_GPR_POS + 1 * 4)(%esp), %esi;
|
|
CFI_RESTORE(esi);
|
|
movl (STACK_GPR_POS + 2 * 4)(%esp), %ebx;
|
|
CFI_RESTORE(ebx);
|
|
|
|
vpxor %ymm0, %ymm0, %ymm0;
|
|
vmovdqa %ymm0, (STACK_OFFSET_AND_F_KEY)(%esp);
|
|
vmovdqa %ymm0, (STACK_CHECKSUM)(%esp);
|
|
|
|
xorl %eax, %eax;
|
|
leave;
|
|
CFI_LEAVE();
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386))
|
|
|
|
/**********************************************************************
|
|
XTS-mode encryption
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): tweak
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
* (esp + 28): encrypt
|
|
*/
|
|
CFI_STARTPROC();
|
|
|
|
GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax);
|
|
|
|
pushl %ebp;
|
|
CFI_PUSH(%ebp);
|
|
movl %esp, %ebp;
|
|
CFI_DEF_CFA_REGISTER(%ebp);
|
|
|
|
subl $(4 * 32 + 3 * 4), %esp;
|
|
andl $-32, %esp;
|
|
|
|
movl %edi, (4 * 32 + 0 * 4)(%esp);
|
|
CFI_REG_ON_STACK(edi, 4 * 32 + 0 * 4);
|
|
movl %esi, (4 * 32 + 1 * 4)(%esp);
|
|
CFI_REG_ON_STACK(esi, 4 * 32 + 1 * 4);
|
|
movl %ebx, (4 * 32 + 2 * 4)(%esp);
|
|
CFI_REG_ON_STACK(ebx, 4 * 32 + 2 * 4);
|
|
|
|
movl %eax, %ebx;
|
|
movl 4+4(%ebp), %edi;
|
|
movl 4+8(%ebp), %esi;
|
|
movl 4+12(%ebp), %edx;
|
|
movl 4+16(%ebp), %ecx;
|
|
movl 4+20(%ebp), %eax;
|
|
|
|
#define tweak_clmul(shift, out, tweak, hi_tweak, tmp1, tmp2) \
|
|
vpsrld $(32-(shift)), hi_tweak, tmp2; \
|
|
vpsllq $(shift), tweak, out; \
|
|
vpclmulqdq $0, CADDR(.Lxts_gfmul_clmul, %ebx), tmp2, tmp1; \
|
|
vpunpckhqdq tmp2, tmp1, tmp1; \
|
|
vpxor tmp1, out, out;
|
|
|
|
/* Prepare tweak. */
|
|
vmovdqu (%esi), %xmm7;
|
|
vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %xmm7, %xmm6;
|
|
tweak_clmul(1, %xmm5, %xmm7, %xmm6, %xmm0, %xmm1);
|
|
vinserti128 $1, %xmm5, %ymm7, %ymm7; /* tweak:tweak1 */
|
|
vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6;
|
|
|
|
/* Process eight blocks per loop. */
|
|
.align 8
|
|
.Lxts_crypt_blk8:
|
|
cmpl $8, %eax;
|
|
jb .Lxts_crypt_blk4;
|
|
|
|
leal -8(%eax), %eax;
|
|
|
|
vmovdqa %ymm7, (0 * 32)(%esp);
|
|
tweak_clmul(2, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
vmovdqa %ymm2, (1 * 32)(%esp);
|
|
tweak_clmul(4, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
vmovdqa %ymm2, (2 * 32)(%esp);
|
|
tweak_clmul(6, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
vmovdqa %ymm2, (3 * 32)(%esp);
|
|
tweak_clmul(8, %ymm7, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6;
|
|
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqa (0 * 32)(%esp), %ymm0;
|
|
vmovdqa (1 * 32)(%esp), %ymm1;
|
|
vmovdqa (2 * 32)(%esp), %ymm2;
|
|
vmovdqa (3 * 32)(%esp), %ymm3;
|
|
vpxor (0 * 16)(%ecx), %ymm0, %ymm0;
|
|
vpxor (2 * 16)(%ecx), %ymm1, %ymm1;
|
|
vpxor (4 * 16)(%ecx), %ymm2, %ymm2;
|
|
vpxor (6 * 16)(%ecx), %ymm3, %ymm3;
|
|
|
|
leal (8 * 16)(%ecx), %ecx;
|
|
|
|
cmpl $1, 4+28(%ebp);
|
|
jne .Lxts_dec_blk8;
|
|
/* AES rounds */
|
|
XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_enc_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lxts_enc_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_enc_blk8_last:
|
|
vpxor (0 * 32)(%esp), %ymm4, %ymm5; /* Xor tweak to last round key. */
|
|
vaesenclast %ymm5, %ymm0, %ymm0;
|
|
vpxor (1 * 32)(%esp), %ymm4, %ymm5;
|
|
vaesenclast %ymm5, %ymm1, %ymm1;
|
|
vpxor (2 * 32)(%esp), %ymm4, %ymm5;
|
|
vpxor (3 * 32)(%esp), %ymm4, %ymm4;
|
|
vaesenclast %ymm5, %ymm2, %ymm2;
|
|
vaesenclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
|
|
jmp .Lxts_crypt_blk8;
|
|
|
|
.align 8
|
|
.Lxts_dec_blk8:
|
|
/* AES rounds */
|
|
XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lxts_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_dec_blk8_last:
|
|
vpxor (0 * 32)(%esp), %ymm4, %ymm5; /* Xor tweak to last round key. */
|
|
vaesdeclast %ymm5, %ymm0, %ymm0;
|
|
vpxor (1 * 32)(%esp), %ymm4, %ymm5;
|
|
vaesdeclast %ymm5, %ymm1, %ymm1;
|
|
vpxor (2 * 32)(%esp), %ymm4, %ymm5;
|
|
vpxor (3 * 32)(%esp), %ymm4, %ymm4;
|
|
vaesdeclast %ymm5, %ymm2, %ymm2;
|
|
vaesdeclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
|
|
jmp .Lxts_crypt_blk8;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lxts_crypt_blk4:
|
|
/* Try exit early as typically input length is large power of 2. */
|
|
cmpl $1, %eax;
|
|
jb .Ldone_xts_crypt;
|
|
cmpl $4, %eax;
|
|
jb .Lxts_crypt_blk1;
|
|
|
|
leal -4(%eax), %eax;
|
|
|
|
vmovdqa %ymm7, %ymm2;
|
|
tweak_clmul(2, %ymm3, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
tweak_clmul(4, %ymm7, %ymm7, %ymm6, %ymm0, %ymm1);
|
|
vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6;
|
|
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vpxor (0 * 16)(%ecx), %ymm2, %ymm0;
|
|
vpxor (2 * 16)(%ecx), %ymm3, %ymm1;
|
|
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
|
|
cmpl $1, 4+28(%ebp);
|
|
jne .Lxts_dec_blk4;
|
|
/* AES rounds */
|
|
XOR2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lxts_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_enc_blk4_last:
|
|
vpxor %ymm4, %ymm2, %ymm2; /* Xor tweak to last round key. */
|
|
vpxor %ymm4, %ymm3, %ymm3;
|
|
vaesenclast %ymm2, %ymm0, %ymm0;
|
|
vaesenclast %ymm3, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
jmp .Lxts_crypt_blk1;
|
|
|
|
.align 8
|
|
.Lxts_dec_blk4:
|
|
/* AES rounds */
|
|
XOR2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lxts_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_dec_blk4_last:
|
|
vpxor %ymm4, %ymm2, %ymm2; /* Xor tweak to last round key. */
|
|
vpxor %ymm4, %ymm3, %ymm3;
|
|
vaesdeclast %ymm2, %ymm0, %ymm0;
|
|
vaesdeclast %ymm3, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lxts_crypt_blk1:
|
|
cmpl $1, %eax;
|
|
jb .Ldone_xts_crypt;
|
|
|
|
leal -1(%eax), %eax;
|
|
|
|
vpxor (%ecx), %xmm7, %xmm0;
|
|
vmovdqa %xmm7, %xmm5;
|
|
tweak_clmul(1, %xmm7, %xmm7, %xmm6, %xmm2, %xmm3);
|
|
vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %xmm7, %xmm6;
|
|
|
|
leal 16(%ecx), %ecx;
|
|
|
|
cmpl $1, 4+28(%ebp);
|
|
jne .Lxts_dec_blk1;
|
|
/* AES rounds. */
|
|
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lxts_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_enc_blk1_last:
|
|
vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */
|
|
vaesenclast %xmm5, %xmm0, %xmm0;
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lxts_crypt_blk1;
|
|
|
|
.align 8
|
|
.Lxts_dec_blk1:
|
|
/* AES rounds. */
|
|
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 4+24(%ebp);
|
|
jb .Lxts_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lxts_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
|
|
/* Last round and output handling. */
|
|
.Lxts_dec_blk1_last:
|
|
vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */
|
|
vaesdeclast %xmm5, %xmm0, %xmm0;
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lxts_crypt_blk1;
|
|
|
|
.align 8
|
|
.Ldone_xts_crypt:
|
|
/* Store IV. */
|
|
vmovdqu %xmm7, (%esi);
|
|
|
|
vpxor %ymm0, %ymm0, %ymm0;
|
|
movl (4 * 32 + 0 * 4)(%esp), %edi;
|
|
CFI_RESTORE(edi);
|
|
movl (4 * 32 + 1 * 4)(%esp), %esi;
|
|
CFI_RESTORE(esi);
|
|
movl (4 * 32 + 2 * 4)(%esp), %ebx;
|
|
CFI_RESTORE(ebx);
|
|
vmovdqa %ymm0, (0 * 32)(%esp);
|
|
vmovdqa %ymm0, (1 * 32)(%esp);
|
|
vmovdqa %ymm0, (2 * 32)(%esp);
|
|
vmovdqa %ymm0, (3 * 32)(%esp);
|
|
leave;
|
|
CFI_LEAVE();
|
|
vzeroall;
|
|
xorl %eax, %eax;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386))
|
|
|
|
/**********************************************************************
|
|
ECB-mode encryption
|
|
**********************************************************************/
|
|
ELF(.type SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386),@function)
|
|
.globl SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386)
|
|
.align 16
|
|
SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386):
|
|
/* input:
|
|
* (esp + 4): round keys
|
|
* (esp + 8): encrypt
|
|
* (esp + 12): dst
|
|
* (esp + 16): src
|
|
* (esp + 20): nblocks
|
|
* (esp + 24): nrounds
|
|
*/
|
|
CFI_STARTPROC();
|
|
pushl %edi;
|
|
CFI_PUSH(%edi);
|
|
pushl %esi;
|
|
CFI_PUSH(%esi);
|
|
|
|
movl 8+4(%esp), %edi;
|
|
movl 8+8(%esp), %esi;
|
|
movl 8+12(%esp), %edx;
|
|
movl 8+16(%esp), %ecx;
|
|
movl 8+20(%esp), %eax;
|
|
|
|
/* Process 8 blocks per loop. */
|
|
.align 8
|
|
.Lecb_blk8:
|
|
cmpl $8, %eax;
|
|
jb .Lecb_blk4;
|
|
|
|
leal -8(%eax), %eax;
|
|
|
|
/* Load input and xor first key. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm0;
|
|
vmovdqu (2 * 16)(%ecx), %ymm1;
|
|
vmovdqu (4 * 16)(%ecx), %ymm2;
|
|
vmovdqu (6 * 16)(%ecx), %ymm3;
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vpxor %ymm4, %ymm2, %ymm2;
|
|
vpxor %ymm4, %ymm3, %ymm3;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
leal (8 * 16)(%ecx), %ecx;
|
|
|
|
testl %esi, %esi;
|
|
jz .Lecb_dec_blk8;
|
|
/* AES rounds */
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_enc_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lecb_enc_blk8_last;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
.Lecb_enc_blk8_last:
|
|
vaesenclast %ymm4, %ymm0, %ymm0;
|
|
vaesenclast %ymm4, %ymm1, %ymm1;
|
|
vaesenclast %ymm4, %ymm2, %ymm2;
|
|
vaesenclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
jmp .Lecb_blk8;
|
|
|
|
.align 8
|
|
.Lecb_dec_blk8:
|
|
/* AES rounds */
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lecb_dec_blk8_last;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
.Lecb_dec_blk8_last:
|
|
vaesdeclast %ymm4, %ymm0, %ymm0;
|
|
vaesdeclast %ymm4, %ymm1, %ymm1;
|
|
vaesdeclast %ymm4, %ymm2, %ymm2;
|
|
vaesdeclast %ymm4, %ymm3, %ymm3;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
vmovdqu %ymm2, (4 * 16)(%edx);
|
|
vmovdqu %ymm3, (6 * 16)(%edx);
|
|
leal (8 * 16)(%edx), %edx;
|
|
jmp .Lecb_blk8;
|
|
|
|
/* Handle trailing four blocks. */
|
|
.align 8
|
|
.Lecb_blk4:
|
|
cmpl $4, %eax;
|
|
jb .Lecb_blk1;
|
|
|
|
leal -4(%eax), %eax;
|
|
|
|
/* Load input and xor first key. */
|
|
vbroadcasti128 (0 * 16)(%edi), %ymm4;
|
|
vmovdqu (0 * 16)(%ecx), %ymm0;
|
|
vmovdqu (2 * 16)(%ecx), %ymm1;
|
|
vpxor %ymm4, %ymm0, %ymm0;
|
|
vpxor %ymm4, %ymm1, %ymm1;
|
|
vbroadcasti128 (1 * 16)(%edi), %ymm4;
|
|
leal (4 * 16)(%ecx), %ecx;
|
|
|
|
testl %esi, %esi;
|
|
jz .Lecb_dec_blk4;
|
|
/* AES rounds */
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lecb_enc_blk4_last;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESENC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
.Lecb_enc_blk4_last:
|
|
vaesenclast %ymm4, %ymm0, %ymm0;
|
|
vaesenclast %ymm4, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
jmp .Lecb_blk1;
|
|
|
|
.align 8
|
|
.Lecb_dec_blk4:
|
|
/* AES rounds */
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (2 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (3 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (4 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (5 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (6 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (7 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (8 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (9 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (10 * 16)(%edi), %ymm4;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (11 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (12 * 16)(%edi), %ymm4;
|
|
jz .Lecb_dec_blk4_last;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (13 * 16)(%edi), %ymm4;
|
|
VAESDEC2(%ymm4, %ymm0, %ymm1);
|
|
vbroadcasti128 (14 * 16)(%edi), %ymm4;
|
|
.Lecb_dec_blk4_last:
|
|
vaesdeclast %ymm4, %ymm0, %ymm0;
|
|
vaesdeclast %ymm4, %ymm1, %ymm1;
|
|
vmovdqu %ymm0, (0 * 16)(%edx);
|
|
vmovdqu %ymm1, (2 * 16)(%edx);
|
|
leal (4 * 16)(%edx), %edx;
|
|
|
|
/* Process trailing one to three blocks, one per loop. */
|
|
.align 8
|
|
.Lecb_blk1:
|
|
cmpl $1, %eax;
|
|
jb .Ldone_ecb;
|
|
|
|
leal -1(%eax), %eax;
|
|
|
|
/* Load input. */
|
|
vmovdqu (%ecx), %xmm2;
|
|
leal 16(%ecx), %ecx;
|
|
|
|
/* Xor first key. */
|
|
vpxor (0 * 16)(%edi), %xmm2, %xmm0;
|
|
|
|
testl %esi, %esi;
|
|
jz .Lecb_dec_blk1;
|
|
/* AES rounds. */
|
|
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesenc (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lecb_enc_blk1_last;
|
|
vaesenc %xmm1, %xmm0, %xmm0;
|
|
vaesenc (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
.Lecb_enc_blk1_last:
|
|
vaesenclast %xmm1, %xmm0, %xmm0;
|
|
jmp .Lecb_blk1_end;
|
|
|
|
.align 8
|
|
.Lecb_dec_blk1:
|
|
/* AES rounds. */
|
|
vaesdec (1 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (2 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (3 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (4 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (5 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (6 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (7 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (8 * 16)(%edi), %xmm0, %xmm0;
|
|
vaesdec (9 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (10 * 16)(%edi), %xmm1;
|
|
cmpl $12, 8+24(%esp);
|
|
jb .Lecb_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (11 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (12 * 16)(%edi), %xmm1;
|
|
jz .Lecb_dec_blk1_last;
|
|
vaesdec %xmm1, %xmm0, %xmm0;
|
|
vaesdec (13 * 16)(%edi), %xmm0, %xmm0;
|
|
vmovdqa (14 * 16)(%edi), %xmm1;
|
|
.Lecb_dec_blk1_last:
|
|
vaesdeclast %xmm1, %xmm0, %xmm0;
|
|
jmp .Lecb_blk1_end;
|
|
|
|
.align 8
|
|
.Lecb_blk1_end:
|
|
vmovdqu %xmm0, (%edx);
|
|
leal 16(%edx), %edx;
|
|
|
|
jmp .Lecb_blk1;
|
|
|
|
.align 8
|
|
.Ldone_ecb:
|
|
popl %esi;
|
|
CFI_POP(%esi);
|
|
popl %edi;
|
|
CFI_POP(%edi);
|
|
vzeroall;
|
|
ret_spec_stop
|
|
CFI_ENDPROC();
|
|
ELF(.size SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386),
|
|
.-SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386))
|
|
|
|
/**********************************************************************
|
|
constants
|
|
**********************************************************************/
|
|
SECTION_RODATA
|
|
|
|
ELF(.type SYM_NAME(_gcry_vaes_consts),@object)
|
|
.align 32
|
|
SYM_NAME(_gcry_vaes_consts):
|
|
.Lbige_addb_0:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lbige_addb_1:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
|
|
.Lbige_addb_2:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
|
|
.Lbige_addb_3:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
|
|
.Lbige_addb_4:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
|
|
.Lbige_addb_5:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
|
|
.Lbige_addb_6:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
|
|
.Lbige_addb_7:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
|
|
.Lbige_addb_8:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
|
|
.Lbige_addb_9:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
|
|
.Lbige_addb_10:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
|
|
.Lbige_addb_11:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
|
|
|
|
.Lle_addd_0:
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_1:
|
|
.byte 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_2:
|
|
.byte 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_3:
|
|
.byte 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_4:
|
|
.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_5:
|
|
.byte 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_6:
|
|
.byte 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_7:
|
|
.byte 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_8:
|
|
.byte 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_9:
|
|
.byte 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_10:
|
|
.byte 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_11:
|
|
.byte 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
|
|
.Lle_addd_4_2:
|
|
.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.Lle_addd_12_2:
|
|
.byte 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
.byte 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
|
|
.Lxts_gfmul_clmul:
|
|
.long 0x00, 0x87, 0x00, 0x00
|
|
.long 0x00, 0x87, 0x00, 0x00
|
|
.Lxts_high_bit_shuf:
|
|
.byte -1, -1, -1, -1, 12, 13, 14, 15
|
|
.byte 4, 5, 6, 7, -1, -1, -1, -1
|
|
.byte -1, -1, -1, -1, 12, 13, 14, 15
|
|
.byte 4, 5, 6, 7, -1, -1, -1, -1
|
|
.Lbswap128_mask:
|
|
.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
|
|
.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
|
|
|
|
ELF(.size SYM_NAME(_gcry_vaes_consts),.-SYM_NAME(_gcry_vaes_consts))
|
|
|
|
#endif /* HAVE_GCC_INLINE_ASM_VAES */
|
|
#endif /* __i386__ */
|