/* VAES/AVX2 i386 accelerated AES for Libgcrypt * Copyright (C) 2023 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #if defined(__i386__) #include #if (defined(HAVE_COMPATIBLE_GCC_I386_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN32_PLATFORM_AS)) && \ defined(ENABLE_AESNI_SUPPORT) && defined(ENABLE_AVX2_SUPPORT) && \ defined(HAVE_GCC_INLINE_ASM_VAES_VPCLMUL) #include "asm-common-i386.h" .text DECL_GET_PC_THUNK(eax); /********************************************************************** helper macros **********************************************************************/ #define AES_OP4(op, key, b0, b1, b2, b3) \ op key, b0, b0; \ op key, b1, b1; \ op key, b2, b2; \ op key, b3, b3; #define VAESENC4(key, b0, b1, b2, b3) \ AES_OP4(vaesenc, key, b0, b1, b2, b3) #define VAESDEC4(key, b0, b1, b2, b3) \ AES_OP4(vaesdec, key, b0, b1, b2, b3) #define XOR4(key, b0, b1, b2, b3) \ AES_OP4(vpxor, key, b0, b1, b2, b3) #define AES_OP2(op, key, b0, b1) \ op key, b0, b0; \ op key, b1, b1; #define VAESENC2(key, b0, b1) \ AES_OP2(vaesenc, key, b0, b1) #define VAESDEC2(key, b0, b1) \ AES_OP2(vaesdec, key, b0, b1) #define XOR2(key, b0, b1) \ AES_OP2(vpxor, key, b0, b1) #define VAESENC6(key, b0, b1, b2, b3, b4, b5) \ AES_OP4(vaesenc, key, b0, b1, b2, b3); \ AES_OP2(vaesenc, key, b4, b5) #define VAESDEC6(key, b0, b1, b2, b3, b4, b5) \ AES_OP4(vaesdec, key, b0, b1, b2, b3); \ AES_OP2(vaesdec, key, b4, b5) #define XOR6(key, b0, b1, b2, b3, b4, b5) \ AES_OP4(vpxor, key, b0, b1, b2, b3); \ AES_OP2(vpxor, key, b4, b5) #define CADDR(name, reg) \ (name - SYM_NAME(_gcry_vaes_consts))(reg) /********************************************************************** CBC-mode decryption **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386): /* input: * (esp + 4): round keys * (esp + 8): iv * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds */ CFI_STARTPROC(); pushl %edi; CFI_PUSH(%edi); pushl %esi; CFI_PUSH(%esi); movl 8+4(%esp), %edi; movl 8+8(%esp), %esi; movl 8+12(%esp), %edx; movl 8+16(%esp), %ecx; movl 8+20(%esp), %eax; /* Process 8 blocks per loop. */ .align 8 .Lcbc_dec_blk8: cmpl $8, %eax; jb .Lcbc_dec_blk4; leal -8(%eax), %eax; /* Load input and xor first key. Update IV. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm0; vmovdqu (2 * 16)(%ecx), %ymm1; vmovdqu (4 * 16)(%ecx), %ymm2; vmovdqu (6 * 16)(%ecx), %ymm3; vmovdqu (%esi), %xmm6; /* Load IV. */ vinserti128 $1, %xmm0, %ymm6, %ymm5; vextracti128 $1, %ymm3, (%esi); /* Store IV. */ vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vpxor %ymm4, %ymm2, %ymm2; vpxor %ymm4, %ymm3, %ymm3; vmovdqu (1 * 16)(%ecx), %ymm6; vmovdqu (3 * 16)(%ecx), %ymm7; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lcbc_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lcbc_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lcbc_dec_blk8_last: vpxor %ymm4, %ymm5, %ymm5; vpxor %ymm4, %ymm6, %ymm6; vpxor %ymm4, %ymm7, %ymm7; vpxor (5 * 16)(%ecx), %ymm4, %ymm4; leal (8 * 16)(%ecx), %ecx; vaesdeclast %ymm5, %ymm0, %ymm0; vaesdeclast %ymm6, %ymm1, %ymm1; vaesdeclast %ymm7, %ymm2, %ymm2; vaesdeclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lcbc_dec_blk8; /* Handle trailing four blocks. */ .align 8 .Lcbc_dec_blk4: cmpl $4, %eax; jb .Lcbc_dec_blk1; leal -4(%eax), %eax; /* Load input and xor first key. Update IV. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm0; vmovdqu (2 * 16)(%ecx), %ymm1; vmovdqu (%esi), %xmm6; /* Load IV. */ vinserti128 $1, %xmm0, %ymm6, %ymm5; vextracti128 $1, %ymm1, (%esi); /* Store IV. */ vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vmovdqu (1 * 16)(%ecx), %ymm6; leal (4 * 16)(%ecx), %ecx; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lcbc_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lcbc_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lcbc_dec_blk4_last: vpxor %ymm4, %ymm5, %ymm5; vpxor %ymm4, %ymm6, %ymm6; vaesdeclast %ymm5, %ymm0, %ymm0; vaesdeclast %ymm6, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lcbc_dec_blk1: cmpl $1, %eax; jb .Ldone_cbc_dec; leal -1(%eax), %eax; /* Load input. */ vmovdqu (%ecx), %xmm2; leal 16(%ecx), %ecx; /* Xor first key. */ vpxor (0 * 16)(%edi), %xmm2, %xmm0; /* AES rounds. */ vaesdec (1 * 16)(%edi), %xmm0, %xmm0; vaesdec (2 * 16)(%edi), %xmm0, %xmm0; vaesdec (3 * 16)(%edi), %xmm0, %xmm0; vaesdec (4 * 16)(%edi), %xmm0, %xmm0; vaesdec (5 * 16)(%edi), %xmm0, %xmm0; vaesdec (6 * 16)(%edi), %xmm0, %xmm0; vaesdec (7 * 16)(%edi), %xmm0, %xmm0; vaesdec (8 * 16)(%edi), %xmm0, %xmm0; vaesdec (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 8+24(%esp); jb .Lcbc_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lcbc_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lcbc_dec_blk1_last: vpxor (%esi), %xmm1, %xmm1; vaesdeclast %xmm1, %xmm0, %xmm0; vmovdqu %xmm2, (%esi); vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lcbc_dec_blk1; .align 8 .Ldone_cbc_dec: popl %esi; CFI_POP(%esi); popl %edi; CFI_POP(%edi); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386), .-SYM_NAME(_gcry_vaes_avx2_cbc_dec_i386)) /********************************************************************** CFB-mode decryption **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386): /* input: * (esp + 4): round keys * (esp + 8): iv * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds */ CFI_STARTPROC(); pushl %edi; CFI_PUSH(%edi); pushl %esi; CFI_PUSH(%esi); movl 8+4(%esp), %edi; movl 8+8(%esp), %esi; movl 8+12(%esp), %edx; movl 8+16(%esp), %ecx; movl 8+20(%esp), %eax; /* Process 8 blocks per loop. */ .align 8 .Lcfb_dec_blk8: cmpl $8, %eax; jb .Lcfb_dec_blk4; leal -8(%eax), %eax; /* Load IV. */ vmovdqu (%esi), %xmm0; /* Load input and xor first key. Update IV. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm5; vinserti128 $1, %xmm5, %ymm0, %ymm0; vmovdqu (1 * 16)(%ecx), %ymm1; vmovdqu (3 * 16)(%ecx), %ymm2; vmovdqu (5 * 16)(%ecx), %ymm3; vmovdqu (7 * 16)(%ecx), %xmm6; vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vpxor %ymm4, %ymm2, %ymm2; vpxor %ymm4, %ymm3, %ymm3; vbroadcasti128 (1 * 16)(%edi), %ymm4; vmovdqu %xmm6, (%esi); /* Store IV. */ vmovdqu (2 * 16)(%ecx), %ymm6; vmovdqu (4 * 16)(%ecx), %ymm7; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lcfb_dec_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lcfb_dec_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lcfb_dec_blk8_last: vpxor %ymm4, %ymm5, %ymm5; vpxor %ymm4, %ymm6, %ymm6; vpxor %ymm4, %ymm7, %ymm7; vpxor (6 * 16)(%ecx), %ymm4, %ymm4; leal (8 * 16)(%ecx), %ecx; vaesenclast %ymm5, %ymm0, %ymm0; vaesenclast %ymm6, %ymm1, %ymm1; vaesenclast %ymm7, %ymm2, %ymm2; vaesenclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lcfb_dec_blk8; /* Handle trailing four blocks. */ .align 8 .Lcfb_dec_blk4: cmpl $4, %eax; jb .Lcfb_dec_blk1; leal -4(%eax), %eax; /* Load IV. */ vmovdqu (%esi), %xmm0; /* Load input and xor first key. Update IV. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm5; vinserti128 $1, %xmm5, %ymm0, %ymm0; vmovdqu (1 * 16)(%ecx), %ymm1; vmovdqu (3 * 16)(%ecx), %xmm6; vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vbroadcasti128 (1 * 16)(%edi), %ymm4; vmovdqu %xmm6, (%esi); /* Store IV. */ vmovdqu (2 * 16)(%ecx), %ymm6; leal (4 * 16)(%ecx), %ecx; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lcfb_dec_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lcfb_dec_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lcfb_dec_blk4_last: vpxor %ymm4, %ymm5, %ymm5; vpxor %ymm4, %ymm6, %ymm6; vaesenclast %ymm5, %ymm0, %ymm0; vaesenclast %ymm6, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lcfb_dec_blk1: cmpl $1, %eax; jb .Ldone_cfb_dec; leal -1(%eax), %eax; /* Load IV. */ vmovdqu (%esi), %xmm0; /* Xor first key. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; /* Load input as next IV. */ vmovdqu (%ecx), %xmm2; leal 16(%ecx), %ecx; /* AES rounds. */ vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; vmovdqu %xmm2, (%esi); /* Store IV. */ cmpl $12, 8+24(%esp); jb .Lcfb_dec_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lcfb_dec_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lcfb_dec_blk1_last: vpxor %xmm2, %xmm1, %xmm1; vaesenclast %xmm1, %xmm0, %xmm0; vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lcfb_dec_blk1; .align 8 .Ldone_cfb_dec: popl %esi; CFI_POP(%esi); popl %edi; CFI_POP(%edi); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386), .-SYM_NAME(_gcry_vaes_avx2_cfb_dec_i386)) /********************************************************************** CTR-mode encryption **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): /* input: * (esp + 4): round keys * (esp + 8): iv * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds */ CFI_STARTPROC(); GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax); pushl %ebp; CFI_PUSH(%ebp); movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); subl $(3 * 32 + 3 * 4), %esp; andl $-32, %esp; movl %edi, (3 * 32 + 0 * 4)(%esp); CFI_REG_ON_STACK(edi, 3 * 32 + 0 * 4); movl %esi, (3 * 32 + 1 * 4)(%esp); CFI_REG_ON_STACK(esi, 3 * 32 + 1 * 4); movl %ebx, (3 * 32 + 2 * 4)(%esp); CFI_REG_ON_STACK(ebx, 3 * 32 + 2 * 4); movl %eax, %ebx; movl 4+4(%ebp), %edi; movl 4+8(%ebp), %esi; movl 4+12(%ebp), %edx; movl 4+16(%ebp), %ecx; #define prepare_ctr_const(minus_one, minus_two) \ vpcmpeqd minus_one, minus_one, minus_one; \ vpsrldq $8, minus_one, minus_one; /* 0:-1 */ \ vpaddq minus_one, minus_one, minus_two; /* 0:-2 */ #define inc_le128(x, minus_one, tmp) \ vpcmpeqq minus_one, x, tmp; \ vpsubq minus_one, x, x; \ vpslldq $8, tmp, tmp; \ vpsubq tmp, x, x; #define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ vpcmpeqq minus_one, x, tmp1; \ vpcmpeqq minus_two, x, tmp2; \ vpor tmp1, tmp2, tmp2; \ vpsubq minus_two, x, x; \ vpslldq $8, tmp2, tmp2; \ vpsubq tmp2, x, x; #define handle_ctr_128bit_add(nblks) \ movl 12(%esi), %eax; \ bswapl %eax; \ addl $nblks, %eax; \ bswapl %eax; \ movl %eax, 12(%esi); \ jnc 1f; \ \ movl 8(%esi), %eax; \ bswapl %eax; \ adcl $0, %eax; \ bswapl %eax; \ movl %eax, 8(%esi); \ \ movl 4(%esi), %eax; \ bswapl %eax; \ adcl $0, %eax; \ bswapl %eax; \ movl %eax, 4(%esi); \ \ movl 0(%esi), %eax; \ bswapl %eax; \ adcl $0, %eax; \ bswapl %eax; \ movl %eax, 0(%esi); \ .align 8; \ 1:; cmpl $12, 4+20(%ebp); jae .Lctr_enc_blk12_loop; jmp .Lctr_enc_blk4; /* Process 12 blocks per loop. */ .align 16 .Lctr_enc_blk12_loop: subl $12, 4+20(%ebp); vbroadcasti128 (%esi), %ymm6; /* detect if carry handling is needed */ movl 12(%esi), %eax; addl $(12 << 24), %eax; jc .Lctr_enc_blk12_handle_carry; movl %eax, 12(%esi); .Lctr_enc_blk12_byte_bige_add: /* Increment counters. */ vpaddb CADDR(.Lbige_addb_0, %ebx), %ymm6, %ymm0; vpaddb CADDR(.Lbige_addb_2, %ebx), %ymm6, %ymm1; vpaddb CADDR(.Lbige_addb_4, %ebx), %ymm6, %ymm2; vpaddb CADDR(.Lbige_addb_6, %ebx), %ymm6, %ymm3; vpaddb CADDR(.Lbige_addb_8, %ebx), %ymm6, %ymm5; vpaddb CADDR(.Lbige_addb_10, %ebx), %ymm6, %ymm6; .Lctr_enc_blk12_rounds: /* AES rounds */ vbroadcasti128 (0 * 16)(%edi), %ymm4; XOR6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lctr_enc_blk12_last; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lctr_enc_blk12_last; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lctr_enc_blk12_last: vpxor (0 * 16)(%ecx), %ymm4, %ymm7; /* Xor src to last round key. */ vaesenclast %ymm7, %ymm0, %ymm0; vmovdqu %ymm0, (0 * 16)(%edx); vpxor (2 * 16)(%ecx), %ymm4, %ymm7; vpxor (4 * 16)(%ecx), %ymm4, %ymm0; vaesenclast %ymm7, %ymm1, %ymm1; vaesenclast %ymm0, %ymm2, %ymm2; vpxor (6 * 16)(%ecx), %ymm4, %ymm7; vpxor (8 * 16)(%ecx), %ymm4, %ymm0; vpxor (10 * 16)(%ecx), %ymm4, %ymm4; leal (12 * 16)(%ecx), %ecx; vaesenclast %ymm7, %ymm3, %ymm3; vaesenclast %ymm0, %ymm5, %ymm5; vaesenclast %ymm4, %ymm6, %ymm6; vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); vmovdqu %ymm5, (8 * 16)(%edx); vmovdqu %ymm6, (10 * 16)(%edx); leal (12 * 16)(%edx), %edx; cmpl $12, 4+20(%ebp); jae .Lctr_enc_blk12_loop; jmp .Lctr_enc_blk4; .align 8 .Lctr_enc_blk12_handle_only_ctr_carry: handle_ctr_128bit_add(12); jmp .Lctr_enc_blk12_byte_bige_add; .align 8 .Lctr_enc_blk12_handle_carry: jz .Lctr_enc_blk12_handle_only_ctr_carry; /* Increment counters (handle carry). */ prepare_ctr_const(%ymm4, %ymm7); vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2; vpshufb %xmm2, %xmm6, %xmm1; /* be => le */ vmovdqa %xmm1, %xmm0; inc_le128(%xmm1, %xmm4, %xmm5); vinserti128 $1, %xmm1, %ymm0, %ymm6; /* ctr: +1:+0 */ handle_ctr_128bit_add(12); vpshufb %ymm2, %ymm6, %ymm0; vmovdqa %ymm0, (0 * 32)(%esp); add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +3:+2 */ vpshufb %ymm2, %ymm6, %ymm0; vmovdqa %ymm0, (1 * 32)(%esp); add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +5:+4 */ vpshufb %ymm2, %ymm6, %ymm0; vmovdqa %ymm0, (2 * 32)(%esp); add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +7:+6 */ vpshufb %ymm2, %ymm6, %ymm3; add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +9:+8 */ vpshufb %ymm2, %ymm6, %ymm5; add2_le128(%ymm6, %ymm4, %ymm7, %ymm2, %ymm1); /* ctr: +11:+10 */ vmovdqa (0 * 32)(%esp), %ymm0; vmovdqa (1 * 32)(%esp), %ymm1; vmovdqa (2 * 32)(%esp), %ymm2; vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm6, %ymm6; jmp .Lctr_enc_blk12_rounds; /* Handle trailing four blocks. */ .align 8 .Lctr_enc_blk4: cmpl $4, 4+20(%ebp); jb .Lctr_enc_blk1; subl $4, 4+20(%ebp); vbroadcasti128 (%esi), %ymm3; /* detect if carry handling is needed */ movl 12(%esi), %eax; addl $(4 << 24), %eax; jc .Lctr_enc_blk4_handle_carry; movl %eax, 12(%esi); .Lctr_enc_blk4_byte_bige_add: /* Increment counters. */ vpaddb CADDR(.Lbige_addb_0, %ebx), %ymm3, %ymm0; vpaddb CADDR(.Lbige_addb_2, %ebx), %ymm3, %ymm1; .Lctr_enc_blk4_rounds: /* AES rounds */ vbroadcasti128 (0 * 16)(%edi), %ymm4; XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lctr_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lctr_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lctr_enc_blk4_last: vpxor (0 * 16)(%ecx), %ymm4, %ymm5; /* Xor src to last round key. */ vpxor (2 * 16)(%ecx), %ymm4, %ymm6; leal (4 * 16)(%ecx), %ecx; vaesenclast %ymm5, %ymm0, %ymm0; vaesenclast %ymm6, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; jmp .Lctr_enc_blk1; .align 8 .Lctr_enc_blk4_handle_only_ctr_carry: handle_ctr_128bit_add(4); jmp .Lctr_enc_blk4_byte_bige_add; .align 8 .Lctr_enc_blk4_handle_carry: jz .Lctr_enc_blk4_handle_only_ctr_carry; /* Increment counters (handle carry). */ prepare_ctr_const(%ymm4, %ymm7); vpshufb CADDR(.Lbswap128_mask, %ebx), %xmm3, %xmm1; /* be => le */ vmovdqa %xmm1, %xmm0; inc_le128(%xmm1, %xmm4, %xmm5); vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm0; handle_ctr_128bit_add(4); add2_le128(%ymm3, %ymm4, %ymm7, %ymm5, %ymm6); /* ctr: +3:+2 */ vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm1; jmp .Lctr_enc_blk4_rounds; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lctr_enc_blk1: cmpl $1, 4+20(%ebp); jb .Ldone_ctr_enc; subl $1, 4+20(%ebp); /* Load and increament counter. */ vmovdqu (%esi), %xmm0; handle_ctr_128bit_add(1); /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 4+24(%ebp); jb .Lctr_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lctr_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lctr_enc_blk1_last: vpxor (%ecx), %xmm1, %xmm1; /* Xor src to last round key. */ leal 16(%ecx), %ecx; vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */ vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lctr_enc_blk1; .align 8 .Ldone_ctr_enc: vpxor %ymm0, %ymm0, %ymm0; movl (3 * 32 + 0 * 4)(%esp), %edi; CFI_RESTORE(edi); movl (3 * 32 + 1 * 4)(%esp), %esi; CFI_RESTORE(esi); movl (3 * 32 + 2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); vmovdqa %ymm0, (0 * 32)(%esp); vmovdqa %ymm0, (1 * 32)(%esp); vmovdqa %ymm0, (2 * 32)(%esp); leave; CFI_LEAVE(); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386), .-SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386)) /********************************************************************** Little-endian 32-bit CTR-mode encryption (GCM-SIV) **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386): /* input: * (esp + 4): round keys * (esp + 8): counter * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds */ CFI_STARTPROC(); GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax); pushl %ebp; CFI_PUSH(%ebp); movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); subl $(3 * 4), %esp; movl %edi, (0 * 4)(%esp); CFI_REG_ON_STACK(edi, 0 * 4); movl %esi, (1 * 4)(%esp); CFI_REG_ON_STACK(esi, 1 * 4); movl %ebx, (2 * 4)(%esp); CFI_REG_ON_STACK(ebx, 2 * 4); movl %eax, %ebx; movl 4+4(%ebp), %edi; movl 4+8(%ebp), %esi; movl 4+12(%ebp), %edx; movl 4+16(%ebp), %ecx; movl 4+20(%ebp), %eax; vbroadcasti128 (%esi), %ymm7; /* Load CTR. */ /* Process 12 blocks per loop. */ .align 8 .Lctr32le_enc_blk12: cmpl $12, %eax; jb .Lctr32le_enc_blk4; leal -12(%eax), %eax; vbroadcasti128 (0 * 16)(%edi), %ymm4; /* Increment counters. */ vpaddd CADDR(.Lle_addd_0, %ebx), %ymm7, %ymm0; vpaddd CADDR(.Lle_addd_2, %ebx), %ymm7, %ymm1; vpaddd CADDR(.Lle_addd_4, %ebx), %ymm7, %ymm2; vpaddd CADDR(.Lle_addd_6, %ebx), %ymm7, %ymm3; vpaddd CADDR(.Lle_addd_8, %ebx), %ymm7, %ymm5; vpaddd CADDR(.Lle_addd_10, %ebx), %ymm7, %ymm6; vpaddd CADDR(.Lle_addd_12_2, %ebx), %ymm7, %ymm7; vmovdqu %xmm7, (%esi); /* Store CTR. */ /* AES rounds */ XOR6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lctr32le_enc_blk8_last; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lctr32le_enc_blk8_last; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC6(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3, %ymm5, %ymm6); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lctr32le_enc_blk8_last: vpxor (0 * 16)(%ecx), %ymm4, %ymm7; /* Xor src to last round key. */ vaesenclast %ymm7, %ymm0, %ymm0; vpxor (2 * 16)(%ecx), %ymm4, %ymm7; vaesenclast %ymm7, %ymm1, %ymm1; vpxor (4 * 16)(%ecx), %ymm4, %ymm7; vaesenclast %ymm7, %ymm2, %ymm2; vpxor (6 * 16)(%ecx), %ymm4, %ymm7; vaesenclast %ymm7, %ymm3, %ymm3; vpxor (8 * 16)(%ecx), %ymm4, %ymm7; vpxor (10 * 16)(%ecx), %ymm4, %ymm4; vaesenclast %ymm7, %ymm5, %ymm5; vbroadcasti128 (%esi), %ymm7; /* Reload CTR. */ vaesenclast %ymm4, %ymm6, %ymm6; leal (12 * 16)(%ecx), %ecx; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); vmovdqu %ymm5, (8 * 16)(%edx); vmovdqu %ymm6, (10 * 16)(%edx); leal (12 * 16)(%edx), %edx; jmp .Lctr32le_enc_blk12; /* Handle trailing four blocks. */ .align 8 .Lctr32le_enc_blk4: cmpl $4, %eax; jb .Lctr32le_enc_blk1; leal -4(%eax), %eax; vbroadcasti128 (0 * 16)(%edi), %ymm4; /* Increment counters. */ vpaddd CADDR(.Lle_addd_0, %ebx), %ymm7, %ymm0; vpaddd CADDR(.Lle_addd_2, %ebx), %ymm7, %ymm1; vpaddd CADDR(.Lle_addd_4_2, %ebx), %ymm7, %ymm7; /* AES rounds */ XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lctr32le_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lctr32le_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lctr32le_enc_blk4_last: vpxor (0 * 16)(%ecx), %ymm4, %ymm5; /* Xor src to last round key. */ vpxor (2 * 16)(%ecx), %ymm4, %ymm6; leal (4 * 16)(%ecx), %ecx; vaesenclast %ymm5, %ymm0, %ymm0; vaesenclast %ymm6, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lctr32le_enc_blk1: cmpl $1, %eax; jb .Ldone_ctr32le_enc; leal -1(%eax), %eax; /* Load and increament counter. */ vmovdqu %xmm7, %xmm0; vpaddd CADDR(.Lle_addd_1, %ebx), %xmm7, %xmm7; /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 4+24(%ebp); jb .Lctr32le_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lctr32le_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lctr32le_enc_blk1_last: vpxor (%ecx), %xmm1, %xmm1; /* Xor src to last round key. */ leal 16(%ecx), %ecx; vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */ vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lctr32le_enc_blk1; .align 8 .Ldone_ctr32le_enc: vmovdqu %xmm7, (%esi); /* Store CTR. */ movl (0 * 4)(%esp), %edi; CFI_RESTORE(edi); movl (1 * 4)(%esp), %esi; CFI_RESTORE(esi); movl (2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); leave; CFI_LEAVE(); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386), .-SYM_NAME(_gcry_vaes_avx2_ctr32le_enc_i386)) /********************************************************************** OCB-mode encryption/decryption/authentication **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386): /* input: * (esp + 4): round keys * (esp + 8): dst * (esp + 12): src * (esp + 16): nblocks * (esp + 20): nrounds * (esp + 24): offset * (esp + 28): checksum * (esp + 32): blkn * (esp + 36): L table * (esp + 44): encrypt/decrypt/auth mode */ CFI_STARTPROC(); pushl %ebp; CFI_PUSH(%ebp); movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); #define STACK_VEC_POS 0 #define STACK_TMP_Y0 (STACK_VEC_POS + 0 * 32) #define STACK_TMP_Y1 (STACK_VEC_POS + 1 * 32) #define STACK_TMP_Y2 (STACK_VEC_POS + 2 * 32) #define STACK_TMP_Y3 (STACK_VEC_POS + 3 * 32) #define STACK_TMP_Y4 (STACK_VEC_POS + 4 * 32) #define STACK_TMP_Y5 (STACK_VEC_POS + 5 * 32) #define STACK_FXL_KEY (STACK_VEC_POS + 6 * 32) #define STACK_OFFSET_AND_F_KEY (STACK_VEC_POS + 7 * 32) #define STACK_CHECKSUM (STACK_VEC_POS + 8 * 32) #define STACK_GPR_POS (9 * 32) #define STACK_END_POS (STACK_GPR_POS + 3 * 4) subl $STACK_END_POS, %esp; andl $-32, %esp; movl %edi, (STACK_GPR_POS + 0 * 4)(%esp); CFI_REG_ON_STACK(edi, STACK_GPR_POS + 0 * 4); movl %esi, (STACK_GPR_POS + 1 * 4)(%esp); CFI_REG_ON_STACK(esi, STACK_GPR_POS + 1 * 4); movl %ebx, (STACK_GPR_POS + 2 * 4)(%esp); CFI_REG_ON_STACK(ebx, STACK_GPR_POS + 2 * 4); movl 4+4(%ebp), %edi; movl 4+8(%ebp), %esi; movl 4+12(%ebp), %edx; movl 4+32(%ebp), %ebx; movl 4+24(%ebp), %eax; movl 4+20(%ebp), %ecx; leal (, %ecx, 4), %ecx; vmovdqu (%eax), %xmm1; /* offset */ vmovdqa (%edi), %xmm0; /* first key */ vpxor %xmm0, %xmm1, %xmm1; /* offset ^ first key */ vpxor (%edi, %ecx, 4), %xmm0, %xmm0; /* first key ^ last key */ vinserti128 $1, %xmm0, %ymm0, %ymm0; vpxor %ymm2, %ymm2, %ymm2; vmovdqa %xmm1, (STACK_OFFSET_AND_F_KEY)(%esp); vmovdqa %ymm2, (STACK_CHECKSUM)(%esp); vmovdqa %ymm0, (STACK_FXL_KEY)(%esp); cmpl $12, 4+16(%ebp); jae .Locb_crypt_blk12_loop; jmp .Locb_crypt_blk4; /* Process 12 blocks per loop. */ .align 16 .Locb_crypt_blk12_loop: subl $12, 4+16(%ebp); movl 4+36(%ebp), %ecx; vmovdqa (%ecx), %xmm7; /* Preload L[0] */ testl $1, %ebx; jz .Locb_crypt_blk12_nblk_even; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ leal 1(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+1) shll $4, %eax; vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1; vpxor (%ecx, %eax), %xmm1, %xmm1; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm1; vmovdqa %ymm1, (STACK_TMP_Y0)(%esp); leal 3(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+3) shll $4, %eax; vpxor (%ecx, %eax), %xmm0, %xmm1; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm2; leal 5(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+5) shll $4, %eax; vpxor (%ecx, %eax), %xmm0, %xmm1; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm3; leal 7(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+7) shll $4, %eax; vpxor (%ecx, %eax), %xmm0, %xmm1; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm4; leal 9(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+9) shll $4, %eax; vpxor (%ecx, %eax), %xmm0, %xmm1; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm5; leal 11(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+11) shll $4, %eax; vpxor (%ecx, %eax), %xmm0, %xmm1; leal 12(%ebx), %ebx; vpxor %xmm7, %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm6; cmpl $1, 4+40(%ebp); jb .Locb_dec_blk12; ja .Locb_auth_blk12; jmp .Locb_enc_blk12; .align 8 .Locb_crypt_blk12_nblk_even: /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ vpxor (STACK_OFFSET_AND_F_KEY)(%esp), %xmm7, %xmm1; leal 2(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+2) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm1; vmovdqa %ymm1, (STACK_TMP_Y0)(%esp); vpxor %xmm7, %xmm0, %xmm1; leal 4(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+4) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm2; vpxor %xmm7, %xmm0, %xmm1; leal 6(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+6) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm3; vpxor %xmm7, %xmm0, %xmm1; leal 8(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+8) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm4; vpxor %xmm7, %xmm0, %xmm1; leal 10(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+10) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm5; vpxor %xmm7, %xmm0, %xmm1; leal 12(%ebx), %ebx; tzcntl %ebx, %eax; // ntz(blkn+12) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm0; vinserti128 $1, %xmm0, %ymm1, %ymm6; cmpl $1, 4+40(%ebp); jb .Locb_dec_blk12; ja .Locb_auth_blk12; .align 8 .Locb_enc_blk12: vmovdqa %ymm2, (STACK_TMP_Y1)(%esp); vmovdqa %ymm3, (STACK_TMP_Y2)(%esp); vmovdqa %ymm4, (STACK_TMP_Y3)(%esp); vmovdqa %ymm5, (STACK_TMP_Y4)(%esp); vmovdqa %ymm6, (STACK_TMP_Y5)(%esp); vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp); vmovdqu 0*16(%edx), %ymm1; vmovdqu 2*16(%edx), %ymm2; vmovdqu 4*16(%edx), %ymm3; vmovdqu 6*16(%edx), %ymm4; vmovdqu 8*16(%edx), %ymm5; vmovdqu 10*16(%edx), %ymm6; leal 12*16(%edx), %edx; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor %ymm1, %ymm2, %ymm0; vpxor %ymm3, %ymm4, %ymm7; vpxor %ymm5, %ymm0, %ymm0; vpxor %ymm6, %ymm7, %ymm7; vpxor %ymm0, %ymm7, %ymm7; vbroadcasti128 (1 * 16)(%edi), %ymm0; vpxor (STACK_CHECKSUM)(%esp), %ymm7, %ymm7; /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ vpxor (STACK_TMP_Y0)(%esp), %ymm1, %ymm1; vpxor (STACK_TMP_Y1)(%esp), %ymm2, %ymm2; vpxor (STACK_TMP_Y2)(%esp), %ymm3, %ymm3; vpxor (STACK_TMP_Y3)(%esp), %ymm4, %ymm4; vpxor (STACK_TMP_Y4)(%esp), %ymm5, %ymm5; vpxor (STACK_TMP_Y5)(%esp), %ymm6, %ymm6; vmovdqa %ymm7, (STACK_CHECKSUM)(%esp); /* AES rounds */ VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); cmpl $12, 4+20(%ebp); jb .Locb_enc_blk12_last; vbroadcasti128 (10 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); jz .Locb_enc_blk12_last; vbroadcasti128 (12 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); /* Last round and output handling. */ .Locb_enc_blk12_last: vmovdqa (STACK_FXL_KEY)(%esp), %ymm0; vpxor (STACK_TMP_Y0)(%esp), %ymm0, %ymm7; vaesenclast %ymm7, %ymm1, %ymm1; vpxor (STACK_TMP_Y1)(%esp), %ymm0, %ymm7; vmovdqu %ymm1, 0*16(%esi); vpxor (STACK_TMP_Y2)(%esp), %ymm0, %ymm1; vaesenclast %ymm7, %ymm2, %ymm2; vpxor (STACK_TMP_Y3)(%esp), %ymm0, %ymm7; vaesenclast %ymm1, %ymm3, %ymm3; vpxor (STACK_TMP_Y4)(%esp), %ymm0, %ymm1; vaesenclast %ymm7, %ymm4, %ymm4; vpxor (STACK_TMP_Y5)(%esp), %ymm0, %ymm7; vaesenclast %ymm1, %ymm5, %ymm5; vaesenclast %ymm7, %ymm6, %ymm6; vmovdqu %ymm2, 2*16(%esi); vmovdqu %ymm3, 4*16(%esi); vmovdqu %ymm4, 6*16(%esi); vmovdqu %ymm5, 8*16(%esi); vmovdqu %ymm6, 10*16(%esi); leal 12*16(%esi), %esi; cmpl $12, 4+16(%ebp); jae .Locb_crypt_blk12_loop; jmp .Locb_crypt_blk12_cleanup; .align 8 .Locb_auth_blk12: vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp); vbroadcasti128 (1 * 16)(%edi), %ymm0; /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ vmovdqa (STACK_TMP_Y0)(%esp), %ymm1; vpxor 0*16(%edx), %ymm1, %ymm1; vpxor 2*16(%edx), %ymm2, %ymm2; vpxor 4*16(%edx), %ymm3, %ymm3; vpxor 6*16(%edx), %ymm4, %ymm4; vpxor 8*16(%edx), %ymm5, %ymm5; vpxor 10*16(%edx), %ymm6, %ymm6; leal 12*16(%edx), %edx; /* AES rounds */ VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (10 * 16)(%edi), %ymm0; cmpl $12, 4+20(%ebp); jb .Locb_auth_blk12_last; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (12 * 16)(%edi), %ymm0; jz .Locb_auth_blk12_last; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESENC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (14 * 16)(%edi), %ymm0; /* Last round and output handling. */ .Locb_auth_blk12_last: vaesenclast %ymm0, %ymm1, %ymm1; vaesenclast %ymm0, %ymm2, %ymm2; vaesenclast %ymm0, %ymm3, %ymm3; vaesenclast %ymm0, %ymm4, %ymm4; vaesenclast %ymm0, %ymm5, %ymm5; vaesenclast %ymm0, %ymm6, %ymm6; vpxor %ymm1, %ymm2, %ymm0; vpxor %ymm3, %ymm4, %ymm4; vpxor %ymm5, %ymm0, %ymm0; vpxor %ymm6, %ymm4, %ymm4; vpxor %ymm0, %ymm4, %ymm4; vpxor (STACK_CHECKSUM)(%esp), %ymm4, %ymm4; vmovdqa %ymm4, (STACK_CHECKSUM)(%esp); cmpl $12, 4+16(%ebp); jae .Locb_crypt_blk12_loop; jmp .Locb_crypt_blk12_cleanup; .align 8 .Locb_dec_blk12: vmovdqa %xmm0, (STACK_OFFSET_AND_F_KEY)(%esp); /* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ vmovdqa (STACK_TMP_Y0)(%esp), %ymm1; vmovdqu 0*16(%edx), %ymm0; vmovdqu 2*16(%edx), %ymm7; vpxor %ymm0, %ymm1, %ymm1; vmovdqa %ymm2, (STACK_TMP_Y1)(%esp); vpxor %ymm7, %ymm2, %ymm2; vmovdqu 4*16(%edx), %ymm0; vmovdqu 6*16(%edx), %ymm7; vmovdqa %ymm3, (STACK_TMP_Y2)(%esp); vmovdqa %ymm4, (STACK_TMP_Y3)(%esp); vpxor %ymm0, %ymm3, %ymm3; vpxor %ymm7, %ymm4, %ymm4; vmovdqu 8*16(%edx), %ymm0; vmovdqu 10*16(%edx), %ymm7; leal 12*16(%edx), %edx; vmovdqa %ymm5, (STACK_TMP_Y4)(%esp); vmovdqa %ymm6, (STACK_TMP_Y5)(%esp); vpxor %ymm0, %ymm5, %ymm5; vbroadcasti128 (1 * 16)(%edi), %ymm0; vpxor %ymm7, %ymm6, %ymm6; /* AES rounds */ VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); cmpl $12, 4+20(%ebp); jb .Locb_dec_blk12_last; vbroadcasti128 (10 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); jz .Locb_dec_blk12_last; vbroadcasti128 (12 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESDEC6(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6); /* Last round and output handling. */ .Locb_dec_blk12_last: vmovdqa (STACK_FXL_KEY)(%esp), %ymm0; vpxor (STACK_TMP_Y0)(%esp), %ymm0, %ymm7; vaesdeclast %ymm7, %ymm1, %ymm1; vmovdqu %ymm1, 0*16(%esi); vpxor (STACK_TMP_Y1)(%esp), %ymm0, %ymm1; vpxor (STACK_TMP_Y2)(%esp), %ymm0, %ymm7; vaesdeclast %ymm1, %ymm2, %ymm2; vpxor (STACK_TMP_Y3)(%esp), %ymm0, %ymm1; vaesdeclast %ymm7, %ymm3, %ymm3; vpxor (STACK_TMP_Y4)(%esp), %ymm0, %ymm7; vaesdeclast %ymm1, %ymm4, %ymm4; vpxor (STACK_TMP_Y5)(%esp), %ymm0, %ymm0; vaesdeclast %ymm7, %ymm5, %ymm5; vaesdeclast %ymm0, %ymm6, %ymm6; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor %ymm2, %ymm3, %ymm0; vpxor %ymm4, %ymm5, %ymm7; vpxor %ymm6, %ymm0, %ymm0; vpxor 0*16(%esi), %ymm7, %ymm7; vpxor %ymm0, %ymm7, %ymm7; vpxor (STACK_CHECKSUM)(%esp), %ymm7, %ymm7; vmovdqu %ymm2, 2*16(%esi); vmovdqu %ymm3, 4*16(%esi); vmovdqu %ymm4, 6*16(%esi); vmovdqu %ymm5, 8*16(%esi); vmovdqu %ymm6, 10*16(%esi); leal 12*16(%esi), %esi; vmovdqa %ymm7, (STACK_CHECKSUM)(%esp); cmpl $12, 4+16(%ebp); jae .Locb_crypt_blk12_loop; .align 8 .Locb_crypt_blk12_cleanup: vpxor %ymm0, %ymm0, %ymm0; vmovdqa %ymm0, (STACK_TMP_Y0)(%esp); vmovdqa %ymm0, (STACK_TMP_Y1)(%esp); vmovdqa %ymm0, (STACK_TMP_Y2)(%esp); vmovdqa %ymm0, (STACK_TMP_Y3)(%esp); vmovdqa %ymm0, (STACK_TMP_Y4)(%esp); vmovdqa %ymm0, (STACK_TMP_Y5)(%esp); /* Process trailing four blocks. */ .align 8 .Locb_crypt_blk4: cmpl $4, 4+16(%ebp); jb .Locb_crypt_blk1; subl $4, 4+16(%ebp); movl 4+36(%ebp), %ecx; vmovdqa (%ecx), %xmm7; /* Preload L[0] */ testl $1, %ebx; jz .Locb_crypt_blk4_nblk_even; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ leal 1(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+1) shll $4, %eax; vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1; vpxor (%ecx, %eax), %xmm1, %xmm1; vpxor %xmm7, %xmm1, %xmm2; vinserti128 $1, %xmm2, %ymm1, %ymm6; leal 3(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+3) shll $4, %eax; vpxor (%ecx, %eax), %xmm2, %xmm3; leal 4(%ebx), %ebx; vpxor %xmm7, %xmm3, %xmm4; vinserti128 $1, %xmm4, %ymm3, %ymm7; vmovdqa %xmm4, (STACK_OFFSET_AND_F_KEY)(%esp); cmpl $1, 4+40(%ebp); jb .Locb_dec_blk4; ja .Locb_auth_blk4; jmp .Locb_enc_blk4; .align 8 .Locb_crypt_blk4_nblk_even: /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1; vpxor %xmm7, %xmm1, %xmm1; leal 2(%ebx), %eax; tzcntl %eax, %eax; // ntz(blkn+2) shll $4, %eax; vpxor (%ecx, %eax), %xmm1, %xmm2; vinserti128 $1, %xmm2, %ymm1, %ymm6; vpxor %xmm7, %xmm2, %xmm3; leal 4(%ebx), %ebx; tzcntl %ebx, %eax; // ntz(blkn+4) shll $4, %eax; vpxor (%ecx, %eax), %xmm3, %xmm4; vinserti128 $1, %xmm4, %ymm3, %ymm7; vmovdqa %xmm4, (STACK_OFFSET_AND_F_KEY)(%esp); cmpl $1, 4+40(%ebp); jb .Locb_dec_blk4; ja .Locb_auth_blk4; .align 8 .Locb_enc_blk4: vmovdqu 0*16(%edx), %ymm1; vmovdqu 2*16(%edx), %ymm2; leal 4*16(%edx), %edx; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor %ymm1, %ymm2, %ymm5; vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5; vmovdqa %ymm5, (STACK_CHECKSUM)(%esp); /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ vpxor %ymm6, %ymm1, %ymm1; vpxor %ymm7, %ymm2, %ymm2; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); cmpl $12, 4+20(%ebp); jb .Locb_enc_blk4_last; vbroadcasti128 (10 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); jz .Locb_enc_blk4_last; vbroadcasti128 (12 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); /* Last round and output handling. */ .Locb_enc_blk4_last: vmovdqa (STACK_FXL_KEY)(%esp), %ymm0; vpxor %ymm0, %ymm6, %ymm6; /* Xor offset to last round key. */ vpxor %ymm0, %ymm7, %ymm7; vaesenclast %ymm6, %ymm1, %ymm1; vaesenclast %ymm7, %ymm2, %ymm2; vmovdqu %ymm1, 0*16(%esi); vmovdqu %ymm2, 2*16(%esi); leal 4*16(%esi), %esi; jmp .Locb_crypt_blk1; .align 8 .Locb_auth_blk4: /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ vpxor 0*16(%edx), %ymm6, %ymm1; vpxor 2*16(%edx), %ymm7, %ymm2; leal 4*16(%edx), %edx; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (10 * 16)(%edi), %ymm0; cmpl $12, 4+20(%ebp); jb .Locb_auth_blk4_last; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (12 * 16)(%edi), %ymm0; jz .Locb_auth_blk4_last; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESENC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (14 * 16)(%edi), %ymm0; /* Last round and output handling. */ .Locb_auth_blk4_last: vaesenclast %ymm0, %ymm1, %ymm1; vaesenclast %ymm0, %ymm2, %ymm2; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor %ymm1, %ymm2, %ymm5; vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5; vmovdqa %ymm5, (STACK_CHECKSUM)(%esp); jmp .Locb_crypt_blk1; .align 8 .Locb_dec_blk4: /* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ vpxor 0*16(%edx), %ymm6, %ymm1; vpxor 2*16(%edx), %ymm7, %ymm2; leal 4*16(%edx), %edx; /* AES rounds */ vbroadcasti128 (1 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (2 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (3 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (4 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (5 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (6 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (7 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (8 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (9 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); cmpl $12, 4+20(%ebp); jb .Locb_dec_blk4_last; vbroadcasti128 (10 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (11 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); jz .Locb_dec_blk4_last; vbroadcasti128 (12 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); vbroadcasti128 (13 * 16)(%edi), %ymm0; VAESDEC2(%ymm0, %ymm1, %ymm2); /* Last round and output handling. */ .Locb_dec_blk4_last: vmovdqa (STACK_FXL_KEY)(%esp), %ymm0; vpxor %ymm0, %ymm6, %ymm6; /* Xor offset to last round key. */ vpxor %ymm0, %ymm7, %ymm7; vaesdeclast %ymm6, %ymm1, %ymm1; vaesdeclast %ymm7, %ymm2, %ymm2; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor %ymm1, %ymm2, %ymm5; vpxor (STACK_CHECKSUM)(%esp), %ymm5, %ymm5; vmovdqu %ymm1, 0*16(%esi); vmovdqu %ymm2, 2*16(%esi); leal 4*16(%esi), %esi; vmovdqa %ymm5, (STACK_CHECKSUM)(%esp); /* Process trailing one to three blocks, one per loop. */ .align 8 .Locb_crypt_blk1: cmpl $1, 4+16(%ebp); jb .Locb_crypt_done; subl $1, 4+16(%ebp); movl 4+36(%ebp), %ecx; leal 1(%ebx), %ebx; tzcntl %ebx, %eax; // ntz(blkn+1) shll $4, %eax; vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm7; vpxor (%ecx, %eax), %xmm7, %xmm7; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ vmovdqa %xmm7, (STACK_OFFSET_AND_F_KEY)(%esp); cmpl $1, 4+40(%ebp); jb .Locb_dec_blk1; ja .Locb_auth_blk1; vmovdqu (%edx), %xmm0; leal 16(%edx), %edx; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor (STACK_CHECKSUM)(%esp), %xmm0, %xmm1; vmovdqa %xmm1, (STACK_CHECKSUM)(%esp); /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ vpxor %xmm7, %xmm0, %xmm0; /* AES rounds. */ vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; cmpl $12, 4+20(%ebp); jb .Locb_enc_blk1_last; vaesenc (10 * 16)(%edi), %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; jz .Locb_enc_blk1_last; vaesenc (12 * 16)(%edi), %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; /* Last round and output handling. */ .Locb_enc_blk1_last: vpxor (STACK_FXL_KEY)(%esp), %xmm7, %xmm1; vaesenclast %xmm1, %xmm0, %xmm0; vmovdqu %xmm0, (%esi); leal 16(%esi), %esi; jmp .Locb_crypt_blk1; .align 8 .Locb_auth_blk1: /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ vpxor (%edx), %xmm7, %xmm0; leal 16(%edx), %edx; /* AES rounds. */ vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 4+20(%ebp); jb .Locb_auth_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Locb_auth_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Locb_auth_blk1_last: vpxor (STACK_CHECKSUM)(%esp), %xmm1, %xmm1; vaesenclast %xmm1, %xmm0, %xmm0; vmovdqa %xmm0, (STACK_CHECKSUM)(%esp); jmp .Locb_crypt_blk1; .align 8 .Locb_dec_blk1: /* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ vpxor (%edx), %xmm7, %xmm0; leal 16(%edx), %edx; /* AES rounds. */ vaesdec (1 * 16)(%edi), %xmm0, %xmm0; vaesdec (2 * 16)(%edi), %xmm0, %xmm0; vaesdec (3 * 16)(%edi), %xmm0, %xmm0; vaesdec (4 * 16)(%edi), %xmm0, %xmm0; vaesdec (5 * 16)(%edi), %xmm0, %xmm0; vaesdec (6 * 16)(%edi), %xmm0, %xmm0; vaesdec (7 * 16)(%edi), %xmm0, %xmm0; vaesdec (8 * 16)(%edi), %xmm0, %xmm0; vaesdec (9 * 16)(%edi), %xmm0, %xmm0; cmpl $12, 4+20(%ebp); jb .Locb_dec_blk1_last; vaesdec (10 * 16)(%edi), %xmm0, %xmm0; vaesdec (11 * 16)(%edi), %xmm0, %xmm0; jz .Locb_dec_blk1_last; vaesdec (12 * 16)(%edi), %xmm0, %xmm0; vaesdec (13 * 16)(%edi), %xmm0, %xmm0; /* Last round and output handling. */ .Locb_dec_blk1_last: vpxor (STACK_FXL_KEY)(%esp), %xmm7, %xmm1; vaesdeclast %xmm1, %xmm0, %xmm0; /* Checksum_i = Checksum_{i-1} xor P_i */ vpxor (STACK_CHECKSUM)(%esp), %xmm0, %xmm1; vmovdqu %xmm0, (%esi); leal 16(%esi), %esi; vmovdqa %xmm1, (STACK_CHECKSUM)(%esp); jmp .Locb_crypt_blk1; .align 8 .Locb_crypt_done: movl 4+24(%ebp), %ecx; vmovdqa (STACK_OFFSET_AND_F_KEY)(%esp), %xmm1; vpxor (%edi), %xmm1, %xmm1; vmovdqu %xmm1, (%ecx); movl 4+28(%ebp), %eax; vmovdqa (STACK_CHECKSUM)(%esp), %xmm2; vpxor (STACK_CHECKSUM + 16)(%esp), %xmm2, %xmm2; vpxor (%eax), %xmm2, %xmm2; vmovdqu %xmm2, (%eax); movl (STACK_GPR_POS + 0 * 4)(%esp), %edi; CFI_RESTORE(edi); movl (STACK_GPR_POS + 1 * 4)(%esp), %esi; CFI_RESTORE(esi); movl (STACK_GPR_POS + 2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); vpxor %ymm0, %ymm0, %ymm0; vmovdqa %ymm0, (STACK_OFFSET_AND_F_KEY)(%esp); vmovdqa %ymm0, (STACK_CHECKSUM)(%esp); xorl %eax, %eax; leave; CFI_LEAVE(); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386), .-SYM_NAME(_gcry_vaes_avx2_ocb_crypt_i386)) /********************************************************************** XTS-mode encryption **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386): /* input: * (esp + 4): round keys * (esp + 8): tweak * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds * (esp + 28): encrypt */ CFI_STARTPROC(); GET_DATA_POINTER(SYM_NAME(_gcry_vaes_consts), eax); pushl %ebp; CFI_PUSH(%ebp); movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); subl $(4 * 32 + 3 * 4), %esp; andl $-32, %esp; movl %edi, (4 * 32 + 0 * 4)(%esp); CFI_REG_ON_STACK(edi, 4 * 32 + 0 * 4); movl %esi, (4 * 32 + 1 * 4)(%esp); CFI_REG_ON_STACK(esi, 4 * 32 + 1 * 4); movl %ebx, (4 * 32 + 2 * 4)(%esp); CFI_REG_ON_STACK(ebx, 4 * 32 + 2 * 4); movl %eax, %ebx; movl 4+4(%ebp), %edi; movl 4+8(%ebp), %esi; movl 4+12(%ebp), %edx; movl 4+16(%ebp), %ecx; movl 4+20(%ebp), %eax; #define tweak_clmul(shift, out, tweak, hi_tweak, tmp1, tmp2) \ vpsrld $(32-(shift)), hi_tweak, tmp2; \ vpsllq $(shift), tweak, out; \ vpclmulqdq $0, CADDR(.Lxts_gfmul_clmul, %ebx), tmp2, tmp1; \ vpunpckhqdq tmp2, tmp1, tmp1; \ vpxor tmp1, out, out; /* Prepare tweak. */ vmovdqu (%esi), %xmm7; vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %xmm7, %xmm6; tweak_clmul(1, %xmm5, %xmm7, %xmm6, %xmm0, %xmm1); vinserti128 $1, %xmm5, %ymm7, %ymm7; /* tweak:tweak1 */ vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6; /* Process eight blocks per loop. */ .align 8 .Lxts_crypt_blk8: cmpl $8, %eax; jb .Lxts_crypt_blk4; leal -8(%eax), %eax; vmovdqa %ymm7, (0 * 32)(%esp); tweak_clmul(2, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1); vmovdqa %ymm2, (1 * 32)(%esp); tweak_clmul(4, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1); vmovdqa %ymm2, (2 * 32)(%esp); tweak_clmul(6, %ymm2, %ymm7, %ymm6, %ymm0, %ymm1); vmovdqa %ymm2, (3 * 32)(%esp); tweak_clmul(8, %ymm7, %ymm7, %ymm6, %ymm0, %ymm1); vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6; vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqa (0 * 32)(%esp), %ymm0; vmovdqa (1 * 32)(%esp), %ymm1; vmovdqa (2 * 32)(%esp), %ymm2; vmovdqa (3 * 32)(%esp), %ymm3; vpxor (0 * 16)(%ecx), %ymm0, %ymm0; vpxor (2 * 16)(%ecx), %ymm1, %ymm1; vpxor (4 * 16)(%ecx), %ymm2, %ymm2; vpxor (6 * 16)(%ecx), %ymm3, %ymm3; leal (8 * 16)(%ecx), %ecx; cmpl $1, 4+28(%ebp); jne .Lxts_dec_blk8; /* AES rounds */ XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lxts_enc_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lxts_enc_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lxts_enc_blk8_last: vpxor (0 * 32)(%esp), %ymm4, %ymm5; /* Xor tweak to last round key. */ vaesenclast %ymm5, %ymm0, %ymm0; vpxor (1 * 32)(%esp), %ymm4, %ymm5; vaesenclast %ymm5, %ymm1, %ymm1; vpxor (2 * 32)(%esp), %ymm4, %ymm5; vpxor (3 * 32)(%esp), %ymm4, %ymm4; vaesenclast %ymm5, %ymm2, %ymm2; vaesenclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lxts_crypt_blk8; .align 8 .Lxts_dec_blk8: /* AES rounds */ XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lxts_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lxts_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lxts_dec_blk8_last: vpxor (0 * 32)(%esp), %ymm4, %ymm5; /* Xor tweak to last round key. */ vaesdeclast %ymm5, %ymm0, %ymm0; vpxor (1 * 32)(%esp), %ymm4, %ymm5; vaesdeclast %ymm5, %ymm1, %ymm1; vpxor (2 * 32)(%esp), %ymm4, %ymm5; vpxor (3 * 32)(%esp), %ymm4, %ymm4; vaesdeclast %ymm5, %ymm2, %ymm2; vaesdeclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lxts_crypt_blk8; /* Handle trailing four blocks. */ .align 8 .Lxts_crypt_blk4: /* Try exit early as typically input length is large power of 2. */ cmpl $1, %eax; jb .Ldone_xts_crypt; cmpl $4, %eax; jb .Lxts_crypt_blk1; leal -4(%eax), %eax; vmovdqa %ymm7, %ymm2; tweak_clmul(2, %ymm3, %ymm7, %ymm6, %ymm0, %ymm1); tweak_clmul(4, %ymm7, %ymm7, %ymm6, %ymm0, %ymm1); vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %ymm7, %ymm6; vbroadcasti128 (0 * 16)(%edi), %ymm4; vpxor (0 * 16)(%ecx), %ymm2, %ymm0; vpxor (2 * 16)(%ecx), %ymm3, %ymm1; leal (4 * 16)(%ecx), %ecx; cmpl $1, 4+28(%ebp); jne .Lxts_dec_blk4; /* AES rounds */ XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lxts_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lxts_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lxts_enc_blk4_last: vpxor %ymm4, %ymm2, %ymm2; /* Xor tweak to last round key. */ vpxor %ymm4, %ymm3, %ymm3; vaesenclast %ymm2, %ymm0, %ymm0; vaesenclast %ymm3, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; jmp .Lxts_crypt_blk1; .align 8 .Lxts_dec_blk4: /* AES rounds */ XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 4+24(%ebp); jb .Lxts_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lxts_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; /* Last round and output handling. */ .Lxts_dec_blk4_last: vpxor %ymm4, %ymm2, %ymm2; /* Xor tweak to last round key. */ vpxor %ymm4, %ymm3, %ymm3; vaesdeclast %ymm2, %ymm0, %ymm0; vaesdeclast %ymm3, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lxts_crypt_blk1: cmpl $1, %eax; jb .Ldone_xts_crypt; leal -1(%eax), %eax; vpxor (%ecx), %xmm7, %xmm0; vmovdqa %xmm7, %xmm5; tweak_clmul(1, %xmm7, %xmm7, %xmm6, %xmm2, %xmm3); vpshufb CADDR(.Lxts_high_bit_shuf, %ebx), %xmm7, %xmm6; leal 16(%ecx), %ecx; cmpl $1, 4+28(%ebp); jne .Lxts_dec_blk1; /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 4+24(%ebp); jb .Lxts_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lxts_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lxts_enc_blk1_last: vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */ vaesenclast %xmm5, %xmm0, %xmm0; vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lxts_crypt_blk1; .align 8 .Lxts_dec_blk1: /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesdec (1 * 16)(%edi), %xmm0, %xmm0; vaesdec (2 * 16)(%edi), %xmm0, %xmm0; vaesdec (3 * 16)(%edi), %xmm0, %xmm0; vaesdec (4 * 16)(%edi), %xmm0, %xmm0; vaesdec (5 * 16)(%edi), %xmm0, %xmm0; vaesdec (6 * 16)(%edi), %xmm0, %xmm0; vaesdec (7 * 16)(%edi), %xmm0, %xmm0; vaesdec (8 * 16)(%edi), %xmm0, %xmm0; vaesdec (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 4+24(%ebp); jb .Lxts_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lxts_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; /* Last round and output handling. */ .Lxts_dec_blk1_last: vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */ vaesdeclast %xmm5, %xmm0, %xmm0; vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lxts_crypt_blk1; .align 8 .Ldone_xts_crypt: /* Store IV. */ vmovdqu %xmm7, (%esi); vpxor %ymm0, %ymm0, %ymm0; movl (4 * 32 + 0 * 4)(%esp), %edi; CFI_RESTORE(edi); movl (4 * 32 + 1 * 4)(%esp), %esi; CFI_RESTORE(esi); movl (4 * 32 + 2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); vmovdqa %ymm0, (0 * 32)(%esp); vmovdqa %ymm0, (1 * 32)(%esp); vmovdqa %ymm0, (2 * 32)(%esp); vmovdqa %ymm0, (3 * 32)(%esp); leave; CFI_LEAVE(); vzeroall; xorl %eax, %eax; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386), .-SYM_NAME(_gcry_vaes_avx2_xts_crypt_i386)) /********************************************************************** ECB-mode encryption **********************************************************************/ ELF(.type SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386),@function) .globl SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386) .align 16 SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386): /* input: * (esp + 4): round keys * (esp + 8): encrypt * (esp + 12): dst * (esp + 16): src * (esp + 20): nblocks * (esp + 24): nrounds */ CFI_STARTPROC(); pushl %edi; CFI_PUSH(%edi); pushl %esi; CFI_PUSH(%esi); movl 8+4(%esp), %edi; movl 8+8(%esp), %esi; movl 8+12(%esp), %edx; movl 8+16(%esp), %ecx; movl 8+20(%esp), %eax; /* Process 8 blocks per loop. */ .align 8 .Lecb_blk8: cmpl $8, %eax; jb .Lecb_blk4; leal -8(%eax), %eax; /* Load input and xor first key. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm0; vmovdqu (2 * 16)(%ecx), %ymm1; vmovdqu (4 * 16)(%ecx), %ymm2; vmovdqu (6 * 16)(%ecx), %ymm3; vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vpxor %ymm4, %ymm2, %ymm2; vpxor %ymm4, %ymm3, %ymm3; vbroadcasti128 (1 * 16)(%edi), %ymm4; leal (8 * 16)(%ecx), %ecx; testl %esi, %esi; jz .Lecb_dec_blk8; /* AES rounds */ VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lecb_enc_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lecb_enc_blk8_last; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; .Lecb_enc_blk8_last: vaesenclast %ymm4, %ymm0, %ymm0; vaesenclast %ymm4, %ymm1, %ymm1; vaesenclast %ymm4, %ymm2, %ymm2; vaesenclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lecb_blk8; .align 8 .Lecb_dec_blk8: /* AES rounds */ VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lecb_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lecb_dec_blk8_last; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (14 * 16)(%edi), %ymm4; .Lecb_dec_blk8_last: vaesdeclast %ymm4, %ymm0, %ymm0; vaesdeclast %ymm4, %ymm1, %ymm1; vaesdeclast %ymm4, %ymm2, %ymm2; vaesdeclast %ymm4, %ymm3, %ymm3; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); vmovdqu %ymm2, (4 * 16)(%edx); vmovdqu %ymm3, (6 * 16)(%edx); leal (8 * 16)(%edx), %edx; jmp .Lecb_blk8; /* Handle trailing four blocks. */ .align 8 .Lecb_blk4: cmpl $4, %eax; jb .Lecb_blk1; leal -4(%eax), %eax; /* Load input and xor first key. */ vbroadcasti128 (0 * 16)(%edi), %ymm4; vmovdqu (0 * 16)(%ecx), %ymm0; vmovdqu (2 * 16)(%ecx), %ymm1; vpxor %ymm4, %ymm0, %ymm0; vpxor %ymm4, %ymm1, %ymm1; vbroadcasti128 (1 * 16)(%edi), %ymm4; leal (4 * 16)(%ecx), %ecx; testl %esi, %esi; jz .Lecb_dec_blk4; /* AES rounds */ VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lecb_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lecb_enc_blk4_last; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESENC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; .Lecb_enc_blk4_last: vaesenclast %ymm4, %ymm0, %ymm0; vaesenclast %ymm4, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; jmp .Lecb_blk1; .align 8 .Lecb_dec_blk4: /* AES rounds */ VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (2 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (3 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (4 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (5 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (6 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (7 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (8 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (9 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (10 * 16)(%edi), %ymm4; cmpl $12, 8+24(%esp); jb .Lecb_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (11 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (12 * 16)(%edi), %ymm4; jz .Lecb_dec_blk4_last; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (13 * 16)(%edi), %ymm4; VAESDEC2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (14 * 16)(%edi), %ymm4; .Lecb_dec_blk4_last: vaesdeclast %ymm4, %ymm0, %ymm0; vaesdeclast %ymm4, %ymm1, %ymm1; vmovdqu %ymm0, (0 * 16)(%edx); vmovdqu %ymm1, (2 * 16)(%edx); leal (4 * 16)(%edx), %edx; /* Process trailing one to three blocks, one per loop. */ .align 8 .Lecb_blk1: cmpl $1, %eax; jb .Ldone_ecb; leal -1(%eax), %eax; /* Load input. */ vmovdqu (%ecx), %xmm2; leal 16(%ecx), %ecx; /* Xor first key. */ vpxor (0 * 16)(%edi), %xmm2, %xmm0; testl %esi, %esi; jz .Lecb_dec_blk1; /* AES rounds. */ vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; vaesenc (7 * 16)(%edi), %xmm0, %xmm0; vaesenc (8 * 16)(%edi), %xmm0, %xmm0; vaesenc (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 8+24(%esp); jb .Lecb_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lecb_enc_blk1_last; vaesenc %xmm1, %xmm0, %xmm0; vaesenc (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; .Lecb_enc_blk1_last: vaesenclast %xmm1, %xmm0, %xmm0; jmp .Lecb_blk1_end; .align 8 .Lecb_dec_blk1: /* AES rounds. */ vaesdec (1 * 16)(%edi), %xmm0, %xmm0; vaesdec (2 * 16)(%edi), %xmm0, %xmm0; vaesdec (3 * 16)(%edi), %xmm0, %xmm0; vaesdec (4 * 16)(%edi), %xmm0, %xmm0; vaesdec (5 * 16)(%edi), %xmm0, %xmm0; vaesdec (6 * 16)(%edi), %xmm0, %xmm0; vaesdec (7 * 16)(%edi), %xmm0, %xmm0; vaesdec (8 * 16)(%edi), %xmm0, %xmm0; vaesdec (9 * 16)(%edi), %xmm0, %xmm0; vmovdqa (10 * 16)(%edi), %xmm1; cmpl $12, 8+24(%esp); jb .Lecb_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (11 * 16)(%edi), %xmm0, %xmm0; vmovdqa (12 * 16)(%edi), %xmm1; jz .Lecb_dec_blk1_last; vaesdec %xmm1, %xmm0, %xmm0; vaesdec (13 * 16)(%edi), %xmm0, %xmm0; vmovdqa (14 * 16)(%edi), %xmm1; .Lecb_dec_blk1_last: vaesdeclast %xmm1, %xmm0, %xmm0; jmp .Lecb_blk1_end; .align 8 .Lecb_blk1_end: vmovdqu %xmm0, (%edx); leal 16(%edx), %edx; jmp .Lecb_blk1; .align 8 .Ldone_ecb: popl %esi; CFI_POP(%esi); popl %edi; CFI_POP(%edi); vzeroall; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386), .-SYM_NAME(_gcry_vaes_avx2_ecb_crypt_i386)) /********************************************************************** constants **********************************************************************/ SECTION_RODATA ELF(.type SYM_NAME(_gcry_vaes_consts),@object) .align 32 SYM_NAME(_gcry_vaes_consts): .Lbige_addb_0: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lbige_addb_1: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 .Lbige_addb_2: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 .Lbige_addb_3: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 .Lbige_addb_4: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 .Lbige_addb_5: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5 .Lbige_addb_6: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6 .Lbige_addb_7: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7 .Lbige_addb_8: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 .Lbige_addb_9: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9 .Lbige_addb_10: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 .Lbige_addb_11: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11 .Lle_addd_0: .byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_1: .byte 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_2: .byte 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_3: .byte 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_4: .byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_5: .byte 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_6: .byte 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_7: .byte 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_8: .byte 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_9: .byte 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_10: .byte 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_11: .byte 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_4_2: .byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lle_addd_12_2: .byte 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .byte 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 .Lxts_gfmul_clmul: .long 0x00, 0x87, 0x00, 0x00 .long 0x00, 0x87, 0x00, 0x00 .Lxts_high_bit_shuf: .byte -1, -1, -1, -1, 12, 13, 14, 15 .byte 4, 5, 6, 7, -1, -1, -1, -1 .byte -1, -1, -1, -1, 12, 13, 14, 15 .byte 4, 5, 6, 7, -1, -1, -1, -1 .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 ELF(.size SYM_NAME(_gcry_vaes_consts),.-SYM_NAME(_gcry_vaes_consts)) #endif /* HAVE_GCC_INLINE_ASM_VAES */ #endif /* __i386__ */