Lncvrt/grub
1
0 Fork 0
mirror of https://git.savannah.gnu.org/git/grub.git synced 2026-10-24 08:09:22 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
d19a74a17fd92361921a1e8903321d4eac56bcb4
grub/grub-core/lib/libgcrypt/cipher/rijndael-armv8-aarch32-ce.S
T
Vladimir Serbinenko 3312af6e07 libgcrypt: Import libgcrypt 1.11 
We currently use an old version of libgcrypt which results in us having
fewer ciphers and missing on many other improvements.

Signed-off-by: Vladimir Serbinenko <phcoder@gmail.com>
Reviewed-by: Daniel Kiper <daniel.kiper@oracle.com>
2025-07-11 23:12:50 +02:00

2135 lines
51 KiB
ArmAsm
Raw Blame History

/* rijndael-armv8-aarch32-ce.S - ARMv8/CE accelerated AES
* Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
*
* This file is part of Libgcrypt.
*
* Libgcrypt is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* Libgcrypt is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#if defined(HAVE_ARM_ARCH_V6) && defined(__ARMEL__) && \
defined(HAVE_COMPATIBLE_GCC_ARM_PLATFORM_AS) && \
defined(HAVE_GCC_INLINE_ASM_AARCH32_CRYPTO)
.syntax unified
.arch armv8-a
.fpu crypto-neon-fp-armv8
.arm
.text
#ifdef __PIC__
# define GET_DATA_POINTER(reg, name, rtmp) \
ldr reg, 1f; \
ldr rtmp, 2f; \
b 3f; \
1: .word _GLOBAL_OFFSET_TABLE_-(3f+8); \
2: .word name(GOT); \
3: add reg, pc, reg; \
ldr reg, [reg, rtmp];
#else
# define GET_DATA_POINTER(reg, name, rtmp) ldr reg, =name
#endif
/* AES macros */
#define aes_preload_keys(keysched, rekeysched) \
vldmia keysched!, {q5-q7}; \
mov rekeysched, keysched; \
vldmialo keysched!, {q8-q15}; /* 128-bit */ \
addeq keysched, #(2*16); \
vldmiaeq keysched!, {q10-q15}; /* 192-bit */ \
addhi keysched, #(4*16); \
vldmiahi keysched!, {q12-q15}; /* 256-bit */ \
#define do_aes_one128(ed, mcimc, qo, qb) \
aes##ed.8 qb, q5; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q6; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q7; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q8; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q9; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q10; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q11; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q12; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q13; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q14; \
veor qo, qb, q15;
#define do_aes_one128re(ed, mcimc, qo, qb, keysched, rekeysched) \
vldm rekeysched, {q8-q9}; \
do_aes_one128(ed, mcimc, qo, qb);
#define do_aes_one192(ed, mcimc, qo, qb, keysched, rekeysched) \
vldm rekeysched!, {q8}; \
aes##ed.8 qb, q5; \
aes##mcimc.8 qb, qb; \
vldm rekeysched, {q9}; \
aes##ed.8 qb, q6; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q7; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q8; \
aes##mcimc.8 qb, qb; \
vldmia keysched!, {q8}; \
aes##ed.8 qb, q9; \
aes##mcimc.8 qb, qb; \
sub rekeysched, #(1*16); \
aes##ed.8 qb, q10; \
aes##mcimc.8 qb, qb; \
vldm keysched, {q9}; \
aes##ed.8 qb, q11; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q12; \
aes##mcimc.8 qb, qb; \
sub keysched, #16; \
aes##ed.8 qb, q13; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q14; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q15; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q8; \
veor qo, qb, q9; \
#define do_aes_one256(ed, mcimc, qo, qb, keysched, rekeysched) \
vldmia rekeysched!, {q8}; \
aes##ed.8 qb, q5; \
aes##mcimc.8 qb, qb; \
vldmia rekeysched!, {q9}; \
aes##ed.8 qb, q6; \
aes##mcimc.8 qb, qb; \
vldmia rekeysched!, {q10}; \
aes##ed.8 qb, q7; \
aes##mcimc.8 qb, qb; \
vldm rekeysched, {q11}; \
aes##ed.8 qb, q8; \
aes##mcimc.8 qb, qb; \
vldmia keysched!, {q8}; \
aes##ed.8 qb, q9; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q10; \
aes##mcimc.8 qb, qb; \
vldmia keysched!, {q9}; \
aes##ed.8 qb, q11; \
aes##mcimc.8 qb, qb; \
sub rekeysched, #(3*16); \
aes##ed.8 qb, q12; \
aes##mcimc.8 qb, qb; \
vldmia keysched!, {q10}; \
aes##ed.8 qb, q13; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q14; \
aes##mcimc.8 qb, qb; \
vldm keysched, {q11}; \
aes##ed.8 qb, q15; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q8; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q9; \
aes##mcimc.8 qb, qb; \
aes##ed.8 qb, q10; \
veor qo, qb, q11; \
sub keysched, #(3*16); \
#define aes_round_4(ed, mcimc, b0, b1, b2, b3, key) \
aes##ed.8 b0, key; \
aes##mcimc.8 b0, b0; \
aes##ed.8 b1, key; \
aes##mcimc.8 b1, b1; \
aes##ed.8 b2, key; \
aes##mcimc.8 b2, b2; \
aes##ed.8 b3, key; \
aes##mcimc.8 b3, b3;
#define do_aes_4_128(ed, mcimc, b0, b1, b2, b3) \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q5); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q6); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q7); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q8); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q9); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q10); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q11); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q12); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q13); \
aes##ed.8 b0, q14; \
veor b0, b0, q15; \
aes##ed.8 b1, q14; \
veor b1, b1, q15; \
aes##ed.8 b2, q14; \
veor b2, b2, q15; \
aes##ed.8 b3, q14; \
veor b3, b3, q15;
#define do_aes_4_128re(ed, mcimc, b0, b1, b2, b3, keysched, rekeysched) \
vldm rekeysched, {q8-q9}; \
do_aes_4_128(ed, mcimc, b0, b1, b2, b3);
#define do_aes_4_192(ed, mcimc, b0, b1, b2, b3, keysched, rekeysched) \
vldm rekeysched!, {q8}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q5); \
vldm rekeysched, {q9}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q6); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q7); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q8); \
vldmia keysched!, {q8}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q9); \
sub rekeysched, #(1*16); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q10); \
vldm keysched, {q9}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q11); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q12); \
sub keysched, #16; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q13); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q14); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q15); \
aes##ed.8 b0, q8; \
veor b0, b0, q9; \
aes##ed.8 b1, q8; \
veor b1, b1, q9; \
aes##ed.8 b2, q8; \
veor b2, b2, q9; \
aes##ed.8 b3, q8; \
veor b3, b3, q9;
#define do_aes_4_256(ed, mcimc, b0, b1, b2, b3, keysched, rekeysched) \
vldmia rekeysched!, {q8}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q5); \
vldmia rekeysched!, {q9}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q6); \
vldmia rekeysched!, {q10}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q7); \
vldm rekeysched, {q11}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q8); \
vldmia keysched!, {q8}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q9); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q10); \
vldmia keysched!, {q9}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q11); \
sub rekeysched, #(3*16); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q12); \
vldmia keysched!, {q10}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q13); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q14); \
vldm keysched, {q11}; \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q15); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q8); \
aes_round_4(ed, mcimc, b0, b1, b2, b3, q9); \
sub keysched, #(3*16); \
aes##ed.8 b0, q10; \
veor b0, b0, q11; \
aes##ed.8 b1, q10; \
veor b1, b1, q11; \
aes##ed.8 b2, q10; \
veor b2, b2, q11; \
aes##ed.8 b3, q10; \
veor b3, b3, q11;
/* Other functional macros */
#define CLEAR_REG(reg) vmov.i8 reg, #0;
/*
* unsigned int _gcry_aes_enc_armv8_ce(void *keysched, byte *dst,
* const byte *src,
* unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_enc_armv8_ce
.type _gcry_aes_enc_armv8_ce,%function;
_gcry_aes_enc_armv8_ce:
/* input:
* r0: keysched
* r1: dst
* r2: src
* r3: nrounds
*/
vldmia r0!, {q1-q3} /* load 3 round keys */
cmp r3, #12
vld1.8 {q0}, [r2]
bhi .Lenc1_256
beq .Lenc1_192
.Lenc1_128:
.Lenc1_tail:
vldmia r0, {q8-q15} /* load 8 round keys */
aese.8 q0, q1
aesmc.8 q0, q0
CLEAR_REG(q1)
aese.8 q0, q2
aesmc.8 q0, q0
CLEAR_REG(q2)
aese.8 q0, q3
aesmc.8 q0, q0
CLEAR_REG(q3)
aese.8 q0, q8
aesmc.8 q0, q0
CLEAR_REG(q8)
aese.8 q0, q9
aesmc.8 q0, q0
CLEAR_REG(q9)
aese.8 q0, q10
aesmc.8 q0, q0
CLEAR_REG(q10)
aese.8 q0, q11
aesmc.8 q0, q0
CLEAR_REG(q11)
aese.8 q0, q12
aesmc.8 q0, q0
CLEAR_REG(q12)
aese.8 q0, q13
aesmc.8 q0, q0
CLEAR_REG(q13)
aese.8 q0, q14
veor q0, q15
CLEAR_REG(q14)
CLEAR_REG(q15)
vst1.8 {q0}, [r1]
CLEAR_REG(q0)
mov r0, #0
bx lr
.Lenc1_192:
aese.8 q0, q1
aesmc.8 q0, q0
vmov q1, q3
aese.8 q0, q2
aesmc.8 q0, q0
vldm r0!, {q2-q3} /* load 3 round keys */
b .Lenc1_tail
.Lenc1_256:
vldm r0!, {q15} /* load 1 round key */
aese.8 q0, q1
aesmc.8 q0, q0
aese.8 q0, q2
aesmc.8 q0, q0
aese.8 q0, q3
aesmc.8 q0, q0
vldm r0!, {q1-q3} /* load 3 round keys */
aese.8 q0, q15
aesmc.8 q0, q0
b .Lenc1_tail
.size _gcry_aes_enc_armv8_ce,.-_gcry_aes_enc_armv8_ce;
/*
* unsigned int _gcry_aes_dec_armv8_ce(void *keysched, byte *dst,
* const byte *src,
* unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_dec_armv8_ce
.type _gcry_aes_dec_armv8_ce,%function;
_gcry_aes_dec_armv8_ce:
/* input:
* r0: keysched
* r1: dst
* r2: src
* r3: nrounds
*/
vldmia r0!, {q1-q3} /* load 3 round keys */
cmp r3, #12
vld1.8 {q0}, [r2]
bhi .Ldec1_256
beq .Ldec1_192
.Ldec1_128:
.Ldec1_tail:
vldmia r0, {q8-q15} /* load 8 round keys */
aesd.8 q0, q1
aesimc.8 q0, q0
CLEAR_REG(q1)
aesd.8 q0, q2
aesimc.8 q0, q0
CLEAR_REG(q2)
aesd.8 q0, q3
aesimc.8 q0, q0
CLEAR_REG(q3)
aesd.8 q0, q8
aesimc.8 q0, q0
CLEAR_REG(q8)
aesd.8 q0, q9
aesimc.8 q0, q0
CLEAR_REG(q9)
aesd.8 q0, q10
aesimc.8 q0, q0
CLEAR_REG(q10)
aesd.8 q0, q11
aesimc.8 q0, q0
CLEAR_REG(q11)
aesd.8 q0, q12
aesimc.8 q0, q0
CLEAR_REG(q12)
aesd.8 q0, q13
aesimc.8 q0, q0
CLEAR_REG(q13)
aesd.8 q0, q14
veor q0, q15
CLEAR_REG(q14)
CLEAR_REG(q15)
vst1.8 {q0}, [r1]
CLEAR_REG(q0)
mov r0, #0
bx lr
.Ldec1_192:
aesd.8 q0, q1
aesimc.8 q0, q0
vmov q1, q3
aesd.8 q0, q2
aesimc.8 q0, q0
vldm r0!, {q2-q3} /* load 3 round keys */
b .Ldec1_tail
.Ldec1_256:
vldm r0!, {q15} /* load 1 round key */
aesd.8 q0, q1
aesimc.8 q0, q0
aesd.8 q0, q2
aesimc.8 q0, q0
aesd.8 q0, q3
aesimc.8 q0, q0
vldm r0!, {q1-q3} /* load 3 round keys */
aesd.8 q0, q15
aesimc.8 q0, q0
b .Ldec1_tail
.size _gcry_aes_dec_armv8_ce,.-_gcry_aes_dec_armv8_ce;
/*
* void _gcry_aes_cbc_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, size_t nblocks,
* int cbc_mac, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_cbc_enc_armv8_ce
.type _gcry_aes_cbc_enc_armv8_ce,%function;
_gcry_aes_cbc_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: cbc_mac => r5
* st+8: nrounds => r6
*/
push {r4-r6,lr} /* 4*4 = 16b */
ldr r4, [sp, #(16+0)]
ldr r5, [sp, #(16+4)]
cmp r4, #0
ldr r6, [sp, #(16+8)]
beq .Lcbc_enc_skip
cmp r5, #0
vpush {q4-q7}
moveq r5, #16
movne r5, #0
cmp r6, #12
vld1.8 {q1}, [r3] /* load IV */
aes_preload_keys(r0, lr);
beq .Lcbc_enc_loop192
bhi .Lcbc_enc_loop256
#define CBC_ENC(bits, ...) \
.Lcbc_enc_loop##bits: \
vld1.8 {q0}, [r2]!; /* load plaintext */ \
veor q1, q0, q1; \
subs r4, r4, #1; \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
vst1.8 {q1}, [r1], r5; /* store ciphertext */ \
\
bne .Lcbc_enc_loop##bits; \
b .Lcbc_enc_done;
CBC_ENC(128)
CBC_ENC(192, r0, lr)
CBC_ENC(256, r0, lr)
#undef CBC_ENC
.Lcbc_enc_done:
vst1.8 {q1}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lcbc_enc_skip:
pop {r4-r6,pc}
.size _gcry_aes_cbc_enc_armv8_ce,.-_gcry_aes_cbc_enc_armv8_ce;
/*
* void _gcry_aes_cbc_dec_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_cbc_dec_armv8_ce
.type _gcry_aes_cbc_dec_armv8_ce,%function;
_gcry_aes_cbc_dec_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
push {r4-r6,lr} /* 4*4 = 16b */
ldr r4, [sp, #(16+0)]
ldr r5, [sp, #(16+4)]
cmp r4, #0
beq .Lcbc_dec_skip
vpush {q4-q7}
cmp r5, #12
vld1.8 {q0}, [r3] /* load IV */
aes_preload_keys(r0, r6);
beq .Lcbc_dec_entry_192
bhi .Lcbc_dec_entry_256
#define CBC_DEC(bits, ...) \
.Lcbc_dec_entry_##bits: \
cmp r4, #4; \
blo .Lcbc_dec_loop_##bits; \
\
.Lcbc_dec_loop4_##bits: \
\
vld1.8 {q1-q2}, [r2]!; /* load ciphertext */ \
sub r4, r4, #4; \
vld1.8 {q3-q4}, [r2]; /* load ciphertext */ \
cmp r4, #4; \
sub r2, #32; \
\
do_aes_4_##bits(d, imc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
veor q1, q1, q0; \
vld1.8 {q0}, [r2]!; /* load next IV */ \
veor q2, q2, q0; \
vld1.8 {q0}, [r2]!; /* load next IV */ \
vst1.8 {q1-q2}, [r1]!; /* store plaintext */ \
veor q3, q3, q0; \
vld1.8 {q0}, [r2]!; /* load next IV */ \
veor q4, q4, q0; \
vld1.8 {q0}, [r2]!; /* load next IV */ \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lcbc_dec_loop4_##bits; \
cmp r4, #0; \
beq .Lcbc_dec_done; \
\
.Lcbc_dec_loop_##bits: \
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
subs r4, r4, #1; \
vmov q2, q1; \
\
do_aes_one##bits(d, imc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q1, q0; \
vmov q0, q2; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
\
bne .Lcbc_dec_loop_##bits; \
b .Lcbc_dec_done;
CBC_DEC(128)
CBC_DEC(192, r0, r6)
CBC_DEC(256, r0, r6)
#undef CBC_DEC
.Lcbc_dec_done:
vst1.8 {q0}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lcbc_dec_skip:
pop {r4-r6,pc}
.size _gcry_aes_cbc_dec_armv8_ce,.-_gcry_aes_cbc_dec_armv8_ce;
/*
* void _gcry_aes_ecb_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* size_t nblocks,
* unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_ecb_enc_armv8_ce
.type _gcry_aes_ecb_enc_armv8_ce,%function;
_gcry_aes_ecb_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: nblocks
* st+0: nrounds => r4
*/
push {r4-r6,lr} /* 4*4 = 16b */
cmp r3, #0
beq .Lecb_enc_skip
ldr r4, [sp, #(16+0)]
vpush {q4-q7}
cmp r4, #12
aes_preload_keys(r0, lr);
beq .Lecb_entry_192e
bhi .Lecb_entry_256e
#define ECB_CRYPT(bits, e_d, mc_imc, ...) \
.Lecb_entry_##bits##e_d: \
cmp r3, #4; \
blo .Lecb_loop_##bits##e_d; \
\
.Lecb_loop4_##bits##e_d: \
vld1.8 {q1-q2}, [r2]!; /* load plaintext */ \
sub r3, r3, #4; \
vld1.8 {q3-q4}, [r2]!; /* load plaintext */ \
cmp r3, #4; \
\
do_aes_4_##bits(e_d, mc_imc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
vst1.8 {q1-q2}, [r1]!; /* store ciphertext */ \
vst1.8 {q3-q4}, [r1]!; /* store ciphertext */ \
\
bhs .Lecb_loop4_##bits##e_d; \
cmp r3, #0; \
beq .Lecb_done_##e_d; \
\
.Lecb_loop_##bits##e_d: \
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
subs r3, r3, #1; \
\
do_aes_one##bits(e_d, mc_imc, q1, q1, ##__VA_ARGS__); \
\
vst1.8 {q1}, [r1]!; /* store plaintext */ \
bne .Lecb_loop_##bits##e_d; \
b .Lecb_done_##e_d;
ECB_CRYPT(128, e, mc)
ECB_CRYPT(192, e, mc, r0, lr)
ECB_CRYPT(256, e, mc, r0, lr)
.Lecb_done_e:
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lecb_enc_skip:
pop {r4-r6,pc}
.size _gcry_aes_ecb_enc_armv8_ce,.-_gcry_aes_ecb_enc_armv8_ce;
/*
* void _gcry_aes_ecb_dec_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* size_t nblocks,
* unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_ecb_dec_armv8_ce
.type _gcry_aes_ecb_dec_armv8_ce,%function;
_gcry_aes_ecb_dec_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: nblocks
* st+0: nrounds => r4
*/
push {r4-r6,lr} /* 4*4 = 16b */
cmp r3, #0
beq .Lecb_enc_skip
ldr r4, [sp, #(16+0)]
vpush {q4-q7}
cmp r4, #12
aes_preload_keys(r0, lr);
beq .Lecb_entry_192d
bhi .Lecb_entry_256d
ECB_CRYPT(128, d, imc)
ECB_CRYPT(192, d, imc, r0, lr)
ECB_CRYPT(256, d, imc, r0, lr)
#undef ECB_CRYPT
.Lecb_done_d:
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lecb_dec_skip:
pop {r4-r6,pc}
.size _gcry_aes_ecb_dec_armv8_ce,.-_gcry_aes_ecb_dec_armv8_ce;
/*
* void _gcry_aes_cfb_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_cfb_enc_armv8_ce
.type _gcry_aes_cfb_enc_armv8_ce,%function;
_gcry_aes_cfb_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
push {r4-r6,lr} /* 4*4 = 16b */
ldr r4, [sp, #(16+0)]
ldr r5, [sp, #(16+4)]
cmp r4, #0
beq .Lcfb_enc_skip
vpush {q4-q7}
cmp r5, #12
vld1.8 {q0}, [r3] /* load IV */
aes_preload_keys(r0, r6);
beq .Lcfb_enc_entry_192
bhi .Lcfb_enc_entry_256
#define CFB_ENC(bits, ...) \
.Lcfb_enc_entry_##bits: \
.Lcfb_enc_loop_##bits: \
vld1.8 {q1}, [r2]!; /* load plaintext */ \
subs r4, r4, #1; \
\
do_aes_one##bits(e, mc, q0, q0, ##__VA_ARGS__); \
\
veor q0, q1, q0; \
vst1.8 {q0}, [r1]!; /* store ciphertext */ \
\
bne .Lcfb_enc_loop_##bits; \
b .Lcfb_enc_done;
CFB_ENC(128)
CFB_ENC(192, r0, r6)
CFB_ENC(256, r0, r6)
#undef CFB_ENC
.Lcfb_enc_done:
vst1.8 {q0}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lcfb_enc_skip:
pop {r4-r6,pc}
.size _gcry_aes_cfb_enc_armv8_ce,.-_gcry_aes_cfb_enc_armv8_ce;
/*
* void _gcry_aes_cfb_dec_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_cfb_dec_armv8_ce
.type _gcry_aes_cfb_dec_armv8_ce,%function;
_gcry_aes_cfb_dec_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
push {r4-r6,lr} /* 4*4 = 16b */
ldr r4, [sp, #(16+0)]
ldr r5, [sp, #(16+4)]
cmp r4, #0
beq .Lcfb_dec_skip
vpush {q4-q7}
cmp r5, #12
vld1.8 {q0}, [r3] /* load IV */
aes_preload_keys(r0, r6);
beq .Lcfb_dec_entry_192
bhi .Lcfb_dec_entry_256
#define CFB_DEC(bits, ...) \
.Lcfb_dec_entry_##bits: \
cmp r4, #4; \
blo .Lcfb_dec_loop_##bits; \
\
.Lcfb_dec_loop4_##bits: \
\
vld1.8 {q2-q3}, [r2]!; /* load ciphertext */ \
vmov q1, q0; \
sub r4, r4, #4; \
vld1.8 {q4}, [r2]; /* load ciphertext */ \
sub r2, #32; \
cmp r4, #4; \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
veor q1, q1, q0; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
veor q2, q2, q0; \
vst1.8 {q1-q2}, [r1]!; /* store plaintext */ \
vld1.8 {q0}, [r2]!; \
veor q3, q3, q0; \
vld1.8 {q0}, [r2]!; /* load next IV / ciphertext */ \
veor q4, q4, q0; \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lcfb_dec_loop4_##bits; \
cmp r4, #0; \
beq .Lcfb_dec_done; \
\
.Lcfb_dec_loop_##bits: \
\
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
\
subs r4, r4, #1; \
\
do_aes_one##bits(e, mc, q0, q0, ##__VA_ARGS__); \
\
veor q2, q1, q0; \
vmov q0, q1; \
vst1.8 {q2}, [r1]!; /* store plaintext */ \
\
bne .Lcfb_dec_loop_##bits; \
b .Lcfb_dec_done;
CFB_DEC(128)
CFB_DEC(192, r0, r6)
CFB_DEC(256, r0, r6)
#undef CFB_DEC
.Lcfb_dec_done:
vst1.8 {q0}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
vpop {q4-q7}
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lcfb_dec_skip:
pop {r4-r6,pc}
.size _gcry_aes_cfb_dec_armv8_ce,.-_gcry_aes_cfb_dec_armv8_ce;
/*
* void _gcry_aes_ctr_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_ctr_enc_armv8_ce
.type _gcry_aes_ctr_enc_armv8_ce,%function;
_gcry_aes_ctr_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
cmp r4, #0
beq .Lctr_enc_skip
cmp r5, #12
ldm r3, {r7-r10}
vld1.8 {q0}, [r3] /* load IV */
rev r7, r7
rev r8, r8
rev r9, r9
rev r10, r10
aes_preload_keys(r0, r6);
beq .Lctr_enc_entry_192
bhi .Lctr_enc_entry_256
#define CTR_ENC(bits, ...) \
.Lctr_enc_entry_##bits: \
cmp r4, #4; \
blo .Lctr_enc_loop_##bits; \
\
.Lctr_enc_loop4_##bits: \
cmp r10, #0xfffffffc; \
sub r4, r4, #4; \
blo .Lctr_enc_loop4_##bits##_nocarry; \
cmp r9, #0xffffffff; \
bne .Lctr_enc_loop4_##bits##_nocarry; \
\
adds r10, #1; \
vmov q1, q0; \
blcs .Lctr_overflow_one; \
rev r11, r10; \
vmov.32 d1[1], r11; \
\
adds r10, #1; \
vmov q2, q0; \
blcs .Lctr_overflow_one; \
rev r11, r10; \
vmov.32 d1[1], r11; \
\
adds r10, #1; \
vmov q3, q0; \
blcs .Lctr_overflow_one; \
rev r11, r10; \
vmov.32 d1[1], r11; \
\
adds r10, #1; \
vmov q4, q0; \
blcs .Lctr_overflow_one; \
rev r11, r10; \
vmov.32 d1[1], r11; \
\
b .Lctr_enc_loop4_##bits##_store_ctr; \
\
.Lctr_enc_loop4_##bits##_nocarry: \
\
veor q2, q2; \
vrev64.8 q1, q0; \
vceq.u32 d5, d5; \
vadd.u64 q3, q2, q2; \
vadd.u64 q4, q3, q2; \
vadd.u64 q0, q3, q3; \
vsub.u64 q2, q1, q2; \
vsub.u64 q3, q1, q3; \
vsub.u64 q4, q1, q4; \
vsub.u64 q0, q1, q0; \
vrev64.8 q1, q1; \
vrev64.8 q2, q2; \
vrev64.8 q3, q3; \
vrev64.8 q0, q0; \
vrev64.8 q4, q4; \
add r10, #4; \
\
.Lctr_enc_loop4_##bits##_store_ctr: \
\
vst1.8 {q0}, [r3]; \
cmp r4, #4; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
veor q1, q1, q0; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
veor q2, q2, q0; \
veor q3, q3, q1; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
vst1.8 {q2}, [r1]!; /* store plaintext */ \
veor q4, q4, q0; \
vld1.8 {q0}, [r3]; /* reload IV */ \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lctr_enc_loop4_##bits; \
cmp r4, #0; \
beq .Lctr_enc_done; \
\
.Lctr_enc_loop_##bits: \
\
adds r10, #1; \
vmov q1, q0; \
blcs .Lctr_overflow_one; \
rev r11, r10; \
subs r4, r4, #1; \
vld1.8 {q2}, [r2]!; /* load ciphertext */ \
vmov.32 d1[1], r11; \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q2, q1; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
\
bne .Lctr_enc_loop_##bits; \
b .Lctr_enc_done;
CTR_ENC(128)
CTR_ENC(192, r0, r6)
CTR_ENC(256, r0, r6)
#undef CTR_ENC
.Lctr_enc_done:
vst1.8 {q0}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lctr_enc_skip:
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.Lctr_overflow_one:
adcs r9, #0
adcs r8, #0
adc r7, #0
rev r11, r9
rev r12, r8
vmov.32 d1[0], r11
rev r11, r7
vmov.32 d0[1], r12
vmov.32 d0[0], r11
bx lr
.size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce;
/*
* void _gcry_aes_ctr32le_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv,
* unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_ctr32le_enc_armv8_ce
.type _gcry_aes_ctr32le_enc_armv8_ce,%function;
_gcry_aes_ctr32le_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
cmp r4, #0
beq .Lctr32le_enc_skip
cmp r5, #12
vld1.8 {q0}, [r3] /* load IV */
aes_preload_keys(r0, r6);
beq .Lctr32le_enc_entry_192
bhi .Lctr32le_enc_entry_256
#define CTR_ENC(bits, ...) \
.Lctr32le_enc_entry_##bits: \
cmp r4, #4; \
blo .Lctr32le_enc_loop_##bits; \
\
.Lctr32le_enc_loop4_##bits: \
veor q2, q2; \
sub r4, r4, #4; \
vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \
vmov q1, q0; \
vadd.u32 q3, q2, q2; /* q3 <= -2:0:0:0 */ \
vadd.u32 q0, q3, q3; /* q0 <= -4:0:0:0 */ \
vadd.u32 q4, q3, q2; /* q4 <= -3:0:0:0 */ \
vsub.u32 q0, q1, q0; \
vsub.u32 q2, q1, q2; \
vst1.8 {q0}, [r3]; \
vsub.u32 q3, q1, q3; \
vsub.u32 q4, q1, q4; \
\
cmp r4, #4; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
veor q1, q1, q0; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
veor q2, q2, q0; \
veor q3, q3, q1; \
vld1.8 {q0}, [r2]!; /* load ciphertext */ \
vst1.8 {q2}, [r1]!; /* store plaintext */ \
veor q4, q4, q0; \
vld1.8 {q0}, [r3]; /* reload IV */ \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lctr32le_enc_loop4_##bits; \
cmp r4, #0; \
beq .Lctr32le_enc_done; \
\
.Lctr32le_enc_loop_##bits: \
\
veor q2, q2; \
vmov q1, q0; \
vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \
subs r4, r4, #1; \
vsub.u32 q0, q0, q2; \
vld1.8 {q2}, [r2]!; /* load ciphertext */ \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q2, q1; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
\
bne .Lctr32le_enc_loop_##bits; \
b .Lctr32le_enc_done;
CTR_ENC(128)
CTR_ENC(192, r0, r6)
CTR_ENC(256, r0, r6)
#undef CTR_ENC
.Lctr32le_enc_done:
vst1.8 {q0}, [r3] /* store IV */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lctr32le_enc_skip:
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_ctr32le_enc_armv8_ce,.-_gcry_aes_ctr32le_enc_armv8_ce;
/*
* long _gcry_aes_ocb_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *offset,
* unsigned char *checksum,
* unsigned char *L_table,
* size_t nblocks,
* unsigned int nrounds,
* unsigned int blkn);
*/
.align 3
.globl _gcry_aes_ocb_enc_armv8_ce
.type _gcry_aes_ocb_enc_armv8_ce,%function;
_gcry_aes_ocb_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: offset
* st+0: checksum => r4
* st+4: Ls => r5
* st+8: nblocks => r6 (0 < nblocks <= 32)
* st+12: nrounds => r7
* st+16: blkn => lr
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r7, [sp, #(104+12)]
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
ldr r6, [sp, #(104+8)]
ldr lr, [sp, #(104+16)]
cmp r7, #12
vld1.8 {q0}, [r3] /* load offset */
aes_preload_keys(r0, r12);
beq .Locb_enc_entry_192
bhi .Locb_enc_entry_256
#define OCB_ENC(bits, ...) \
.Locb_enc_entry_##bits: \
cmp r6, #4; \
add lr, #1; \
blo .Locb_enc_loop_##bits; \
\
.Locb_enc_loop4_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* Checksum_i = Checksum_{i-1} xor P_i */ \
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ \
\
add r9, lr, #1; \
add r10, lr, #2; \
add r11, lr, #3; \
rbit r8, lr; \
add lr, lr, #4; \
rbit r9, r9; \
rbit r10, r10; \
rbit r11, r11; \
clz r8, r8; /* ntz(i+0) */ \
clz r9, r9; /* ntz(i+1) */ \
clz r10, r10; /* ntz(i+2) */ \
clz r11, r11; /* ntz(i+3) */ \
add r8, r5, r8, lsl #4; \
add r9, r5, r9, lsl #4; \
add r10, r5, r10, lsl #4; \
add r11, r5, r11, lsl #4; \
\
sub r6, #4; \
\
vld1.8 {q9}, [r8]; /* load L_{ntz(i+0)} */ \
vld1.8 {q1-q2}, [r2]!; /* load P_i+<0-1> */ \
vld1.8 {q8}, [r4]; /* load Checksum_{i-1} */ \
veor q0, q0, q9; /* Offset_i+0 */ \
vld1.8 {q9}, [r9]; /* load L_{ntz(i+1)} */ \
veor q8, q8, q1; /* Checksum_i+0 */ \
veor q1, q1, q0; /* P_i+0 xor Offset_i+0 */\
vld1.8 {q3-q4}, [r2]!; /* load P_i+<2-3> */ \
vst1.8 {q0}, [r1]!; /* store Offset_i+0 */\
veor q0, q0, q9; /* Offset_i+1 */ \
vld1.8 {q9}, [r10]; /* load L_{ntz(i+2)} */ \
veor q8, q8, q2; /* Checksum_i+1 */ \
veor q2, q2, q0; /* P_i+1 xor Offset_i+1 */\
vst1.8 {q0}, [r1]!; /* store Offset_i+1 */\
veor q0, q0, q9; /* Offset_i+2 */ \
vld1.8 {q9}, [r11]; /* load L_{ntz(i+3)} */ \
veor q8, q8, q3; /* Checksum_i+2 */ \
veor q3, q3, q0; /* P_i+2 xor Offset_i+2 */\
vst1.8 {q0}, [r1]!; /* store Offset_i+2 */\
veor q0, q0, q9; /* Offset_i+3 */ \
veor q8, q8, q4; /* Checksum_i+3 */ \
veor q4, q4, q0; /* P_i+3 xor Offset_i+3 */\
vst1.8 {q0}, [r1]; /* store Offset_i+3 */\
sub r1, #(3*16); \
vst1.8 {q8}, [r4]; /* store Checksum_i+3 */\
\
cmp r6, #4; \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
mov r8, r1; \
vld1.8 {q8-q9}, [r1]!; \
veor q1, q1, q8; \
veor q2, q2, q9; \
vld1.8 {q8-q9}, [r1]!; \
vst1.8 {q1-q2}, [r8]!; \
veor q3, q3, q8; \
veor q4, q4, q9; \
vst1.8 {q3-q4}, [r8]; \
\
bhs .Locb_enc_loop4_##bits; \
cmp r6, #0; \
beq .Locb_enc_done; \
\
.Locb_enc_loop_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* Checksum_i = Checksum_{i-1} xor P_i */ \
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ \
\
rbit r8, lr; \
add lr, #1; \
clz r8, r8; /* ntz(i) */ \
add r8, r5, r8, lsl #4; \
\
vld1.8 {q1}, [r2]!; /* load plaintext */ \
vld1.8 {q2}, [r8]; /* load L_{ntz(i)} */ \
vld1.8 {q3}, [r4]; /* load checksum */ \
subs r6, #1; \
veor q0, q0, q2; \
veor q3, q3, q1; \
veor q1, q1, q0; \
vst1.8 {q3}, [r4]; /* store checksum */ \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q1, q0; \
vst1.8 {q1}, [r1]!; /* store ciphertext */ \
\
bne .Locb_enc_loop_##bits; \
b .Locb_enc_done;
OCB_ENC(128re, r0, r12)
OCB_ENC(192, r0, r12)
OCB_ENC(256, r0, r12)
#undef OCB_ENC
.Locb_enc_done:
vst1.8 {q0}, [r3] /* store offset */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
mov r0, #0
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_ocb_enc_armv8_ce,.-_gcry_aes_ocb_enc_armv8_ce;
/*
* long _gcry_aes_ocb_dec_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *offset,
* unsigned char *checksum,
* unsigned char *L_table,
* size_t nblocks,
* unsigned int nrounds,
* unsigned int blkn);
*/
.align 3
.globl _gcry_aes_ocb_dec_armv8_ce
.type _gcry_aes_ocb_dec_armv8_ce,%function;
_gcry_aes_ocb_dec_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: offset
* st+0: checksum => r4
* st+4: Ls => r5
* st+8: nblocks => r6 (0 < nblocks <= 32)
* st+12: nrounds => r7
* st+16: blkn => lr
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r7, [sp, #(104+12)]
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
ldr r6, [sp, #(104+8)]
ldr lr, [sp, #(104+16)]
cmp r7, #12
vld1.8 {q0}, [r3] /* load offset */
aes_preload_keys(r0, r12);
beq .Locb_dec_entry_192
bhi .Locb_dec_entry_256
#define OCB_DEC(bits, ...) \
.Locb_dec_entry_##bits: \
cmp r6, #4; \
add lr, #1; \
blo .Locb_dec_loop_##bits; \
\
.Locb_dec_loop4_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ \
/* Checksum_i = Checksum_{i-1} xor P_i */ \
\
add r9, lr, #1; \
add r10, lr, #2; \
add r11, lr, #3; \
rbit r8, lr; \
add lr, lr, #4; \
rbit r9, r9; \
rbit r10, r10; \
rbit r11, r11; \
clz r8, r8; /* ntz(i+0) */ \
clz r9, r9; /* ntz(i+1) */ \
clz r10, r10; /* ntz(i+2) */ \
clz r11, r11; /* ntz(i+3) */ \
add r8, r5, r8, lsl #4; \
add r9, r5, r9, lsl #4; \
add r10, r5, r10, lsl #4; \
add r11, r5, r11, lsl #4; \
\
sub r6, #4; \
\
vld1.8 {q9}, [r8]; /* load L_{ntz(i+0)} */ \
vld1.8 {q1-q2}, [r2]!; /* load P_i+<0-1> */ \
veor q0, q0, q9; /* Offset_i+0 */ \
vld1.8 {q9}, [r9]; /* load L_{ntz(i+1)} */ \
veor q1, q1, q0; /* P_i+0 xor Offset_i+0 */\
vld1.8 {q3-q4}, [r2]!; /* load P_i+<2-3> */ \
vst1.8 {q0}, [r1]!; /* store Offset_i+0 */\
veor q0, q0, q9; /* Offset_i+1 */ \
vld1.8 {q9}, [r10]; /* load L_{ntz(i+2)} */ \
veor q2, q2, q0; /* P_i+1 xor Offset_i+1 */\
vst1.8 {q0}, [r1]!; /* store Offset_i+1 */\
veor q0, q0, q9; /* Offset_i+2 */ \
vld1.8 {q9}, [r11]; /* load L_{ntz(i+3)} */ \
veor q3, q3, q0; /* P_i+2 xor Offset_i+2 */\
vst1.8 {q0}, [r1]!; /* store Offset_i+2 */\
veor q0, q0, q9; /* Offset_i+3 */ \
veor q4, q4, q0; /* P_i+3 xor Offset_i+3 */\
vst1.8 {q0}, [r1]; /* store Offset_i+3 */\
sub r1, #(3*16); \
\
cmp r6, #4; \
\
do_aes_4_##bits(d, imc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
mov r8, r1; \
vld1.8 {q8-q9}, [r1]!; \
veor q1, q1, q8; \
veor q2, q2, q9; \
vld1.8 {q8-q9}, [r1]!; \
vst1.8 {q1-q2}, [r8]!; \
veor q1, q1, q2; \
vld1.8 {q2}, [r4]; /* load Checksum_{i-1} */ \
veor q3, q3, q8; \
veor q1, q1, q3; \
veor q4, q4, q9; \
veor q1, q1, q4; \
vst1.8 {q3-q4}, [r8]; \
veor q2, q2, q1; \
vst1.8 {q2}, [r4]; /* store Checksum_i+3 */ \
\
bhs .Locb_dec_loop4_##bits; \
cmp r6, #0; \
beq .Locb_dec_done; \
\
.Locb_dec_loop_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ \
/* Checksum_i = Checksum_{i-1} xor P_i */ \
\
rbit r8, lr; \
add lr, #1; \
clz r8, r8; /* ntz(i) */ \
add r8, r5, r8, lsl #4; \
\
vld1.8 {q2}, [r8]; /* load L_{ntz(i)} */ \
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
subs r6, #1; \
veor q0, q0, q2; \
veor q1, q1, q0; \
\
do_aes_one##bits(d, imc, q1, q1, ##__VA_ARGS__) \
\
vld1.8 {q2}, [r4]; /* load checksum */ \
veor q1, q1, q0; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
veor q2, q2, q1; \
vst1.8 {q2}, [r4]; /* store checksum */ \
\
bne .Locb_dec_loop_##bits; \
b .Locb_dec_done;
OCB_DEC(128re, r0, r12)
OCB_DEC(192, r0, r12)
OCB_DEC(256, r0, r12)
#undef OCB_DEC
.Locb_dec_done:
vst1.8 {q0}, [r3] /* store offset */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
mov r0, #0
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_ocb_dec_armv8_ce,.-_gcry_aes_ocb_dec_armv8_ce;
/*
* long _gcry_aes_ocb_auth_armv8_ce (const void *keysched,
* const unsigned char *abuf,
* unsigned char *offset,
* unsigned char *checksum,
* unsigned char *L_table,
* size_t nblocks,
* unsigned int nrounds,
* unsigned int blkn);
*/
.align 3
.globl _gcry_aes_ocb_auth_armv8_ce
.type _gcry_aes_ocb_auth_armv8_ce,%function;
_gcry_aes_ocb_auth_armv8_ce:
/* input:
* r0: keysched
* r1: abuf
* r2: offset
* r3: checksum
* st+0: Ls => r5
* st+4: nblocks => r6 (0 < nblocks <= 32)
* st+8: nrounds => r7
* st+12: blkn => lr
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r7, [sp, #(104+8)]
ldr r5, [sp, #(104+0)]
ldr r6, [sp, #(104+4)]
ldr lr, [sp, #(104+12)]
cmp r7, #12
vld1.8 {q0}, [r2] /* load offset */
aes_preload_keys(r0, r12);
beq .Locb_auth_entry_192
bhi .Locb_auth_entry_256
#define OCB_AUTH(bits, ...) \
.Locb_auth_entry_##bits: \
cmp r6, #4; \
add lr, #1; \
blo .Locb_auth_loop_##bits; \
\
.Locb_auth_loop4_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ \
\
add r9, lr, #1; \
add r10, lr, #2; \
add r11, lr, #3; \
rbit r8, lr; \
add lr, lr, #4; \
rbit r9, r9; \
rbit r10, r10; \
rbit r11, r11; \
clz r8, r8; /* ntz(i+0) */ \
clz r9, r9; /* ntz(i+1) */ \
clz r10, r10; /* ntz(i+2) */ \
clz r11, r11; /* ntz(i+3) */ \
add r8, r5, r8, lsl #4; \
add r9, r5, r9, lsl #4; \
add r10, r5, r10, lsl #4; \
add r11, r5, r11, lsl #4; \
\
sub r6, #4; \
\
vld1.8 {q9}, [r8]; /* load L_{ntz(i+0)} */ \
vld1.8 {q1-q2}, [r1]!; /* load A_i+<0-1> */ \
veor q0, q0, q9; /* Offset_i+0 */ \
vld1.8 {q9}, [r9]; /* load L_{ntz(i+1)} */ \
veor q1, q1, q0; /* A_i+0 xor Offset_i+0 */\
vld1.8 {q3-q4}, [r1]!; /* load A_i+<2-3> */ \
veor q0, q0, q9; /* Offset_i+1 */ \
vld1.8 {q9}, [r10]; /* load L_{ntz(i+2)} */ \
veor q2, q2, q0; /* A_i+1 xor Offset_i+1 */\
veor q0, q0, q9; /* Offset_i+2 */ \
vld1.8 {q9}, [r11]; /* load L_{ntz(i+3)} */ \
veor q3, q3, q0; /* A_i+2 xor Offset_i+2 */\
veor q0, q0, q9; /* Offset_i+3 */ \
veor q4, q4, q0; /* A_i+3 xor Offset_i+3 */\
\
cmp r6, #4; \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
veor q1, q1, q2; \
veor q3, q3, q4; \
vld1.8 {q2}, [r3]; \
veor q1, q1, q3; \
veor q2, q2, q1; \
vst1.8 {q2}, [r3]; \
\
bhs .Locb_auth_loop4_##bits; \
cmp r6, #0; \
beq .Locb_auth_done; \
\
.Locb_auth_loop_##bits: \
\
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ \
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ \
\
rbit r8, lr; \
add lr, #1; \
clz r8, r8; /* ntz(i) */ \
add r8, r5, r8, lsl #4; \
\
vld1.8 {q2}, [r8]; /* load L_{ntz(i)} */ \
vld1.8 {q1}, [r1]!; /* load aadtext */ \
subs r6, #1; \
veor q0, q0, q2; \
vld1.8 {q2}, [r3]; /* load checksum */ \
veor q1, q1, q0; \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__) \
\
veor q2, q2, q1; \
vst1.8 {q2}, [r3]; /* store checksum */ \
\
bne .Locb_auth_loop_##bits; \
b .Locb_auth_done;
OCB_AUTH(128re, r0, r12)
OCB_AUTH(192, r0, r12)
OCB_AUTH(256, r0, r12)
#undef OCB_AUTH
.Locb_auth_done:
vst1.8 {q0}, [r2] /* store offset */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
mov r0, #0
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_ocb_auth_armv8_ce,.-_gcry_aes_ocb_auth_armv8_ce;
/*
* void _gcry_aes_xts_enc_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_xts_enc_armv8_ce
.type _gcry_aes_xts_enc_armv8_ce,%function;
_gcry_aes_xts_enc_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
cmp r4, #0
beq .Lxts_enc_skip
cmp r5, #12
vld1.8 {q0}, [r3] /* load tweak */
mov r7, #0x87;
aes_preload_keys(r0, r6);
beq .Lxts_enc_entry_192
bhi .Lxts_enc_entry_256
#define CTR_XTS(bits, ...) \
.Lxts_enc_entry_##bits: \
cmp r4, #4; \
blo .Lxts_enc_loop_##bits; \
\
.Lxts_enc_loop4_##bits: \
sub r4, r4, #4; \
veor q9, q9, q9; \
\
vld1.8 {q1-q2}, [r2]!; /* load plaintext */ \
veor q1, q1, q0; \
cmp r4, #4; \
vmov.u32 d18[0], r7; \
vst1.8 {q0}, [r1]!; /* store tweak0 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
vld1.8 {q3-q4}, [r2]!; /* load plaintext */ \
veor q2, q2, q0; \
vst1.8 {q0}, [r1]!; /* store tweak1 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
veor q3, q3, q0; \
vst1.8 {q0}, [r1]!; /* store tweak2 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
veor q4, q4, q0; \
vst1.8 {q0}, [r1]; /* store tweak3 to temp */ \
sub r1, r1, #48; \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
do_aes_4_##bits(e, mc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
vld1.8 {q8-q9}, [r1]!; /* load tweak from temp */ \
veor q1, q1, q8; \
veor q2, q2, q9; \
vld1.8 {q8-q9}, [r1]; /* load tweak from temp */ \
sub r1, r1, #32; \
veor q3, q3, q8; \
veor q4, q4, q9; \
vst1.8 {q1-q2}, [r1]!; /* store plaintext */ \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lxts_enc_loop4_##bits; \
cmp r4, #0; \
beq .Lxts_enc_done; \
\
.Lxts_enc_loop_##bits: \
\
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
\
veor q9, q9, q9; \
veor q1, q1, q0; \
vmov.u32 d18[0], r7; \
vmov q2, q0; \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
subs r4, r4, #1; \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q1, q2; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
\
bne .Lxts_enc_loop_##bits; \
b .Lxts_enc_done;
CTR_XTS(128re, r0, r6)
CTR_XTS(192, r0, r6)
CTR_XTS(256, r0, r6)
#undef CTR_XTS
.Lxts_enc_done:
vst1.8 {q0}, [r3] /* store tweak */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lxts_enc_skip:
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_xts_enc_armv8_ce,.-_gcry_aes_xts_enc_armv8_ce;
/*
* void _gcry_aes_xts_dec_armv8_ce (const void *keysched,
* unsigned char *outbuf,
* const unsigned char *inbuf,
* unsigned char *iv, unsigned int nrounds);
*/
.align 3
.globl _gcry_aes_xts_dec_armv8_ce
.type _gcry_aes_xts_dec_armv8_ce,%function;
_gcry_aes_xts_dec_armv8_ce:
/* input:
* r0: keysched
* r1: outbuf
* r2: inbuf
* r3: iv
* st+0: nblocks => r4
* st+4: nrounds => r5
*/
vpush {q4-q7}
push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
ldr r4, [sp, #(104+0)]
ldr r5, [sp, #(104+4)]
cmp r4, #0
beq .Lxts_dec_skip
cmp r5, #12
vld1.8 {q0}, [r3] /* load tweak */
mov r7, #0x87;
aes_preload_keys(r0, r6);
beq .Lxts_dec_entry_192
bhi .Lxts_dec_entry_256
#define CTR_XTS(bits, ...) \
.Lxts_dec_entry_##bits: \
cmp r4, #4; \
blo .Lxts_dec_loop_##bits; \
\
.Lxts_dec_loop4_##bits: \
sub r4, r4, #4; \
veor q9, q9, q9; \
\
vld1.8 {q1-q2}, [r2]!; /* load plaintext */ \
veor q1, q1, q0; \
cmp r4, #4; \
vmov.u32 d18[0], r7; \
vst1.8 {q0}, [r1]!; /* store tweak0 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
vld1.8 {q3-q4}, [r2]!; /* load plaintext */ \
veor q2, q2, q0; \
vst1.8 {q0}, [r1]!; /* store tweak1 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
veor q3, q3, q0; \
vst1.8 {q0}, [r1]!; /* store tweak2 to temp */ \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
veor q4, q4, q0; \
vst1.8 {q0}, [r1]; /* store tweak3 to temp */ \
sub r1, r1, #48; \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
\
do_aes_4_##bits(d, imc, q1, q2, q3, q4, ##__VA_ARGS__); \
\
vld1.8 {q8-q9}, [r1]!; /* load tweak from temp */ \
veor q1, q1, q8; \
veor q2, q2, q9; \
vld1.8 {q8-q9}, [r1]; /* load tweak from temp */ \
sub r1, r1, #32; \
veor q3, q3, q8; \
veor q4, q4, q9; \
vst1.8 {q1-q2}, [r1]!; /* store plaintext */ \
vst1.8 {q3-q4}, [r1]!; /* store plaintext */ \
\
bhs .Lxts_dec_loop4_##bits; \
cmp r4, #0; \
beq .Lxts_dec_done; \
\
.Lxts_dec_loop_##bits: \
\
vld1.8 {q1}, [r2]!; /* load ciphertext */ \
\
veor q9, q9, q9; \
veor q1, q1, q0; \
vmov.u32 d18[0], r7; \
vmov q2, q0; \
\
vshr.s64 d16, d1, #63; \
vshr.u64 d17, d0, #63; \
vadd.u64 q0, q0, q0; \
vand d16, d16, d18; \
veor q0, q0, q8; \
subs r4, r4, #1; \
\
do_aes_one##bits(d, imc, q1, q1, ##__VA_ARGS__); \
\
veor q1, q1, q2; \
vst1.8 {q1}, [r1]!; /* store plaintext */ \
\
bne .Lxts_dec_loop_##bits; \
b .Lxts_dec_done;
CTR_XTS(128re, r0, r6)
CTR_XTS(192, r0, r6)
CTR_XTS(256, r0, r6)
#undef CTR_XTS
.Lxts_dec_done:
vst1.8 {q0}, [r3] /* store tweak */
CLEAR_REG(q0)
CLEAR_REG(q1)
CLEAR_REG(q2)
CLEAR_REG(q3)
CLEAR_REG(q8)
CLEAR_REG(q9)
CLEAR_REG(q10)
CLEAR_REG(q11)
CLEAR_REG(q12)
CLEAR_REG(q13)
CLEAR_REG(q14)
.Lxts_dec_skip:
pop {r4-r12,lr}
vpop {q4-q7}
bx lr
.size _gcry_aes_xts_dec_armv8_ce,.-_gcry_aes_xts_dec_armv8_ce;
/*
* u32 _gcry_aes_sbox4_armv8_ce(u32 in4b);
*/
.align 3
.globl _gcry_aes_sbox4_armv8_ce
.type _gcry_aes_sbox4_armv8_ce,%function;
_gcry_aes_sbox4_armv8_ce:
/* See "Gouvêa, C. P. L. & López, J. Implementing GCM on ARMv8. Topics in
* Cryptology — CT-RSA 2015" for details.
*/
vmov.i8 q0, #0x52
vmov.i8 q1, #0
vmov s0, r0
aese.8 q0, q1
veor d0, d1
vpadd.i32 d0, d0, d1
vmov r0, s0
CLEAR_REG(q0)
bx lr
.size _gcry_aes_sbox4_armv8_ce,.-_gcry_aes_sbox4_armv8_ce;
/*
* void _gcry_aes_invmixcol_armv8_ce(void *dst, const void *src);
*/
.align 3
.globl _gcry_aes_invmixcol_armv8_ce
.type _gcry_aes_invmixcol_armv8_ce,%function;
_gcry_aes_invmixcol_armv8_ce:
vld1.8 {q0}, [r1]
aesimc.8 q0, q0
vst1.8 {q0}, [r0]
CLEAR_REG(q0)
bx lr
.size _gcry_aes_invmixcol_armv8_ce,.-_gcry_aes_invmixcol_armv8_ce;
#endif
Reference in new issue View Git Blame Copy Permalink