[src/trunk]: src/sys/crypto/aes/arch/x86 Implement AES-CCM with SSSE3.

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/3e671acf38ea
branches:  trunk
changeset: 1012183:3e671acf38ea
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sat Jul 25 22:31:04 2020 +0000

description:
Implement AES-CCM with SSSE3.

diffstat:

 sys/crypto/aes/arch/x86/aes_ssse3.h      |   8 +++-
 sys/crypto/aes/arch/x86/aes_ssse3_impl.c |  40 ++++++++++++++++-
 sys/crypto/aes/arch/x86/aes_ssse3_subr.c |  73 +++++++++++++++++++++++++++++++-
 sys/crypto/aes/arch/x86/immintrin.h      |  11 ++++-
 4 files changed, 126 insertions(+), 6 deletions(-)

diffs (221 lines):

diff -r 2d391223e03a -r 3e671acf38ea sys/crypto/aes/arch/x86/aes_ssse3.h
--- a/sys/crypto/aes/arch/x86/aes_ssse3.h       Sat Jul 25 22:29:56 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ssse3.h       Sat Jul 25 22:31:04 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ssse3.h,v 1.2 2020/07/25 22:12:57 riastradh Exp $  */
+/*     $NetBSD: aes_ssse3.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -59,6 +59,12 @@
     uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
 void aes_ssse3_xts_dec(const struct aesdec *, const uint8_t[static 16],
     uint8_t[static 16], size_t, uint8_t[static 16], uint32_t);
+void aes_ssse3_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+    size_t, uint8_t[static 16], uint32_t);
+void aes_ssse3_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+    uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void aes_ssse3_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+    uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
 
 int aes_ssse3_selftest(void);
 
diff -r 2d391223e03a -r 3e671acf38ea sys/crypto/aes/arch/x86/aes_ssse3_impl.c
--- a/sys/crypto/aes/arch/x86/aes_ssse3_impl.c  Sat Jul 25 22:29:56 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ssse3_impl.c  Sat Jul 25 22:31:04 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $     */
+/*     $NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_impl.c,v 1.4 2020/07/25 22:31:04 riastradh Exp $");
 
 #include <crypto/aes/aes.h>
 #include <crypto/aes/aes_impl.h>
@@ -136,6 +136,39 @@
        fpu_kern_leave();
 }
 
+static void
+aes_ssse3_cbcmac_update1_impl(const struct aesenc *enc,
+    const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_ssse3_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aes_ssse3_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_ssse3_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aes_ssse3_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_ssse3_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
 static int
 aes_ssse3_probe(void)
 {
@@ -183,4 +216,7 @@
        .ai_cbc_dec = aes_ssse3_cbc_dec_impl,
        .ai_xts_enc = aes_ssse3_xts_enc_impl,
        .ai_xts_dec = aes_ssse3_xts_dec_impl,
+       .ai_cbcmac_update1 = aes_ssse3_cbcmac_update1_impl,
+       .ai_ccm_enc1 = aes_ssse3_ccm_enc1_impl,
+       .ai_ccm_dec1 = aes_ssse3_ccm_dec1_impl,
 };
diff -r 2d391223e03a -r 3e671acf38ea sys/crypto/aes/arch/x86/aes_ssse3_subr.c
--- a/sys/crypto/aes/arch/x86/aes_ssse3_subr.c  Sat Jul 25 22:29:56 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ssse3_subr.c  Sat Jul 25 22:31:04 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $     */
+/*     $NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ssse3_subr.c,v 1.3 2020/07/25 22:31:04 riastradh Exp $");
 
 #ifdef _KERNEL
 #include <sys/systm.h>
@@ -208,6 +208,75 @@
        storeblock(tweak, t);
 }
 
+void
+aes_ssse3_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+    size_t nbytes, uint8_t auth0[static 16], uint32_t nrounds)
+{
+       __m128i auth;
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       auth = loadblock(auth0);
+       for (; nbytes; nbytes -= 16, in += 16)
+               auth = aes_ssse3_enc1(enc, auth ^ loadblock(in), nrounds);
+       storeblock(auth0, auth);
+}
+
+void
+aes_ssse3_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+       const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0);
+       const __m128i bs32 =
+           _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203);
+       __m128i auth, ctr_be, ctr, ptxt;
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       auth = loadblock(authctr);
+       ctr_be = loadblock(authctr + 16);
+       ctr = _mm_shuffle_epi8(ctr_be, bs32);
+       for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+               ptxt = loadblock(in);
+               auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds);
+               ctr = _mm_add_epi32(ctr, ctr32_inc);
+               ctr_be = _mm_shuffle_epi8(ctr, bs32);
+               storeblock(out, ptxt ^ aes_ssse3_enc1(enc, ctr_be, nrounds));
+       }
+       storeblock(authctr, auth);
+       storeblock(authctr + 16, ctr_be);
+}
+
+void
+aes_ssse3_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+       const __m128i ctr32_inc = _mm_set_epi32(1, 0, 0, 0);
+       const __m128i bs32 =
+           _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203);
+       __m128i auth, ctr_be, ctr, ptxt;
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       auth = loadblock(authctr);
+       ctr_be = loadblock(authctr + 16);
+       ctr = _mm_shuffle_epi8(ctr_be, bs32);
+       for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+               ctr = _mm_add_epi32(ctr, ctr32_inc);
+               ctr_be = _mm_shuffle_epi8(ctr, bs32);
+               ptxt = loadblock(in) ^ aes_ssse3_enc1(enc, ctr_be, nrounds);
+               storeblock(out, ptxt);
+               auth = aes_ssse3_enc1(enc, auth ^ ptxt, nrounds);
+       }
+       storeblock(authctr, auth);
+       storeblock(authctr + 16, ctr_be);
+}
+
 int
 aes_ssse3_selftest(void)
 {
diff -r 2d391223e03a -r 3e671acf38ea sys/crypto/aes/arch/x86/immintrin.h
--- a/sys/crypto/aes/arch/x86/immintrin.h       Sat Jul 25 22:29:56 2020 +0000
+++ b/sys/crypto/aes/arch/x86/immintrin.h       Sat Jul 25 22:31:04 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: immintrin.h,v 1.2 2020/06/29 23:51:35 riastradh Exp $  */
+/*     $NetBSD: immintrin.h,v 1.3 2020/07/25 22:31:04 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -51,6 +51,7 @@
 typedef long long __v2di __attribute__((__vector_size__(16)));
 typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
 typedef int __v4si __attribute__((__vector_size__(16)));
+typedef unsigned __v4su __attribute__((__vector_size__(16)));
 typedef float __v4sf __attribute__((__vector_size__(16)));
 typedef short __v8hi __attribute__((__vector_size__(16)));
 typedef char __v16qi __attribute__((__vector_size__(16)));
@@ -65,6 +66,7 @@
 typedef long long __v2di __attribute__((__vector_size__(16)));
 typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
 typedef int __v4si __attribute__((__vector_size__(16)));
+typedef unsigned __v4su __attribute__((__vector_size__(16)));
 typedef float __v4sf __attribute__((__vector_size__(16)));
 typedef short __v8hi __attribute__((__vector_size__(16)));
 typedef char __v16qi __attribute__((__vector_size__(16)));
@@ -83,6 +85,13 @@
 
 #define        _SSSE3_ATTR     __attribute__((target("ssse3")))
 
+_INTRINSATTR
+static __inline __m128i
+_mm_add_epi32(__m128i __a, __m128i __b)
+{
+       return (__m128i)((__v4su)__a + (__v4su)__b);
+}
+
 #if defined(__GNUC__) && !defined(__clang__)
 #define        _mm_alignr_epi8(hi,lo,bytes)                                          \
        (__m128i)__builtin_ia32_palignr128((__v2di)(__m128i)(hi),             \