bsdkm: x86 crypto acceleration support.

Author: philljj

.gitignore

@@ -239,12 +239,17 @@ linuxkm/linuxkm
 linuxkm/src
 linuxkm/patches/src
 *.nds
+
+# Generated during FreeBSD kernel module build.
 bsdkm/export_syms
 bsdkm/i386
 bsdkm/libwolfssl.ko
 bsdkm/machine
 bsdkm/opt_global.h
 bsdkm/x86
+bsdkm/bus_if.h
+bsdkm/cryptodev_if.h
+bsdkm/device_if.h
 
 # autotools generated
 scripts/unit.test

.wolfssl_known_macro_extras

@@ -36,6 +36,7 @@ BLAKE2B_SELFTEST
 BLAKE2S_SELFTEST
 BLOCKING
 BSDKM_EXPORT_SYMS
+ENABLED_BSDKM_REGISTER
 BSP_DEFAULT_IO_CHANNEL_DEFINED
 BSP_LED_0
 BSP_LED_1

Makefile.am

@@ -247,8 +247,9 @@ if BUILD_BSDKM
         EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \
         AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \
         AM_CCASFLAGS CCASFLAGS \
-        src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS
-
+        src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \
+        ENABLED_ASM ENABLED_INTELASM ENABLED_AESNI ENABLED_AESNI_WITH_AVX \
+        ENABLED_KERNEL_BENCHMARKS
 endif
 
 

bsdkm/Makefile

@@ -1,39 +1,54 @@
-# wolfssl kernel module name and source, and root dir.
-KMOD=libwolfssl
-SRCS=wolfkmod.c
-WOLFSSL_DIR=../
+# wolfssl kernel module name and main source, and wolfssl root dir.
+KMOD        = libwolfssl
+SRCS        = wolfkmod.c
+WOLFSSL_DIR = ../
 
-CFLAGS+=-I${WOLFSSL_DIR}
-CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
+CFLAGS += -I${WOLFSSL_DIR}
+CFLAGS += -DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
 #
 # debug options
 #   verbose printing:
-#   CFLAGS+=-DWOLFSSL_BSDKM_VERBOSE_DEBUG
+#   CFLAGS += -DWOLFSSL_BSDKM_VERBOSE_DEBUG
 #
 #   print memory mallocs / frees:
-#   CFLAGS+=-DWOLFSSL_BSDKM_MEMORY_DEBUG
+#   CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG
 #
-CFLAGS+=$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+#   print fpu_kern_enter / leave:
+#   CFLAGS += WOLFSSL_BSDKM_FPU_DEBUG
+#
+CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+
+.if defined(ENABLED_BSDKM_REGISTER)
+  # These device header files are generated during build.
+  SRCS += bus_if.h cryptodev_if.h device_if.h
+.endif
 
 # FreeBSD make does not support GNU make's patsubst and related. Filter
 # through sed instead.
 WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \
   sed 's|src_libwolfssl_la-||g' | sed 's|\.lo|.o|g' | \
   sed 's|wolfcrypt/src/|${WOLFSSL_DIR}/wolfcrypt/src/|g'
 
+# wolfcrypt test
 .if ${ENABLED_CRYPT_TESTS} == "yes"
     WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/test/test.o
 .else
-    CFLAGS+=-DNO_CRYPT_TEST
+    CFLAGS += -DNO_CRYPT_TEST
+.endif
+
+# wolfcrypt benchmark
+.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
+    WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o
+    CFLAGS += -DWOLFSSL_NO_FLOAT_FMT
 .endif
 
 OBJS += ${WOLFSSL_OBJS}
 
 # Export no public symbols by default.
 .if !defined(BSDKM_EXPORT_SYMS)
-    EXPORT_SYMS=NO
+    EXPORT_SYMS = NO
 .else
-    EXPORT_SYMS=${BSDKM_EXPORT_SYMS}
+    EXPORT_SYMS = ${BSDKM_EXPORT_SYMS}
 .endif
 
 # Default to live kernel src tree makefile at
@@ -45,12 +60,52 @@ OBJS += ${WOLFSSL_OBJS}
 .endif
 .include "${SYSDIR}/conf/kmod.mk"
 
+#
+# To use aesni and friends in FreeBSD kernel we need to adjust build flags.
+# See these kernel makefiles for reference:
+#   - /usr/src/sys/modules/aesni/Makefile
+#   - /usr/src/sys/conf/kern.mk
+#
+WOLFKMOD_SIMD_BASE = -msse -msse2 -msse4.1
+WOLFKMOD_SIMD_AES  = -maes -mpclmul
+WOLFKMOD_SIMD_AVX  = -mavx -mavx2
+
+.if ${ENABLED_AESNI} == "yes"
+  CFLAGS.aes.c += ${WOLFKMOD_SIMD_BASE}
+  CFLAGS.aes.c += ${WOLFKMOD_SIMD_AES}
+.if ${ENABLED_AESNI_WITH_AVX} == "yes"
+  CFLAGS.aes.c += ${WOLFKMOD_SIMD_AVX}
+.endif # ENABLED_AESNI_WITH_AVX #
+  CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc}
+  CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif # ENABLED_AESNI
+
+.if ${ENABLED_ASM} == "yes"
+.for f in chacha dilithium poly1305 sha sha256 sha3 sha512
+  CFLAGS.${f}.c += ${WOLFKMOD_SIMD_BASE}
+  CFLAGS.${f}.c += ${WOLFKMOD_SIMD_AVX}
+  CFLAGS.${f}.c := ${CFLAGS.${f}.c:N-nostdinc}
+  CFLAGS.${f}.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endfor
+
+.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif # ENABLED_ASM == "yes"
+
+# wolfcrypt benchmark always needs simd for the floating point timings.
+.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
+  CFLAGS.benchmark.c += ${WOLFKMOD_SIMD_BASE}
+  CFLAGS.benchmark.c := ${CFLAGS.benchmark.c:N-nostdinc}
+  CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+  .PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif
+
 # Smooth out a few inconsistencies between FreeBSD default compiler flags
 # in /usr/src/sys/conf/kern.mk, vs wolfssl harden flags in
 # m4/ax_harden_compiler_flags.m4. E.g. some FreeBSD header files shorten
 # 64 to 32 bit, and some wolfcrypt functions cast away const.
-CFLAGS+= -Wno-unused-function
-CFLAGS+= -Wno-cast-qual
-CFLAGS+= -Wno-error=cast-qual
-CFLAGS+= -Wno-shorten-64-to-32
-CFLAGS+= -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
+CFLAGS += -Wno-unused-function
+CFLAGS += -Wno-cast-qual
+CFLAGS += -Wno-error=cast-qual
+CFLAGS += -Wno-shorten-64-to-32
+CFLAGS += -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""

bsdkm/README.md

@@ -7,10 +7,10 @@ other loadable modules to link to wolfCrypt.
 Supported features:
 - wolfCrypt in kernel.
 - FIPS-wolfcrypt.
+- crypto acceleration: AES-NI, AVX, etc.
 
 Planned features:
-- crypto acceleration: AES-NI, AVX, etc.
-- kernel opencrypto driver registration.
+- kernel opencrypto driver registration (supported for internal testing presently).
 - full wolfSSL in kernel (kernel TLS).
 
 ## Building and Installing
@@ -44,10 +44,12 @@ sudo kldunload libwolfssl
 
 ### options
 
-| freebsdkm option                 | description                              |
-| :------------------------------- | :--------------------------------------- |
-| --with-bsd-export-syms=LIST      | Export list of symbols as global. <br>. Options are 'all', 'none', or <br> comma separated list of symbols. |
-| --with-kernel-source=PATH        | Path to kernel tree root (default `/usr/src/sys`) |
+| freebsdkm option                   | description                              |
+| :--------------------------------- | :--------------------------------------- |
+| --with-bsd-export-syms=LIST        | Export list of symbols as global. <br>. Options are 'all', 'none', or <br> comma separated list of symbols. |
+| --with-kernel-source=PATH          | Path to kernel tree root (default `/usr/src/sys`) |
+| --enable-kernel-benchmarks         | Run wolfcrypt benchmark at module load    |
+| --enable-freebsdkm-crypto-register | Register with the FreeBSD kernel opencrypto <br>framework (preliminary, for testing) |
 
 ### FIPS
 

bsdkm/bsdkm_wc_port.h

@@ -61,13 +61,13 @@ static inline time_t wolfkmod_time(time_t * tloc) {
 #define WOLFSSL_DEBUG_PRINTF_FN printf
 
 /* str and char utility functions */
-#define XATOI(s) ({                                         \
-      char * endptr = NULL;                                 \
-      long   _xatoi_ret = strtol(s, &endptr, 10);           \
-      if ((s) == endptr || *endptr != '\0') {               \
-        _xatoi_ret = 0;                                     \
-      }                                                     \
-      (int)_xatoi_ret;                                      \
+#define XATOI(s) ({                                                          \
+      char * endptr = NULL;                                                  \
+      long   _xatoi_ret = strtol(s, &endptr, 10);                            \
+      if ((s) == endptr || *endptr != '\0') {                                \
+        _xatoi_ret = 0;                                                      \
+      }                                                                      \
+      (int)_xatoi_ret;                                                       \
 })
 
 #if !defined(XMALLOC_OVERRIDE)
@@ -103,6 +103,33 @@ extern struct malloc_type M_WOLFSSL[1];
     })
 #endif /* WOLFSSL_BSDKM_DEBUG_MEMORY */
 
+
+#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+    int  wolfkmod_vecreg_init(void);
+    void wolfkmod_vecreg_exit(void);
+    int  wolfkmod_vecreg_save(int flags_unused);
+    void wolfkmod_vecreg_restore(void);
+    /* wrapper defines for FPU_KERN(9).
+     * /usr/src/sys/amd64/amd64/fpu.c
+     * /usr/src/sys/amd64/include/pcb.h
+     * */
+    #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+        #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+    #endif
+
+    #define SAVE_VECTOR_REGISTERS(fail_clause) {                             \
+        int _svr_ret = wolfkmod_vecreg_save(0);                              \
+        if (_svr_ret != 0) {                                                 \
+            fail_clause                                                      \
+        }                                                                    \
+    }
+
+    #define SAVE_VECTOR_REGISTERS2() wolfkmod_vecreg_save(0)
+
+    #define RESTORE_VECTOR_REGISTERS() wolfkmod_vecreg_restore()
+
+#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS */
+
 #if !defined(SINGLE_THREADED)
     #define WC_MUTEX_OPS_INLINE
 
@@ -149,7 +176,8 @@ extern struct malloc_type M_WOLFSSL[1];
     typedef volatile int          wolfSSL_Atomic_Int;
     typedef volatile unsigned int wolfSSL_Atomic_Uint;
     #define WOLFSSL_ATOMIC_INITIALIZER(x) (x)
-    #define WOLFSSL_ATOMIC_LOAD(x)  (int)atomic_load_acq_int(&(x))
+    #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
+    #define WOLFSSL_ATOMIC_LOAD_UINT(x) atomic_load_acq_int(&(x))
     #define WOLFSSL_ATOMIC_STORE(x, v)  atomic_store_rel_int(&(x), (v))
     #define WOLFSSL_ATOMIC_OPS
 

bsdkm/include.am

@@ -2,8 +2,10 @@
 # included from Top Level Makefile.am
 # All paths should be given relative to the root
 
-EXTRA_DIST += m4/ax_bsdkm.m4 \
-	      bsdkm/Makefile \
-	      bsdkm/README.md \
-	      bsdkm/wolfkmod.c \
+EXTRA_DIST += m4/ax_bsdkm.m4       \
+	      bsdkm/Makefile       \
+	      bsdkm/README.md      \
+	      bsdkm/wolfkmod.c     \
+	      bsdkm/wolfkmod_aes.c \
+	      bsdkm/x86_vecreg.c   \
 	      bsdkm/bsdkm_wc_port.h