The previous lib_sha256 finup code can process two blocks if needed,
restore it and put it into the sha256_finup helper so it can be
reused by the Crypto API.
Also add sha256_choose_blocks and CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
so that the Crypto API can use the SIMD block function unconditionally.
The Crypto API must not be used in hard IRQs and there is no reason
to have a fallback path for hardirqs.
Signed-off-by: Herbert Xu <herbert@xxxxxxxxxxxxxxxxxxx>
---
include/crypto/internal/sha2.h | 46 ++++++++++++++++++++++++++++++++++
lib/crypto/Kconfig | 8 ++++++
lib/crypto/sha256.c | 35 ++++++++------------------
3 files changed, 65 insertions(+), 24 deletions(-)
diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h
index d641c67abcbc..07e41efc6cc6 100644
--- a/include/crypto/internal/sha2.h
+++ b/include/crypto/internal/sha2.h
@@ -3,7 +3,11 @@
#ifndef _CRYPTO_INTERNAL_SHA2_H
#define _CRYPTO_INTERNAL_SHA2_H
+#include <crypto/internal/simd.h>
#include <crypto/sha2.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
void sha256_update_generic(struct sha256_state *sctx,
const u8 *data, size_t len);
@@ -24,5 +28,47 @@ void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS],
const u8 *data, size_t nblocks);
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
const u8 *data, size_t nblocks);
+void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],
+ const u8 *data, size_t nblocks);
+
+static inline void sha256_choose_blocks(
+ u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
+ bool force_generic, bool force_simd)
+{
+ if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
+ sha256_blocks_generic(state, data, nblocks);
+ else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) &&
+ (force_simd || crypto_simd_usable()))
+ sha256_blocks_simd(state, data, nblocks);
+ else
+ sha256_blocks_arch(state, data, nblocks);
+}
+
+static __always_inline void sha256_finup(
+ struct crypto_sha256_state *sctx, const u8 *src, unsigned int len,
+ u8 out[SHA256_DIGEST_SIZE], size_t digest_size, bool force_generic,
+ bool force_simd)
+{
+ unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1;
+ union {
+ __be64 b64[SHA256_BLOCK_SIZE / 4];
+ u8 u8[SHA256_BLOCK_SIZE * 2];
+ } block = {};
+ int blocks, i;
+
+ sctx->count += len;
+ if (len >= bit_offset * 8)
+ bit_offset += SHA256_BLOCK_SIZE / 8;
+ blocks = (bit_offset + 1) * 8 / SHA256_BLOCK_SIZE;
+ memcpy(&block, src, len);
+ block.u8[len] = 0x80;
+ block.b64[bit_offset] = cpu_to_be64(sctx->count << 3);
+ sha256_choose_blocks(sctx->state, block.u8, blocks, force_generic,
+ force_simd);
+ memzero_explicit(&block, sizeof(block));
+
+ for (i = 0; i < digest_size; i += 4)
+ put_unaligned_be32(sctx->state[i / 4], out + i);
+}
#endif /* _CRYPTO_INTERNAL_SHA2_H */
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 6319358b38c2..1ec1466108cc 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -150,6 +150,14 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256
Declares whether the architecture provides an arch-specific
accelerated implementation of the SHA-256 library interface.
+config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
+ bool
+ help
+ Declares whether the architecture provides an arch-specific
+ accelerated implementation of the SHA-256 library interface
+ that is SIMD-based and therefore not usable in hardirq
+ context.
+
config CRYPTO_LIB_SHA256_GENERIC
tristate
default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256
diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c
index 563f09c9f381..b5ffff032718 100644
--- a/lib/crypto/sha256.c
+++ b/lib/crypto/sha256.c
@@ -15,7 +15,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
-#include <linux/unaligned.h>
/*
* If __DISABLE_EXPORTS is defined, then this file is being compiled for a
@@ -26,14 +25,16 @@
#include "sha256-generic.c"
#endif
+static inline bool sha256_purgatory(void)
+{
+ return __is_defined(__DISABLE_EXPORTS);
+}
+
static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *data,
size_t nblocks, bool force_generic)
{
-#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) && !defined(__DISABLE_EXPORTS)
- if (!force_generic)
- return sha256_blocks_arch(state, data, nblocks);
-#endif
- sha256_blocks_generic(state, data, nblocks);
+ sha256_choose_blocks(state, data, nblocks,
+ force_generic || sha256_purgatory(), false);
}
static inline void __sha256_update(struct sha256_state *sctx, const u8 *data,
@@ -79,25 +80,11 @@ EXPORT_SYMBOL(sha256_update);
static inline void __sha256_final(struct sha256_state *sctx, u8 *out,
size_t digest_size, bool force_generic)
{
- const size_t bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64);
- __be64 *bits = (__be64 *)&sctx->buf[bit_offset];
- size_t partial = sctx->count % SHA256_BLOCK_SIZE;
- size_t i;
-
- sctx->buf[partial++] = 0x80;
- if (partial > bit_offset) {
- memset(&sctx->buf[partial], 0, SHA256_BLOCK_SIZE - partial);
- sha256_blocks(sctx->state, sctx->buf, 1, force_generic);
- partial = 0;
- }
-
- memset(&sctx->buf[partial], 0, bit_offset - partial);
- *bits = cpu_to_be64(sctx->count << 3);
- sha256_blocks(sctx->state, sctx->buf, 1, force_generic);
-
- for (i = 0; i < digest_size; i += 4)
- put_unaligned_be32(sctx->state[i / 4], out + i);
+ unsigned int len = sctx->count % SHA256_BLOCK_SIZE;
+ sctx->count -= len;
+ sha256_finup(&sctx->ctx, sctx->buf, len, out, digest_size,
+ force_generic || sha256_purgatory(), false);
memzero_explicit(sctx, sizeof(*sctx));
}
--
2.39.5
