mirror of
https://github.com/torvalds/linux.git
synced 2026-04-18 14:53:58 -04:00
11d7956d52
Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Signed-off-by: Eric Biggers <ebiggers@google.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
75 lines
2.1 KiB
C
75 lines
2.1 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* SHA-256 optimized for x86_64
|
|
*
|
|
* Copyright 2025 Google LLC
|
|
*/
|
|
#include <asm/fpu/api.h>
|
|
#include <crypto/internal/sha2.h>
|
|
#include <crypto/internal/simd.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/static_call.h>
|
|
|
|
asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS],
|
|
const u8 *data, size_t nblocks);
|
|
asmlinkage void sha256_transform_avx(u32 state[SHA256_STATE_WORDS],
|
|
const u8 *data, size_t nblocks);
|
|
asmlinkage void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS],
|
|
const u8 *data, size_t nblocks);
|
|
asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS],
|
|
const u8 *data, size_t nblocks);
|
|
|
|
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86);
|
|
|
|
DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3);
|
|
|
|
void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS],
|
|
const u8 *data, size_t nblocks)
|
|
{
|
|
if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) {
|
|
kernel_fpu_begin();
|
|
static_call(sha256_blocks_x86)(state, data, nblocks);
|
|
kernel_fpu_end();
|
|
} else {
|
|
sha256_blocks_generic(state, data, nblocks);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(sha256_blocks_arch);
|
|
|
|
bool sha256_is_arch_optimized(void)
|
|
{
|
|
return static_key_enabled(&have_sha256_x86);
|
|
}
|
|
EXPORT_SYMBOL(sha256_is_arch_optimized);
|
|
|
|
static int __init sha256_x86_mod_init(void)
|
|
{
|
|
if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
|
|
static_call_update(sha256_blocks_x86, sha256_ni_transform);
|
|
} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE |
|
|
XFEATURE_MASK_YMM, NULL) &&
|
|
boot_cpu_has(X86_FEATURE_AVX)) {
|
|
if (boot_cpu_has(X86_FEATURE_AVX2) &&
|
|
boot_cpu_has(X86_FEATURE_BMI2))
|
|
static_call_update(sha256_blocks_x86,
|
|
sha256_transform_rorx);
|
|
else
|
|
static_call_update(sha256_blocks_x86,
|
|
sha256_transform_avx);
|
|
} else if (!boot_cpu_has(X86_FEATURE_SSSE3)) {
|
|
return 0;
|
|
}
|
|
static_branch_enable(&have_sha256_x86);
|
|
return 0;
|
|
}
|
|
arch_initcall(sha256_x86_mod_init);
|
|
|
|
static void __exit sha256_x86_mod_exit(void)
|
|
{
|
|
}
|
|
module_exit(sha256_x86_mod_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_DESCRIPTION("SHA-256 optimized for x86_64");
|