bybrooklyn/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-04-18 14:53:58 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
9f4e9553a1f40841ebce9ab749896e9312b1701b
linux/lib/crypto/Makefile
Eric Biggers 3e79c8ec49 lib/crypto: x86/ghash: Migrate optimized code into library 
Remove the "ghash-pclmulqdqni" crypto_shash algorithm.  Move the
corresponding assembly code into lib/crypto/, and wire it up to the
GHASH library.

This makes the GHASH library be optimized with x86's carryless
multiplication instructions.  It also greatly reduces the amount of
x86-specific glue code that is needed, and it fixes the issue where this
GHASH optimization was disabled by default.

Rename and adjust the prototypes of the assembly functions to make them
fit better with the library.  Remove the byte-swaps (pshufb
instructions) that are no longer necessary because the library keeps the
accumulator in POLYVAL format rather than GHASH format.

Rename clmul_ghash_mul() to polyval_mul_pclmul() to reflect that it
really does a POLYVAL style multiplication.  Wire it up to both
ghash_mul_arch() and polyval_mul_arch().

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-15-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
2026-03-23 16:44:29 -07:00

381 lines
12 KiB
Makefile
Raw Blame History

# SPDX-License-Identifier: GPL-2.0
aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $(<) > $(@)
quiet_cmd_perlasm_with_args = PERLASM $@
cmd_perlasm_with_args = $(PERL) $(<) void $(@)
ppc64-perlasm-flavour-y := linux-ppc64
ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
obj-$(CONFIG_KUNIT) += tests/
obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o
libcryptoutils-y := memneq.o utils.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o
libaes-y := aes.o
ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)
CFLAGS_aes.o += -I$(src)/$(SRCARCH)
libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o
libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \
arm64/aes-ce-core.o \
arm64/aes-ce.o \
arm64/aes-neon.o
ifeq ($(CONFIG_PPC),y)
ifeq ($(CONFIG_SPE),y)
libaes-y += powerpc/aes-spe-core.o \
powerpc/aes-spe-keys.o \
powerpc/aes-spe-modes.o \
powerpc/aes-tab-4k.o
else
libaes-y += powerpc/aesp8-ppc.o
quiet_cmd_perlasm_aes = PERLASM $@
cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
# Use if_changed instead of cmd, in case the flavour changed.
$(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE
$(call if_changed,perlasm_aes)
targets += powerpc/aesp8-ppc.S
OBJECT_FILES_NON_STANDARD_powerpc/aesp8-ppc.o := y
endif # !CONFIG_SPE
endif # CONFIG_PPC
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
libaes-$(CONFIG_X86) += x86/aes-aesni.o
endif # CONFIG_CRYPTO_LIB_AES_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_AESCFB) += libaescfb.o
libaescfb-y := aescfb.o
obj-$(CONFIG_CRYPTO_LIB_AESGCM) += libaesgcm.o
libaesgcm-y := aesgcm.o
obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o
libarc4-y := arc4.o
obj-$(CONFIG_CRYPTO_LIB_GF128MUL) += gf128mul.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_BLAKE2B) += libblake2b.o
libblake2b-y := blake2b.o
ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2B_ARCH),y)
CFLAGS_blake2b.o += -I$(src)/$(SRCARCH)
libblake2b-$(CONFIG_ARM) += arm/blake2b-neon-core.o
endif # CONFIG_CRYPTO_LIB_BLAKE2B_ARCH
################################################################################
# blake2s is used by the /dev/random driver which is always builtin
obj-y += blake2s.o
ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2S_ARCH),y)
CFLAGS_blake2s.o += -I$(src)/$(SRCARCH)
obj-$(CONFIG_ARM) += arm/blake2s-core.o
obj-$(CONFIG_X86) += x86/blake2s-core.o
endif
################################################################################
# chacha20_block() is used by the /dev/random driver which is always builtin
obj-y += chacha-block-generic.o
obj-$(CONFIG_CRYPTO_LIB_CHACHA) += libchacha.o
libchacha-y := chacha.o
ifeq ($(CONFIG_CRYPTO_LIB_CHACHA_ARCH),y)
CFLAGS_chacha.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libchacha-y += arm/chacha-scalar-core.o
libchacha-$(CONFIG_KERNEL_MODE_NEON) += arm/chacha-neon-core.o
endif
libchacha-$(CONFIG_ARM64) += arm64/chacha-neon-core.o
ifeq ($(CONFIG_MIPS),y)
libchacha-y += mips/chacha-core.o
AFLAGS_mips/chacha-core.o += -O2 # needed to fill branch delay slots
endif
libchacha-$(CONFIG_PPC) += powerpc/chacha-p10le-8x.o
libchacha-$(CONFIG_RISCV) += riscv/chacha-riscv64-zvkb.o
libchacha-$(CONFIG_S390) += s390/chacha-s390.o
libchacha-$(CONFIG_X86) += x86/chacha-ssse3-x86_64.o \
x86/chacha-avx2-x86_64.o \
x86/chacha-avx512vl-x86_64.o
endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o
libchacha20poly1305-y += chacha20poly1305.o
libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o
libcurve25519-y := curve25519.o
# Disable GCOV in odd or sensitive code
GCOV_PROFILE_curve25519.o := n
ifeq ($(CONFIG_ARCH_SUPPORTS_INT128),y)
libcurve25519-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += curve25519-hacl64.o
else
libcurve25519-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += curve25519-fiat32.o
endif
# clang versions prior to 18 may blow out the stack with KASAN
ifeq ($(CONFIG_CC_IS_CLANG)_$(call clang-min-version, 180000),y_)
KASAN_SANITIZE_curve25519-hacl64.o := n
endif
ifeq ($(CONFIG_CRYPTO_LIB_CURVE25519_ARCH),y)
CFLAGS_curve25519.o += -I$(src)/$(SRCARCH)
libcurve25519-$(CONFIG_ARM) += arm/curve25519-core.o
libcurve25519-$(CONFIG_PPC) += powerpc/curve25519-ppc64le_asm.o
endif
################################################################################
obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o
libdes-y := des.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o
libgf128hash-y := gf128hash.o
ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y)
CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH)
libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o
libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \
arm64/polyval-ce-core.o
ifeq ($(CONFIG_PPC),y)
libgf128hash-y += powerpc/ghashp8-ppc.o
quiet_cmd_perlasm_ghash = PERLASM $@
cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE
$(call if_changed,perlasm_ghash)
targets += powerpc/ghashp8-ppc.S
OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
endif
libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o
libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \
x86/polyval-pclmul-avx.o
endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
# clean-files must be defined unconditionally
clean-files += powerpc/ghashp8-ppc.S
################################################################################
obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o
libmd5-y := md5.o
ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y)
CFLAGS_md5.o += -I$(src)/$(SRCARCH)
libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o
libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o
endif # CONFIG_CRYPTO_LIB_MD5_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_MLDSA) += libmldsa.o
libmldsa-y := mldsa.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_NH) += libnh.o
libnh-y := nh.o
ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y)
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
endif
################################################################################
obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o
libpoly1305-y := poly1305.o
ifeq ($(CONFIG_ARCH_SUPPORTS_INT128),y)
libpoly1305-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += poly1305-donna64.o
else
libpoly1305-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += poly1305-donna32.o
endif
ifeq ($(CONFIG_CRYPTO_LIB_POLY1305_ARCH),y)
CFLAGS_poly1305.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libpoly1305-y += arm/poly1305-core.o
$(obj)/arm/poly1305-core.S: $(src)/arm/poly1305-armv4.pl
$(call cmd,perlasm)
# massage the perlasm code a bit so we only get the NEON routine if we need it
poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
AFLAGS_arm/poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y)
endif
ifeq ($(CONFIG_ARM64),y)
libpoly1305-y += arm64/poly1305-core.o
$(obj)/arm64/poly1305-core.S: $(src)/arm64/poly1305-armv8.pl
$(call cmd,perlasm_with_args)
endif
ifeq ($(CONFIG_MIPS),y)
libpoly1305-y += mips/poly1305-core.o
poly1305-perlasm-flavour-$(CONFIG_32BIT) := o32
poly1305-perlasm-flavour-$(CONFIG_64BIT) := 64
quiet_cmd_perlasm_poly1305 = PERLASM $@
cmd_perlasm_poly1305 = $(PERL) $< $(poly1305-perlasm-flavour-y) $@
# Use if_changed instead of cmd, in case the flavour changed.
$(obj)/mips/poly1305-core.S: $(src)/mips/poly1305-mips.pl FORCE
$(call if_changed,perlasm_poly1305)
targets += mips/poly1305-core.S
endif
libpoly1305-$(CONFIG_PPC) += powerpc/poly1305-p10le_64.o
ifeq ($(CONFIG_RISCV),y)
libpoly1305-y += riscv/poly1305-core.o
poly1305-perlasm-flavour-$(CONFIG_32BIT) := 32
poly1305-perlasm-flavour-$(CONFIG_64BIT) := 64
quiet_cmd_perlasm_poly1305 = PERLASM $@
cmd_perlasm_poly1305 = $(PERL) $< $(poly1305-perlasm-flavour-y) $@
# Use if_changed instead of cmd, in case the flavour changed.
$(obj)/riscv/poly1305-core.S: $(src)/riscv/poly1305-riscv.pl FORCE
$(call if_changed,perlasm_poly1305)
targets += riscv/poly1305-core.S
AFLAGS_riscv/poly1305-core.o += -Dpoly1305_init=poly1305_block_init
endif
ifeq ($(CONFIG_X86),y)
libpoly1305-y += x86/poly1305-x86_64-cryptogams.o
$(obj)/x86/poly1305-x86_64-cryptogams.S: $(src)/x86/poly1305-x86_64-cryptogams.pl
$(call cmd,perlasm)
endif
endif # CONFIG_CRYPTO_LIB_POLY1305_ARCH
# clean-files must be defined unconditionally
clean-files += arm/poly1305-core.S \
arm64/poly1305-core.S \
mips/poly1305-core.S \
riscv/poly1305-core.S \
x86/poly1305-x86_64-cryptogams.S
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o
libsha1-y := sha1.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y)
CFLAGS_sha1.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libsha1-y += arm/sha1-armv4-large.o
libsha1-$(CONFIG_KERNEL_MODE_NEON) += arm/sha1-armv7-neon.o \
arm/sha1-ce-core.o
endif
libsha1-$(CONFIG_ARM64) += arm64/sha1-ce-core.o
ifeq ($(CONFIG_PPC),y)
libsha1-y += powerpc/sha1-powerpc-asm.o
libsha1-$(CONFIG_SPE) += powerpc/sha1-spe-asm.o
endif
libsha1-$(CONFIG_SPARC) += sparc/sha1_asm.o
libsha1-$(CONFIG_X86) += x86/sha1-ssse3-and-avx.o \
x86/sha1-avx2-asm.o \
x86/sha1-ni-asm.o
endif # CONFIG_CRYPTO_LIB_SHA1_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o
libsha256-y := sha256.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA256_ARCH),y)
CFLAGS_sha256.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libsha256-y += arm/sha256-ce.o arm/sha256-core.o
$(obj)/arm/sha256-core.S: $(src)/arm/sha256-armv4.pl
$(call cmd,perlasm)
AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y)
endif
ifeq ($(CONFIG_ARM64),y)
libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o
$(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl
$(call cmd,perlasm_with_args)
endif
libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o
libsha256-$(CONFIG_RISCV) += riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.o
libsha256-$(CONFIG_SPARC) += sparc/sha256_asm.o
libsha256-$(CONFIG_X86) += x86/sha256-ssse3-asm.o \
x86/sha256-avx-asm.o \
x86/sha256-avx2-asm.o \
x86/sha256-ni-asm.o
endif # CONFIG_CRYPTO_LIB_SHA256_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o
libsha512-y := sha512.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA512_ARCH),y)
CFLAGS_sha512.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libsha512-y += arm/sha512-core.o
$(obj)/arm/sha512-core.S: $(src)/arm/sha512-armv4.pl
$(call cmd,perlasm)
AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y)
endif
ifeq ($(CONFIG_ARM64),y)
libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o
$(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl
$(call cmd,perlasm_with_args)
endif
libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
libsha512-$(CONFIG_SPARC) += sparc/sha512_asm.o
libsha512-$(CONFIG_X86) += x86/sha512-ssse3-asm.o \
x86/sha512-avx-asm.o \
x86/sha512-avx2-asm.o
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA3) += libsha3.o
libsha3-y := sha3.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA3_ARCH),y)
CFLAGS_sha3.o += -I$(src)/$(SRCARCH)
libsha3-$(CONFIG_ARM64) += arm64/sha3-ce-core.o
endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
################################################################################
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
libsm3-y := sm3.o
# clean-files must be defined unconditionally
clean-files += arm/sha256-core.S arm/sha512-core.S
clean-files += arm64/sha256-core.S arm64/sha512-core.S
Reference in New Issue View Git Blame Copy Permalink