digest: Move AArch64/X86-64 dispatching to C (Merge BoringSSL 10c24cb).

Merge BoringSSL 10c24cb: sha: Move AArch64/X86-64 dispatching to C. Split `digest::sha2` into multiple parts: ``` git difftool HEAD^1:src/digest/sha2.rs src/digest/sha2/mod.rs git difftool HEAD^1:src/digest/sha2.rs src/digest/sha2/fallback.rs git difftool HEAD^1:src/digest/sha2.rs src/digest/sha2/sha2_32.rs git difftool HEAD^1:src/digest/sha2.rs src/digest/sha2/sha2_64.rs diff src/digest/sha2/sha2_32.rs src/digest/sha2/sha2_64.rs ```
briansmith · Jan 14, 2025 · fb7e969 · fb7e969
2 parents 2323d10 + 10c24cb
commit fb7e969
Show file tree

Hide file tree

Showing 11 changed files with 296 additions and 156 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -693,6 +693,34 @@ jobs:
             rust_channel: nightly
             host_os: ubuntu-22.04
 
+          - target: x86_64-unknown-linux-gnu
+            cpu_model: Haswell
+            features: --features=std
+            mode: # debug
+            rust_channel: nightly
+            host_os: ubuntu-22.04
+
+          - target: x86_64-unknown-linux-gnu
+            cpu_model: Nehalem
+            features: --features=std
+            mode: # debug
+            rust_channel: nightly
+            host_os: ubuntu-22.04
+
+          - target: x86_64-unknown-linux-gnu
+            cpu_model: SandyBridge
+            features: --features=std
+            mode: # debug
+            rust_channel: nightly
+            host_os: ubuntu-22.04
+
+          - target: x86_64-unknown-linux-gnu
+            cpu_model: Westmere
+            features: --features=std
+            mode: # debug
+            rust_channel: nightly
+            host_os: ubuntu-22.04
+
           - target: x86_64-unknown-linux-musl
             host_os: ubuntu-22.04
 

diff --git a/build.rs b/build.rs
@@ -932,7 +932,14 @@ fn prefix_all_symbols(pp: char, prefix_prefix: &str, prefix: &str) -> String {
         "openssl_poly1305_neon2_addmulmod",
         "openssl_poly1305_neon2_blocks",
         "sha256_block_data_order",
+        "sha256_block_data_order_avx",
+        "sha256_block_data_order_ssse3",
+        "sha256_block_data_order_hw",
+        "sha256_block_data_order_nohw",
         "sha512_block_data_order",
+        "sha512_block_data_order_avx",
+        "sha512_block_data_order_hw",
+        "sha512_block_data_order_nohw",
         "vpaes_ctr32_encrypt_blocks",
         "vpaes_encrypt",
         "vpaes_encrypt_key_to_bsaes",

diff --git a/crypto/fipsmodule/sha/asm/sha512-armv8.pl b/crypto/fipsmodule/sha/asm/sha512-armv8.pl
@@ -74,7 +74,7 @@
     *STDOUT=*OUT;
 }
 
-$func="sha${BITS}_block_data_order";
+$func="sha${BITS}_block_data_order_nohw";
 
 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
 
@@ -178,31 +178,10 @@ sub BODY_00_xx {
 
 .text
 
-.extern	OPENSSL_armcap_P
-.hidden	OPENSSL_armcap_P
 .globl	$func
 .type	$func,%function
 .align	6
 $func:
-	AARCH64_VALID_CALL_TARGET
-#ifndef	__KERNEL__
-#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10
-	adrp	x16,:pg_hi21_nc:OPENSSL_armcap_P
-#else
-	adrp	x16,:pg_hi21:OPENSSL_armcap_P
-#endif
-	ldr	w16,[x16,:lo12:OPENSSL_armcap_P]
-___
-$code.=<<___	if ($SZ==4);
-	tst	w16,#ARMV8_SHA256
-	b.ne	.Lv8_entry
-___
-$code.=<<___	if ($SZ==8);
-	tst	w16,#ARMV8_SHA512
-	b.ne	.Lv8_entry
-___
-$code.=<<___;
-#endif
 	AARCH64_SIGN_LINK_REGISTER
 	stp	x29,x30,[sp,#-128]!
 	add	x29,sp,#0
@@ -354,11 +333,12 @@ sub BODY_00_xx {
 $code.=<<___;
 .text
 #ifndef	__KERNEL__
-.type	sha256_block_armv8,%function
+.globl	sha256_block_data_order_hw
+.type	sha256_block_data_order_hw,%function
 .align	6
-sha256_block_armv8:
-.Lv8_entry:
+sha256_block_data_order_hw:
 	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
+	AARCH64_VALID_CALL_TARGET
 	stp		x29,x30,[sp,#-16]!
 	add		x29,sp,#0
 
@@ -423,7 +403,7 @@ sub BODY_00_xx {
 
 	ldr		x29,[sp],#16
 	ret
-.size	sha256_block_armv8,.-sha256_block_armv8
+.size	sha256_block_data_order_hw,.-sha256_block_data_order_hw
 #endif
 ___
 }
@@ -440,10 +420,12 @@ sub BODY_00_xx {
 $code.=<<___;
 .text
 #ifndef	__KERNEL__
-.type	sha512_block_armv8,%function
+.globl	sha512_block_data_order_hw
+.type	sha512_block_data_order_hw,%function
 .align	6
-sha512_block_armv8:
-.Lv8_entry:
+sha512_block_data_order_hw:
+	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
+	AARCH64_VALID_CALL_TARGET
 	stp		x29,x30,[sp,#-16]!
 	add		x29,sp,#0
 
@@ -527,7 +509,7 @@ sub BODY_00_xx {
 
 	ldr		x29,[sp],#16
 	ret
-.size	sha512_block_armv8,.-sha512_block_armv8
+.size	sha512_block_data_order_hw,.-sha512_block_data_order_hw
 #endif
 ___
 }

diff --git a/crypto/fipsmodule/sha/asm/sha512-x86_64.pl b/crypto/fipsmodule/sha/asm/sha512-x86_64.pl
@@ -255,42 +255,12 @@ ()
 $code=<<___;
 .text
 
-.extern	OPENSSL_ia32cap_P
-.globl	$func
-.type	$func,\@function,3
+.globl	${func}_nohw
+.type	${func}_nohw,\@function,3
 .align	16
-$func:
+${func}_nohw:
 .cfi_startproc
 	_CET_ENDBR
-___
-$code.=<<___ if ($SZ==4 || $avx);
-	leaq	OPENSSL_ia32cap_P(%rip),%r11
-	mov	0(%r11),%r9d
-	mov	4(%r11),%r10d
-	mov	8(%r11),%r11d
-___
-$code.=<<___ if ($SZ==4 && $shaext);
-	test	\$`1<<29`,%r11d		# check for SHA
-	jnz	.Lshaext_shortcut
-___
-    # XOP codepath removed.
-$code.=<<___ if ($avx>1);
-	and	\$`1<<8|1<<5|1<<3`,%r11d	# check for BMI2+AVX2+BMI1
-	cmp	\$`1<<8|1<<5|1<<3`,%r11d
-	je	.Lavx2_shortcut
-___
-$code.=<<___ if ($avx);
-	and	\$`1<<30`,%r9d		# mask "Intel CPU" bit
-	and	\$`1<<28|1<<9`,%r10d	# mask AVX and SSSE3 bits
-	or	%r9d,%r10d
-	cmp	\$`1<<28|1<<9|1<<30`,%r10d
-	je	.Lavx_shortcut
-___
-$code.=<<___ if ($SZ==4);
-	test	\$`1<<9`,%r10d
-	jnz	.Lssse3_shortcut
-___
-$code.=<<___;
 	mov	%rsp,%rax		# copy %rsp
 .cfi_def_cfa_register	%rax
 	push	%rbx
@@ -398,7 +368,7 @@ ()
 .Lepilogue:
 	ret
 .cfi_endproc
-.size	$func,.-$func
+.size	${func}_nohw,.-${func}_nohw
 ___
 
 if ($SZ==4) {
@@ -556,11 +526,12 @@ ()
 my @MSG=map("%xmm$_",(3..6));
 
 $code.=<<___;
-.type	sha256_block_data_order_shaext,\@function,3
+.globl	sha256_block_data_order_hw
+.type	sha256_block_data_order_hw,\@function,3
 .align	64
-sha256_block_data_order_shaext:
+sha256_block_data_order_hw:
 .cfi_startproc
-.Lshaext_shortcut:
+	_CET_ENDBR
 ___
 $code.=<<___ if ($win64);
 	lea	`-8-5*16`(%rsp),%rsp
@@ -705,7 +676,7 @@ ()
 $code.=<<___;
 	ret
 .cfi_endproc
-.size	sha256_block_data_order_shaext,.-sha256_block_data_order_shaext
+.size	sha256_block_data_order_hw,.-sha256_block_data_order_hw
 ___
 }}}
 {{{
@@ -770,11 +741,12 @@ ()
 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
 
 $code.=<<___;
+.globl	${func}_ssse3
 .type	${func}_ssse3,\@function,3
 .align	64
 ${func}_ssse3:
 .cfi_startproc
-.Lssse3_shortcut:
+	_CET_ENDBR
 	mov	%rsp,%rax		# copy %rsp
 .cfi_def_cfa_register	%rax
 	push	%rbx
@@ -1133,11 +1105,12 @@ ()
 local *ror = sub { &shrd(@_[0],@_) };
 
 $code.=<<___;
+.globl	${func}_avx
 .type	${func}_avx,\@function,3
 .align	64
 ${func}_avx:
 .cfi_startproc
-.Lavx_shortcut:
+	_CET_ENDBR
 	mov	%rsp,%rax		# copy %rsp
 .cfi_def_cfa_register	%rax
 	push	%rbx
@@ -1602,14 +1575,14 @@ ()
 $code.=<<___;
 .section	.pdata
 .align	4
-	.rva	.LSEH_begin_$func
-	.rva	.LSEH_end_$func
-	.rva	.LSEH_info_$func
+	.rva	.LSEH_begin_${func}_nohw
+	.rva	.LSEH_end_${func}_nohw
+	.rva	.LSEH_info_${func}_nohw
 ___
 $code.=<<___ if ($SZ==4 && $shaext);
-	.rva	.LSEH_begin_${func}_shaext
-	.rva	.LSEH_end_${func}_shaext
-	.rva	.LSEH_info_${func}_shaext
+	.rva	.LSEH_begin_${func}_hw
+	.rva	.LSEH_end_${func}_hw
+	.rva	.LSEH_info_${func}_hw
 ___
 $code.=<<___ if ($SZ==4);
 	.rva	.LSEH_begin_${func}_ssse3
@@ -1624,13 +1597,13 @@ ()
 $code.=<<___;
 .section	.xdata
 .align	8
-.LSEH_info_$func:
+.LSEH_info_${func}_nohw:
 	.byte	9,0,0,0
 	.rva	se_handler
 	.rva	.Lprologue,.Lepilogue			# HandlerData[]
 ___
 $code.=<<___ if ($SZ==4 && $shaext);
-.LSEH_info_${func}_shaext:
+.LSEH_info_${func}_hw:
 	.byte	9,0,0,0
 	.rva	shaext_handler
 ___

diff --git a/src/cpu/intel.rs b/src/cpu/intel.rs
@@ -144,8 +144,14 @@ cfg_if! {
             mask: 1 << 28,
         };
 
+        const SHA: Feature = Feature {
+            word: 2,
+            mask: 1 << 29,
+        };
+
         impl_get_feature!{ MOVBE => Movbe }
         impl_get_feature!{ AVX => Avx }
+        impl_get_feature!{ SHA => Sha }
     }
 }
 

diff --git a/src/digest/sha1.rs b/src/digest/sha1.rs
@@ -14,7 +14,10 @@
 // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 use super::{
-    sha2::{ch, maj, State32, Word},
+    sha2::{
+        fallback::{ch, maj, Word},
+        State32,
+    },
     BlockLen, OutputLen,
 };
 use crate::polyfill::slice;