crypto.ecdsa: fix handling of sign() with custom_hash (#23619)

vlib/crypto/ecdsa/ecdsa.v

@@ -540,7 +540,6 @@ fn calc_digest(key &C.EC_KEY, message []u8, opt SignerOpts) ![]u8 {
 		return error('fail to load group')
 	}
 	num_bits := C.EC_GROUP_get_degree(group)
-	// check for key size matching
 	key_size := (num_bits + 7) / 8
 	// we're working on mutable copy of SignerOpts, with some issues when make it as a mutable.
 	// ie, declaring a mutable parameter that accepts a struct with the `@[params]` attribute is not allowed.
@@ -566,8 +565,22 @@ fn calc_digest(key &C.EC_KEY, message []u8, opt SignerOpts) ![]u8 {
 					return error('Hash into smaller size than current key size was not allowed')
 				}
 			}
-			digest := cfg.custom_hash.sum(message)
-			defer { unsafe { cfg.custom_hash.free() } }
+			// we need to reset the custom hash before writes message
+			cfg.custom_hash.reset()
+			_ := cfg.custom_hash.write(message)!
+			digest := cfg.custom_hash.sum([]u8{})
+			// NOTES:
+			// NIST FIPS 186-4 at the end of section 6.4 states that:
+			// When the length of the output of the hash function is greater than the bit length of n,
+			// then the leftmost n bits of the hash function output block shall be used in any calculation
+			// using the hash function output during the generation or verification of a digital signature
+			// with output of custom_hash was bigger than bit length (key size)
+			// TODO:
+			// Maybe better to pick up only required bytes from digest, ie,
+			// out := digest[..key_size].clone()
+			// unsafe { digest.free() }
+			// return out
+			// Currently, just boildown to the caller
 			return digest
 		}
 	}

vlib/crypto/ecdsa/util_test.v

@@ -2,6 +2,8 @@ module ecdsa
 
 import encoding.hex
 import crypto.pem
+import crypto.sha1
+import crypto.sha512
 
 // This material wss generated with https://emn178.github.io/online-tools/ecdsa/key-generator
 // with curve SECG secp384r1 aka NIST P-384
@@ -126,3 +128,68 @@ UHhnmmVRraSwrVkPdYIeXhH/Ob4+8OLcwrQBMv4RXsD1GVFsgkvEYDTEb/vnMA==
 		return
 	}
 }
+
+fn test_key_signing_verifying_with_custom_hash() ! {
+	// privatekey_sample was P-384 key
+	pvkey := privkey_from_string(privatekey_sample)!
+	// public key part	
+	pbkey := pvkey.public_key()!
+	pbk := pubkey_from_string(public_key_sample)!
+
+	// lets sign the message with default hash, ie, sha384
+	signature := pvkey.sign(message_tobe_signed)!
+	verified := pbkey.verify(message_tobe_signed, signature)!
+	assert verified == true
+
+	// Use the bigger custom hash
+	opt0 := SignerOpts{
+		hash_config:       .with_custom_hash
+		allow_custom_hash: true
+		custom_hash:       sha512.new()
+	}
+	// online-generated signature with sha512 digest with the same params from https://emn178.github.io/online-tools/ecdsa/sign/
+	online_sign0 := hex.decode('3066023100b54b479b64961481074c4200a9dec83fb8a42bb7db53cf97f1da131504a058ead85d0a9e4e32be14098bc9b4d1a5a8dd023100f9c7de178a286329103f684d1eab1ccfe359c65a41a1459d7f535b703c57048f25931b1670ab4ec7a812d94c69063522')!
+	// library generated signature
+	sign0 := pvkey.sign(message_tobe_signed, opt0)!
+	v00 := pbkey.verify(message_tobe_signed, sign0, opt0)!
+	// this own signature should assert into true
+	assert v00 == true
+
+	// verify online-generated signature
+	v01 := pbkey.verify(message_tobe_signed, online_sign0, opt0)!
+	assert v01 == true
+
+	// with public_key_sample key
+	v02 := pbk.verify(message_tobe_signed, sign0, opt0)!
+	assert v02 == true
+	v03 := pbk.verify(message_tobe_signed, online_sign0, opt0)!
+	assert v03 == true
+
+	// Use smaller custom hash
+	opt1 := SignerOpts{
+		hash_config:        .with_custom_hash
+		allow_custom_hash:  true
+		allow_smaller_size: true
+		custom_hash:        sha1.new()
+	}
+	// online-generated signature with SHA1 digest
+	online_sign1 := hex.decode('306602310084299d8a70bf512c25cd2b79ae36509572f2bd6f198baeee074683578a70b4af8008e1cf451a2df1a887cf43daff4eea023100dceb267fe5037025c2af9f37911e05a36cbe666dd90fd6904020b5db056e86f25f9439a0ccb443d113b174cab6e2ad61')!
+	// library generated signature
+	sign1 := pvkey.sign(message_tobe_signed, opt1)!
+	verified1 := pbkey.verify(message_tobe_signed, sign1, opt1)!
+	// this own signature should assert into true
+	assert verified1 == true
+	// verify online-generated signature
+	verified11 := pbkey.verify(message_tobe_signed, online_sign1, opt1)!
+	assert verified11 == true
+
+	// verify with public_key_sample key
+	v11 := pbk.verify(message_tobe_signed, sign1, opt1)!
+	assert v11 == true
+	v12 := pbk.verify(message_tobe_signed, online_sign1, opt1)!
+	assert v12 == true
+
+	pvkey.free()
+	pbkey.free()
+	pbk.free()
+}