crypto.ecdsa: fix bug in .with_no_hash handling (#23612)

vlib/crypto/ecdsa/ecdsa.v

@@ -510,43 +510,29 @@ pub mut:
 	custom_hash &hash.Hash = unsafe { nil }
 }
 
-// calc_digest tries to calculates digest (hash) of the message based on options provided.
+fn calc_digest_with_recommended_hash(key &C.EC_KEY, msg []u8) ![]u8 {
-// If the options was with_no_hash, its return default message without hashing.
-fn calc_digest(key &C.EC_KEY, message []u8, opt SignerOpts) ![]u8 {
-	if message.len == 0 {
-		return error('null-length messages')
-	}
-	// we're working on mutable copy of SignerOpts, with some issues when make it as a mutable.
-	// ie, declaring a mutable parameter that accepts a struct with the `@[params]` attribute is not allowed.
-	mut cfg := opt
-	match cfg.hash_config {
-		.with_no_hash {
-			// return original message
-			return message
-		}
-		.with_recommended_hash {
 	h := recommended_hash(key)!
 	match h {
 		.sha256 {
-					return sha256.sum256(message)
+			return sha256.sum256(msg)
 		}
 		.sha384 {
-					return sha512.sum384(message)
+			return sha512.sum384(msg)
 		}
 		.sha512 {
-					return sha512.sum512(message)
+			return sha512.sum512(msg)
 		}
 		else {
 			return error('Unsupported hash')
 		}
 	}
 }
-		.with_custom_hash {
+
-			if !cfg.allow_custom_hash {
+// calc_digest tries to calculates digest (hash) of the message based on options provided.
-				return error('custom hash was not allowed, set it into true')
+// If the options was .with_no_hash, its has the same behaviour with .with_recommended_hash.
-			}
+fn calc_digest(key &C.EC_KEY, message []u8, opt SignerOpts) ![]u8 {
-			if cfg.custom_hash == unsafe { nil } {
+	if message.len == 0 {
-				return error('Custom hasher was not defined')
+		return error('null-length messages')
 	}
 	// check key size bits
 	group := voidptr(C.EC_KEY_get0_group(key))
@@ -556,6 +542,23 @@ fn calc_digest(key &C.EC_KEY, message []u8, opt SignerOpts) ![]u8 {
 	num_bits := C.EC_GROUP_get_degree(group)
 	// check for key size matching
 	key_size := (num_bits + 7) / 8
+	// we're working on mutable copy of SignerOpts, with some issues when make it as a mutable.
+	// ie, declaring a mutable parameter that accepts a struct with the `@[params]` attribute is not allowed.
+	mut cfg := opt
+	match cfg.hash_config {
+		// There are issues when your message size was exceeds the current key size with .with_no_hash options.
+		// See https://discord.com/channels/592103645835821068/592114487759470596/1334319744098107423
+		// So, we aliased it into .with_recommended_hash
+		.with_no_hash, .with_recommended_hash {
+			return calc_digest_with_recommended_hash(key, message)!
+		}
+		.with_custom_hash {
+			if !cfg.allow_custom_hash {
+				return error('custom hash was not allowed, set it into true')
+			}
+			if cfg.custom_hash == unsafe { nil } {
+				return error('Custom hasher was not defined')
+			}
 			// If current Private Key size is bigger then current hash output size,
 			// by default its not allowed, until set the allow_smaller_size into true
 			if key_size > cfg.custom_hash.size() {

vlib/crypto/ecdsa/ecdsa_test.v

@@ -204,3 +204,27 @@ fn test_private_key_new() ! {
 	pubkey2.free()
 	pubkey3.free()
 }
+
+// See https://discord.com/channels/592103645835821068/592114487759470596/1334319744098107423
+fn test_key_with_msg_exceed_key_size() ! {
+	pv := PrivateKey.new()!
+	msg := 'a'.repeat(200).bytes()
+	opt := SignerOpts{
+		hash_config: .with_no_hash
+	}
+	signed := pv.sign(msg, opt)!
+	pb := pv.public_key()!
+
+	// should be verified
+	st := pb.verify(msg, signed, opt)!
+	assert st
+
+	// different msg should not be verified
+	other_msg := 'a'.repeat(392).bytes()
+	ds := pb.verify(other_msg, signed, opt)!
+	// This should assert to false.
+	assert !ds
+
+	pv.free()
+	pb.free()
+}

vlib/crypto/ecdsa/util_test.v

@@ -27,10 +27,11 @@ fn test_load_pubkey_from_der_serialized_bytes() ! {
 	block, _ := pem.decode(public_key_sample) or { panic(err) }
 	pbkey := pubkey_from_bytes(block.data)!
 
+	// .with_no_hash currently changed to have same behaviour with .with_recommended_hash
 	status_without_hashed := pbkey.verify(message_tobe_signed, expected_signature,
 		hash_config: .with_no_hash
 	)!
-	assert status_without_hashed == false
+	assert status_without_hashed == true
 
 	// expected signature was comes from hashed message with sha384
 	status_with_hashed := pbkey.verify(message_tobe_signed, expected_signature)!