x.json2.decoder2: support fully struct attributes (#22741)

vlib/x/json2/decoder2/attributes_test.v

@@ -0,0 +1,99 @@
+import x.json2.decoder2 as json
+
+struct StruWithJsonAttribute {
+	a     int
+	name2 string @[json: 'name']
+	b     int
+}
+
+struct StruWithSkipAttribute {
+	a    int
+	name ?string @[skip]
+	b    int
+}
+
+struct StruWithJsonSkipAttribute {
+	a    int
+	name ?string @[json: '-']
+	b    int
+}
+
+struct StruWithOmitemptyAttribute {
+	a    int
+	name ?string @[omitempty]
+	b    int
+}
+
+struct StruWithRawAttribute {
+	a      int
+	name   string @[raw]
+	object string @[raw]
+	b      int
+}
+
+struct StruWithRequiredAttribute {
+	a                 int
+	name              string  @[required]
+	skip_and_required ?string @[required; skip]
+	b                 int
+}
+
+fn test_skip_and_rename_attributes() {
+	assert json.decode[StruWithJsonAttribute]('{"name": "hola1", "a": 2, "b": 3}')! == StruWithJsonAttribute{
+		a:     2
+		name2: 'hola1'
+		b:     3
+	}, '`json` attribute not working'
+
+	assert json.decode[StruWithSkipAttribute]('{"name": "hola2", "a": 2, "b": 3}')! == StruWithSkipAttribute{
+		a:    2
+		name: none
+		b:    3
+	}, '`skip` attribute not working'
+
+	assert json.decode[StruWithJsonSkipAttribute]('{"name": "hola3", "a": 2, "b": 3}')! == StruWithJsonSkipAttribute{
+		a:    2
+		name: none
+		b:    3
+	}, " `json: '-'` skip attribute not working"
+
+	assert json.decode[StruWithOmitemptyAttribute]('{"name": "", "a": 2, "b": 3}')! == StruWithOmitemptyAttribute{
+		a:    2
+		name: none
+		b:    3
+	}, '`omitempty` attribute not working'
+
+	assert json.decode[StruWithOmitemptyAttribute]('{"name": "hola", "a": 2, "b": 3}')! == StruWithOmitemptyAttribute{
+		a:    2
+		name: 'hola'
+		b:    3
+	}, '`omitempty` attribute not working'
+}
+
+fn test_raw_attribute() {
+	assert json.decode[StruWithRawAttribute]('{"name": "hola", "a": 2, "object": {"c": 4, "d": 5}, "b": 3}')! == StruWithRawAttribute{
+		a:      2
+		name:   '"hola"'
+		object: '{"c": 4, "d": 5}'
+		b:      3
+	}, '`raw` attribute not working'
+}
+
+fn test_required_attribute() {
+	assert json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "skip_and_required": "hola", "b": 3}')! == StruWithRequiredAttribute{
+		a:                 2
+		name:              'hola'
+		skip_and_required: none
+		b:                 3
+	}, '`required` attribute not working'
+
+	mut has_error := false
+
+	json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "b": 3}') or {
+		has_error = true
+		assert err.msg() == 'missing required field `skip_and_required`'
+	}
+
+	assert has_error, '`required` attribute not working. It should have failed'
+	has_error = false
+}

vlib/x/json2/decoder2/decode.v

@@ -2,42 +2,79 @@ module decoder2
 
 import strconv
 import time
+import strings
+
+const null_in_string = 'null'
+
+const true_in_string = 'true'
+
+const false_in_string = 'false'
+
+const float_zero_in_string = '0.0'
 
 // Node represents a node in a linked list to store ValueInfo.
-struct Node {
+struct Node[T] {
-	value ValueInfo
 mut:
-	next &Node = unsafe { nil } // next is the next node in the linked list.
+	value T
+	next  &Node[T] = unsafe { nil } // next is the next node in the linked list.
 }
 
 // ValueInfo represents the position and length of a value, such as string, number, array, object key, and object value in a JSON string.
 struct ValueInfo {
-	position   int       // The position of the value in the JSON string.
+	position int // The position of the value in the JSON string.
+pub:
 	value_kind ValueKind // The kind of the value.
 mut:
 	length int // The length of the value in the JSON string.
 }
 
+struct StructFieldInfo {
+	field_name_str voidptr
+	field_name_len int
+	json_name_ptr  voidptr
+	json_name_len  int
+	is_omitempty   bool
+	is_skip        bool
+	is_required    bool
+	is_raw         bool
+mut:
+	decoded_with_value_info_node &Node[ValueInfo] = unsafe { nil }
+}
+
 // Decoder represents a JSON decoder.
 struct Decoder {
 	json string // json is the JSON data to be decoded.
 mut:
-	values_info  LinkedList // A linked list to store ValueInfo.
+	values_info  LinkedList[ValueInfo] // A linked list to store ValueInfo.
-	checker_idx  int        // checker_idx is the current index of the decoder.
+	checker_idx  int                   // checker_idx is the current index of the decoder.
-	current_node &Node = unsafe { nil } // The current node in the linked list.
+	current_node &Node[ValueInfo] = unsafe { nil } // The current node in the linked list.
+}
+
+// new_decoder creates a new JSON decoder.
+pub fn new_decoder[T](json string) !Decoder {
+	mut decoder := Decoder{
+		json: json
+	}
+
+	decoder.check_json_format(json)!
+	check_if_json_match[T](json)!
+
+	decoder.current_node = decoder.values_info.head
+
+	return decoder
 }
 
 // LinkedList represents a linked list to store ValueInfo.
-struct LinkedList {
+struct LinkedList[T] {
 mut:
-	head &Node = unsafe { nil } // head is the first node in the linked list.
+	head &Node[T] = unsafe { nil } // head is the first node in the linked list.
-	tail &Node = unsafe { nil } // tail is the last node in the linked list.
+	tail &Node[T] = unsafe { nil } // tail is the last node in the linked list.
 	len  int // len is the length of the linked list.
 }
 
 // push adds a new element to the linked list.
-fn (mut list LinkedList) push(value ValueInfo) {
+fn (mut list LinkedList[T]) push(value T) {
-	new_node := &Node{
+	new_node := &Node[T]{
 		value: value
 	}
 	if list.head == unsafe { nil } {
@@ -51,12 +88,12 @@ fn (mut list LinkedList) push(value ValueInfo) {
 }
 
 // last returns the last element added to the linked list.
-fn (list LinkedList) last() &ValueInfo {
+fn (list &LinkedList[T]) last() &T {
 	return &list.tail.value
 }
 
 // str returns a string representation of the linked list.
-fn (list LinkedList) str() string {
+fn (list &LinkedList[ValueInfo]) str() string {
 	mut result_buffer := []u8{}
 	mut current := list.head
 	for current != unsafe { nil } {
@@ -69,8 +106,24 @@ fn (list LinkedList) str() string {
 	return result_buffer.bytestr()
 }
 
+@[manualfree]
+fn (list &LinkedList[T]) str() string {
+	mut sb := strings.new_builder(128)
+	defer {
+		unsafe { sb.free() }
+	}
+	mut current := list.head
+	for current != unsafe { nil } {
+		value_as_string := current.value.str()
+		sb.write_string(value_as_string)
+		sb.write_u8(u8(` `))
+		current = current.next
+	}
+	return sb.str()
+}
+
 @[unsafe]
-fn (list &LinkedList) free() {
+fn (list &LinkedList[T]) free() {
 	mut current := list.head
 	for current != unsafe { nil } {
 		mut next := current.next
@@ -201,7 +254,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
 			}
 
 			is_not_ok := unsafe {
-				vmemcmp(checker.json.str + checker.checker_idx, 'null'.str, 4)
+				vmemcmp(checker.json.str + checker.checker_idx, null_in_string.str, null_in_string.len)
 			}
 
 			if is_not_ok != 0 {
@@ -453,7 +506,8 @@ fn (mut checker Decoder) check_json_format(val string) ! {
 					}
 
 					is_not_ok := unsafe {
-						vmemcmp(checker.json.str + checker.checker_idx, 'true'.str, 4)
+						vmemcmp(checker.json.str + checker.checker_idx, true_in_string.str,
+							true_in_string.len)
 					}
 
 					if is_not_ok != 0 {
@@ -468,7 +522,8 @@ fn (mut checker Decoder) check_json_format(val string) ! {
 					}
 
 					is_not_ok := unsafe {
-						vmemcmp(checker.json.str + checker.checker_idx, 'false'.str, 5)
+						vmemcmp(checker.json.str + checker.checker_idx, false_in_string.str,
+							false_in_string.len)
 					}
 
 					if is_not_ok != 0 {
@@ -518,6 +573,7 @@ pub fn decode[T](val string) !T {
 }
 
 // decode_value decodes a value from the JSON nodes.
+@[manualfree]
 fn (mut decoder Decoder) decode_value[T](mut val T) ! {
 	$if T is $option {
 		mut unwrapped_val := create_value_from_optional(val.$(field.name))
@@ -599,11 +655,45 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
 	} $else $if T.unaliased_typ is $struct {
 		struct_info := decoder.current_node.value
 
+		// struct field info linked list
+		mut struct_fields_info := LinkedList[StructFieldInfo]{}
+
+		$for field in T.fields {
+			mut json_name_str := field.name.str
+			mut json_name_len := field.name.len
+
+			for attr in field.attrs {
+				if attr.starts_with('json:') {
+					if attr.len <= 6 {
+						return error('`json` attribute must have an argument')
+					}
+					json_name_str = unsafe { attr.str + 6 }
+					json_name_len = attr.len - 6
+					break
+				}
+				continue
+			}
+
+			struct_fields_info.push(StructFieldInfo{
+				field_name_str: voidptr(field.name.str)
+				field_name_len: field.name.len
+				json_name_ptr:  voidptr(json_name_str)
+				json_name_len:  json_name_len
+				is_omitempty:   field.attrs.contains('omitempty')
+				is_skip:        field.attrs.contains('skip') || field.attrs.contains('json: -')
+				is_required:    field.attrs.contains('required')
+				is_raw:         field.attrs.contains('raw')
+			})
+		}
 		if struct_info.value_kind == .object {
 			struct_position := struct_info.position
 			struct_end := struct_position + struct_info.length
 
 			decoder.current_node = decoder.current_node.next
+
+			mut current_field_info := struct_fields_info.head
+
+			// json object loop
 			for {
 				if decoder.current_node == unsafe { nil } {
 					break
@@ -615,33 +705,149 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
 					break
 				}
 
-				decoder.current_node = decoder.current_node.next
+				current_field_info = struct_fields_info.head
 
-				$for field in T.fields {
+				// field loop
-					if key_info.length - 2 == field.name.len {
+				for {
-						// This `vmemcmp` compares the name of a key in a JSON with a given struct field.
+					if current_field_info == unsafe { nil } {
+						decoder.current_node = decoder.current_node.next
+						break
+					}
+
+					if current_field_info.value.is_skip {
+						if current_field_info.value.is_required == false {
+							current_field_info = current_field_info.next
+							continue
+						}
+					}
+
+					if current_field_info.value.is_omitempty {
+						match decoder.current_node.next.value.value_kind {
+							.null {
+								current_field_info = current_field_info.next
+								continue
+							}
+							.string_ {
+								if decoder.current_node.next.value.length == 2 {
+									current_field_info = current_field_info.next
+									continue
+								}
+							}
+							.number {
+								if decoder.json[decoder.current_node.next.value.position] == `0` {
+									if decoder.current_node.next.value.length == 1 {
+										current_field_info = current_field_info.next
+										continue
+									} else if decoder.current_node.next.value.length == 3 {
+										if unsafe {
+											vmemcmp(decoder.json.str +
+												decoder.current_node.next.value.position,
+												float_zero_in_string.str, float_zero_in_string.len) == 0
+										} {
+											current_field_info = current_field_info.next
+											continue
+										}
+									}
+								}
+							}
+							else {}
+						}
+					}
+
+					// check if the key matches the field name
+					if key_info.length - 2 == current_field_info.value.json_name_len {
 						if unsafe {
-							vmemcmp(decoder.json.str + key_info.position + 1, field.name.str,
+							vmemcmp(decoder.json.str + key_info.position + 1, current_field_info.value.json_name_ptr,
-								field.name.len) == 0
+								current_field_info.value.json_name_len) == 0
 						} {
-							$if field.typ is $option {
+							$for field in T.fields {
-								mut unwrapped_val := create_value_from_optional(val.$(field.name))
+								if field.name.len == current_field_info.value.field_name_len {
-								decoder.decode_value(mut unwrapped_val)!
+									if unsafe {
-								val.$(field.name) = unwrapped_val
+										(&u8(current_field_info.value.field_name_str)).vstring_with_len(field.name.len) == field.name
-							} $else {
+									} {
-								decoder.decode_value(mut val.$(field.name))!
+										// value node
+										decoder.current_node = decoder.current_node.next
+
+										if current_field_info.value.is_skip {
+											if current_field_info.value.is_required == false {
+												return error('This should not happen. Please, file a bug. `skip` field should not be processed here without a `required` attribute')
+											}
+											current_field_info.value.decoded_with_value_info_node = decoder.current_node
+											break
+										}
+
+										if current_field_info.value.is_raw {
+											$if field.typ is $enum {
+												// workaround to avoid the error: enums can only be assigned `int` values
+												return error('`raw` attribute cannot be used with enum fields')
+											} $else $if field.typ is string || field.typ is ?string {
+												position := decoder.current_node.value.position
+												end := position + decoder.current_node.value.length
+
+												val.$(field.name) = decoder.json[position..end]
+												decoder.current_node = decoder.current_node.next
+
+												for {
+													if decoder.current_node == unsafe { nil }
+														|| decoder.current_node.value.position + decoder.current_node.value.length >= end {
+														break
+													}
+
+													decoder.current_node = decoder.current_node.next
+												}
+											} $else {
+												return error('`raw` attribute can only be used with string fields')
+											}
+										} else {
+											$if field.typ is $option {
+												mut unwrapped_val := create_value_from_optional(val.$(field.name))
+												decoder.decode_value(mut unwrapped_val)!
+												val.$(field.name) = unwrapped_val
+											} $else {
+												decoder.decode_value(mut val.$(field.name))!
+											}
+										}
+										current_field_info.value.decoded_with_value_info_node = decoder.current_node
+										break
+									}
+								}
 							}
 						}
 					}
+					current_field_info = current_field_info.next
 				}
 			}
+
+			// check if all required fields are present
+			current_field_info = struct_fields_info.head
+
+			for {
+				if current_field_info == unsafe { nil } {
+					break
+				}
+
+				if current_field_info.value.is_required == false {
+					current_field_info = current_field_info.next
+					continue
+				}
+				if current_field_info.value.decoded_with_value_info_node == unsafe { nil } {
+					return error('missing required field `${unsafe {
+						tos(current_field_info.value.field_name_str, current_field_info.value.field_name_len)
+					}}`')
+				}
+				current_field_info = current_field_info.next
+			}
+		}
+		unsafe {
+			struct_fields_info.free()
 		}
 		return
 	} $else $if T.unaliased_typ is bool {
 		value_info := decoder.current_node.value
 
 		unsafe {
-			val = vmemcmp(decoder.json.str + value_info.position, 'true'.str, 4) == 0
+			val = vmemcmp(decoder.json.str + value_info.position, true_in_string.str,
+				true_in_string.len) == 0
 		}
 	} $else $if T.unaliased_typ in [$float, $int, $enum] {
 		value_info := decoder.current_node.value
@@ -750,7 +956,7 @@ fn create_value_from_optional[T](val ?T) T {
 	return T{}
 }
 
-fn utf8_byte_length(unicode_value u32) int {
+fn utf8_byte_len(unicode_value u32) int {
 	if unicode_value <= 0x7F {
 		return 1
 	} else if unicode_value <= 0x7FF {
@@ -800,7 +1006,7 @@ fn (mut decoder Decoder) calculate_string_space_and_escapes() !(int, []int) {
 						idx + 5]
 					unicode_value := u32(strconv.parse_int(hex_str, 16, 32)!)
 					// Determine the number of bytes needed for this Unicode character in UTF-8
-					space_required += utf8_byte_length(unicode_value)
+					space_required += utf8_byte_len(unicode_value)
 					idx += 4 // Skip the next 4 hex digits
 
 					// REVIEW: If the Unicode character is a surrogate pair, we need to skip the next \uXXXX sequence?
@@ -829,7 +1035,7 @@ fn generate_unicode_escape_sequence(escape_sequence_byte []u8) ![]u8 {
 	}
 
 	unicode_value := u32(strconv.parse_int(escape_sequence_byte.bytestr(), 16, 32)!)
-	mut utf8_bytes := []u8{cap: utf8_byte_length(unicode_value)}
+	mut utf8_bytes := []u8{cap: utf8_byte_len(unicode_value)}
 
 	if unicode_value <= 0x7F {
 		utf8_bytes << u8(unicode_value)