From 210239fb01dd992c8c7aa6d013da4ffeaffde9a6 Mon Sep 17 00:00:00 2001 From: Hitalo Souza Date: Fri, 6 Dec 2024 07:53:07 -0400 Subject: [PATCH] x.json2.decoder2: support fully struct attributes (#22741) --- vlib/x/json2/decoder2/attributes_test.v | 99 +++++++++ vlib/x/json2/decoder2/decode.v | 274 +++++++++++++++++++++--- 2 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 vlib/x/json2/decoder2/attributes_test.v diff --git a/vlib/x/json2/decoder2/attributes_test.v b/vlib/x/json2/decoder2/attributes_test.v new file mode 100644 index 0000000000..2a7a566211 --- /dev/null +++ b/vlib/x/json2/decoder2/attributes_test.v @@ -0,0 +1,99 @@ +import x.json2.decoder2 as json + +struct StruWithJsonAttribute { + a int + name2 string @[json: 'name'] + b int +} + +struct StruWithSkipAttribute { + a int + name ?string @[skip] + b int +} + +struct StruWithJsonSkipAttribute { + a int + name ?string @[json: '-'] + b int +} + +struct StruWithOmitemptyAttribute { + a int + name ?string @[omitempty] + b int +} + +struct StruWithRawAttribute { + a int + name string @[raw] + object string @[raw] + b int +} + +struct StruWithRequiredAttribute { + a int + name string @[required] + skip_and_required ?string @[required; skip] + b int +} + +fn test_skip_and_rename_attributes() { + assert json.decode[StruWithJsonAttribute]('{"name": "hola1", "a": 2, "b": 3}')! == StruWithJsonAttribute{ + a: 2 + name2: 'hola1' + b: 3 + }, '`json` attribute not working' + + assert json.decode[StruWithSkipAttribute]('{"name": "hola2", "a": 2, "b": 3}')! == StruWithSkipAttribute{ + a: 2 + name: none + b: 3 + }, '`skip` attribute not working' + + assert json.decode[StruWithJsonSkipAttribute]('{"name": "hola3", "a": 2, "b": 3}')! == StruWithJsonSkipAttribute{ + a: 2 + name: none + b: 3 + }, " `json: '-'` skip attribute not working" + + assert json.decode[StruWithOmitemptyAttribute]('{"name": "", "a": 2, "b": 3}')! == StruWithOmitemptyAttribute{ + a: 2 + name: none + b: 3 + }, '`omitempty` attribute not working' + + assert json.decode[StruWithOmitemptyAttribute]('{"name": "hola", "a": 2, "b": 3}')! == StruWithOmitemptyAttribute{ + a: 2 + name: 'hola' + b: 3 + }, '`omitempty` attribute not working' +} + +fn test_raw_attribute() { + assert json.decode[StruWithRawAttribute]('{"name": "hola", "a": 2, "object": {"c": 4, "d": 5}, "b": 3}')! == StruWithRawAttribute{ + a: 2 + name: '"hola"' + object: '{"c": 4, "d": 5}' + b: 3 + }, '`raw` attribute not working' +} + +fn test_required_attribute() { + assert json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "skip_and_required": "hola", "b": 3}')! == StruWithRequiredAttribute{ + a: 2 + name: 'hola' + skip_and_required: none + b: 3 + }, '`required` attribute not working' + + mut has_error := false + + json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "b": 3}') or { + has_error = true + assert err.msg() == 'missing required field `skip_and_required`' + } + + assert has_error, '`required` attribute not working. It should have failed' + has_error = false +} diff --git a/vlib/x/json2/decoder2/decode.v b/vlib/x/json2/decoder2/decode.v index 9774984341..4547c4e796 100644 --- a/vlib/x/json2/decoder2/decode.v +++ b/vlib/x/json2/decoder2/decode.v @@ -2,42 +2,79 @@ module decoder2 import strconv import time +import strings + +const null_in_string = 'null' + +const true_in_string = 'true' + +const false_in_string = 'false' + +const float_zero_in_string = '0.0' // Node represents a node in a linked list to store ValueInfo. -struct Node { - value ValueInfo +struct Node[T] { mut: - next &Node = unsafe { nil } // next is the next node in the linked list. + value T + next &Node[T] = unsafe { nil } // next is the next node in the linked list. } // ValueInfo represents the position and length of a value, such as string, number, array, object key, and object value in a JSON string. struct ValueInfo { - position int // The position of the value in the JSON string. + position int // The position of the value in the JSON string. +pub: value_kind ValueKind // The kind of the value. mut: length int // The length of the value in the JSON string. } +struct StructFieldInfo { + field_name_str voidptr + field_name_len int + json_name_ptr voidptr + json_name_len int + is_omitempty bool + is_skip bool + is_required bool + is_raw bool +mut: + decoded_with_value_info_node &Node[ValueInfo] = unsafe { nil } +} + // Decoder represents a JSON decoder. struct Decoder { json string // json is the JSON data to be decoded. mut: - values_info LinkedList // A linked list to store ValueInfo. - checker_idx int // checker_idx is the current index of the decoder. - current_node &Node = unsafe { nil } // The current node in the linked list. + values_info LinkedList[ValueInfo] // A linked list to store ValueInfo. + checker_idx int // checker_idx is the current index of the decoder. + current_node &Node[ValueInfo] = unsafe { nil } // The current node in the linked list. +} + +// new_decoder creates a new JSON decoder. +pub fn new_decoder[T](json string) !Decoder { + mut decoder := Decoder{ + json: json + } + + decoder.check_json_format(json)! + check_if_json_match[T](json)! + + decoder.current_node = decoder.values_info.head + + return decoder } // LinkedList represents a linked list to store ValueInfo. -struct LinkedList { +struct LinkedList[T] { mut: - head &Node = unsafe { nil } // head is the first node in the linked list. - tail &Node = unsafe { nil } // tail is the last node in the linked list. + head &Node[T] = unsafe { nil } // head is the first node in the linked list. + tail &Node[T] = unsafe { nil } // tail is the last node in the linked list. len int // len is the length of the linked list. } // push adds a new element to the linked list. -fn (mut list LinkedList) push(value ValueInfo) { - new_node := &Node{ +fn (mut list LinkedList[T]) push(value T) { + new_node := &Node[T]{ value: value } if list.head == unsafe { nil } { @@ -51,12 +88,12 @@ fn (mut list LinkedList) push(value ValueInfo) { } // last returns the last element added to the linked list. -fn (list LinkedList) last() &ValueInfo { +fn (list &LinkedList[T]) last() &T { return &list.tail.value } // str returns a string representation of the linked list. -fn (list LinkedList) str() string { +fn (list &LinkedList[ValueInfo]) str() string { mut result_buffer := []u8{} mut current := list.head for current != unsafe { nil } { @@ -69,8 +106,24 @@ fn (list LinkedList) str() string { return result_buffer.bytestr() } +@[manualfree] +fn (list &LinkedList[T]) str() string { + mut sb := strings.new_builder(128) + defer { + unsafe { sb.free() } + } + mut current := list.head + for current != unsafe { nil } { + value_as_string := current.value.str() + sb.write_string(value_as_string) + sb.write_u8(u8(` `)) + current = current.next + } + return sb.str() +} + @[unsafe] -fn (list &LinkedList) free() { +fn (list &LinkedList[T]) free() { mut current := list.head for current != unsafe { nil } { mut next := current.next @@ -201,7 +254,7 @@ fn (mut checker Decoder) check_json_format(val string) ! { } is_not_ok := unsafe { - vmemcmp(checker.json.str + checker.checker_idx, 'null'.str, 4) + vmemcmp(checker.json.str + checker.checker_idx, null_in_string.str, null_in_string.len) } if is_not_ok != 0 { @@ -453,7 +506,8 @@ fn (mut checker Decoder) check_json_format(val string) ! { } is_not_ok := unsafe { - vmemcmp(checker.json.str + checker.checker_idx, 'true'.str, 4) + vmemcmp(checker.json.str + checker.checker_idx, true_in_string.str, + true_in_string.len) } if is_not_ok != 0 { @@ -468,7 +522,8 @@ fn (mut checker Decoder) check_json_format(val string) ! { } is_not_ok := unsafe { - vmemcmp(checker.json.str + checker.checker_idx, 'false'.str, 5) + vmemcmp(checker.json.str + checker.checker_idx, false_in_string.str, + false_in_string.len) } if is_not_ok != 0 { @@ -518,6 +573,7 @@ pub fn decode[T](val string) !T { } // decode_value decodes a value from the JSON nodes. +@[manualfree] fn (mut decoder Decoder) decode_value[T](mut val T) ! { $if T is $option { mut unwrapped_val := create_value_from_optional(val.$(field.name)) @@ -599,11 +655,45 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! { } $else $if T.unaliased_typ is $struct { struct_info := decoder.current_node.value + // struct field info linked list + mut struct_fields_info := LinkedList[StructFieldInfo]{} + + $for field in T.fields { + mut json_name_str := field.name.str + mut json_name_len := field.name.len + + for attr in field.attrs { + if attr.starts_with('json:') { + if attr.len <= 6 { + return error('`json` attribute must have an argument') + } + json_name_str = unsafe { attr.str + 6 } + json_name_len = attr.len - 6 + break + } + continue + } + + struct_fields_info.push(StructFieldInfo{ + field_name_str: voidptr(field.name.str) + field_name_len: field.name.len + json_name_ptr: voidptr(json_name_str) + json_name_len: json_name_len + is_omitempty: field.attrs.contains('omitempty') + is_skip: field.attrs.contains('skip') || field.attrs.contains('json: -') + is_required: field.attrs.contains('required') + is_raw: field.attrs.contains('raw') + }) + } if struct_info.value_kind == .object { struct_position := struct_info.position struct_end := struct_position + struct_info.length decoder.current_node = decoder.current_node.next + + mut current_field_info := struct_fields_info.head + + // json object loop for { if decoder.current_node == unsafe { nil } { break @@ -615,33 +705,149 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! { break } - decoder.current_node = decoder.current_node.next + current_field_info = struct_fields_info.head - $for field in T.fields { - if key_info.length - 2 == field.name.len { - // This `vmemcmp` compares the name of a key in a JSON with a given struct field. + // field loop + for { + if current_field_info == unsafe { nil } { + decoder.current_node = decoder.current_node.next + break + } + + if current_field_info.value.is_skip { + if current_field_info.value.is_required == false { + current_field_info = current_field_info.next + continue + } + } + + if current_field_info.value.is_omitempty { + match decoder.current_node.next.value.value_kind { + .null { + current_field_info = current_field_info.next + continue + } + .string_ { + if decoder.current_node.next.value.length == 2 { + current_field_info = current_field_info.next + continue + } + } + .number { + if decoder.json[decoder.current_node.next.value.position] == `0` { + if decoder.current_node.next.value.length == 1 { + current_field_info = current_field_info.next + continue + } else if decoder.current_node.next.value.length == 3 { + if unsafe { + vmemcmp(decoder.json.str + + decoder.current_node.next.value.position, + float_zero_in_string.str, float_zero_in_string.len) == 0 + } { + current_field_info = current_field_info.next + continue + } + } + } + } + else {} + } + } + + // check if the key matches the field name + if key_info.length - 2 == current_field_info.value.json_name_len { if unsafe { - vmemcmp(decoder.json.str + key_info.position + 1, field.name.str, - field.name.len) == 0 + vmemcmp(decoder.json.str + key_info.position + 1, current_field_info.value.json_name_ptr, + current_field_info.value.json_name_len) == 0 } { - $if field.typ is $option { - mut unwrapped_val := create_value_from_optional(val.$(field.name)) - decoder.decode_value(mut unwrapped_val)! - val.$(field.name) = unwrapped_val - } $else { - decoder.decode_value(mut val.$(field.name))! + $for field in T.fields { + if field.name.len == current_field_info.value.field_name_len { + if unsafe { + (&u8(current_field_info.value.field_name_str)).vstring_with_len(field.name.len) == field.name + } { + // value node + decoder.current_node = decoder.current_node.next + + if current_field_info.value.is_skip { + if current_field_info.value.is_required == false { + return error('This should not happen. Please, file a bug. `skip` field should not be processed here without a `required` attribute') + } + current_field_info.value.decoded_with_value_info_node = decoder.current_node + break + } + + if current_field_info.value.is_raw { + $if field.typ is $enum { + // workaround to avoid the error: enums can only be assigned `int` values + return error('`raw` attribute cannot be used with enum fields') + } $else $if field.typ is string || field.typ is ?string { + position := decoder.current_node.value.position + end := position + decoder.current_node.value.length + + val.$(field.name) = decoder.json[position..end] + decoder.current_node = decoder.current_node.next + + for { + if decoder.current_node == unsafe { nil } + || decoder.current_node.value.position + decoder.current_node.value.length >= end { + break + } + + decoder.current_node = decoder.current_node.next + } + } $else { + return error('`raw` attribute can only be used with string fields') + } + } else { + $if field.typ is $option { + mut unwrapped_val := create_value_from_optional(val.$(field.name)) + decoder.decode_value(mut unwrapped_val)! + val.$(field.name) = unwrapped_val + } $else { + decoder.decode_value(mut val.$(field.name))! + } + } + current_field_info.value.decoded_with_value_info_node = decoder.current_node + break + } + } } } } + current_field_info = current_field_info.next } } + + // check if all required fields are present + current_field_info = struct_fields_info.head + + for { + if current_field_info == unsafe { nil } { + break + } + + if current_field_info.value.is_required == false { + current_field_info = current_field_info.next + continue + } + if current_field_info.value.decoded_with_value_info_node == unsafe { nil } { + return error('missing required field `${unsafe { + tos(current_field_info.value.field_name_str, current_field_info.value.field_name_len) + }}`') + } + current_field_info = current_field_info.next + } + } + unsafe { + struct_fields_info.free() } return } $else $if T.unaliased_typ is bool { value_info := decoder.current_node.value unsafe { - val = vmemcmp(decoder.json.str + value_info.position, 'true'.str, 4) == 0 + val = vmemcmp(decoder.json.str + value_info.position, true_in_string.str, + true_in_string.len) == 0 } } $else $if T.unaliased_typ in [$float, $int, $enum] { value_info := decoder.current_node.value @@ -750,7 +956,7 @@ fn create_value_from_optional[T](val ?T) T { return T{} } -fn utf8_byte_length(unicode_value u32) int { +fn utf8_byte_len(unicode_value u32) int { if unicode_value <= 0x7F { return 1 } else if unicode_value <= 0x7FF { @@ -800,7 +1006,7 @@ fn (mut decoder Decoder) calculate_string_space_and_escapes() !(int, []int) { idx + 5] unicode_value := u32(strconv.parse_int(hex_str, 16, 32)!) // Determine the number of bytes needed for this Unicode character in UTF-8 - space_required += utf8_byte_length(unicode_value) + space_required += utf8_byte_len(unicode_value) idx += 4 // Skip the next 4 hex digits // REVIEW: If the Unicode character is a surrogate pair, we need to skip the next \uXXXX sequence? @@ -829,7 +1035,7 @@ fn generate_unicode_escape_sequence(escape_sequence_byte []u8) ![]u8 { } unicode_value := u32(strconv.parse_int(escape_sequence_byte.bytestr(), 16, 32)!) - mut utf8_bytes := []u8{cap: utf8_byte_length(unicode_value)} + mut utf8_bytes := []u8{cap: utf8_byte_len(unicode_value)} if unicode_value <= 0x7F { utf8_bytes << u8(unicode_value)