decoder2: rework decoding of sumtypes (#24949)

vlib/x/json2/decoder2/decode.v

@@ -12,7 +12,7 @@ const false_in_string = 'false'
 
 const float_zero_in_string = '0.0'
 
-const whitespace_chars = [` `, `\t`, `\n`]!
+const whitespace_chars = [` `, `\t`, `\n`, `\r`]!
 
 // Node represents a node in a linked list to store ValueInfo.
 struct Node[T] {
@@ -315,9 +315,6 @@ fn (mut checker Decoder) check_json_format(val string) ! {
 			// check if the JSON string is an empty array
 			if checker_end >= checker.checker_idx + 2 {
 				checker.checker_idx++
-				if val[checker.checker_idx] == `]` {
-					return
-				}
 			} else {
 				return checker.error('EOF error: There are not enough length for an array')
 			}
@@ -332,7 +329,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
 				}
 
 				if val[checker.checker_idx] == `]` {
-					return
+					break
 				}
 
 				if checker.checker_idx >= checker_end - 1 {
@@ -806,8 +803,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
 											$if field.typ is $option {
 												// it would be nicer to do this at the start of the function
 												// but options cant be passed to generic functions
-												if decoder.current_node.value.length == 4
+												if decoder.current_node.value.value_kind == .null {
-													&& decoder.json[decoder.current_node.value.position..decoder.current_node.value.position + 4] == 'null' {
 													val.$(field.name) = none
 												} else {
 													mut unwrapped_val := create_value_from_optional(val.$(field.name)) or {
@@ -981,18 +977,6 @@ fn create_value_from_optional[T](val ?T) ?T {
 	return T{}
 }
 
-fn utf8_byte_len(unicode_value u32) int {
-	if unicode_value <= 0x7F {
-		return 1
-	} else if unicode_value <= 0x7FF {
-		return 2
-	} else if unicode_value <= 0xFFFF {
-		return 3
-	} else {
-		return 4
-	}
-}
-
 // string_buffer_to_generic_number converts a buffer of bytes (data) into a generic type T and
 // stores the result in the provided result pointer.
 // The function supports conversion to the following types:

vlib/x/json2/decoder2/decode_sumtype.v

@@ -2,125 +2,305 @@ module decoder2
 
 import time
 
+fn copy_type[T](t T) T {
+	return T{}
+}
+
 fn (mut decoder Decoder) get_decoded_sumtype_workaround[T](initialized_sumtype T) !T {
 	$if initialized_sumtype is $sumtype {
 		$for v in initialized_sumtype.variants {
 			if initialized_sumtype is v {
-				// workaround for auto generated function considering sumtype as array
+				$if initialized_sumtype !is $option {
-				unsafe {
+					mut val := copy_type(initialized_sumtype)
-					$if initialized_sumtype is $map {
+					decoder.decode_value(mut val)!
-						mut val := initialized_sumtype.clone()
+					return T(val)
-						decoder.decode_value(mut val)!
+				} $else {
-						return T(val)
+					if decoder.current_node.value.value_kind == .null {
-					} $else {
+						decoder.current_node = decoder.current_node.next
-						mut val := initialized_sumtype
+						return T(initialized_sumtype)
-						decoder.decode_value(mut val)!
+					} else {
-						return T(val)
+						decoder.error('sumtype option only support decoding null->none (for now)')!
 					}
 				}
 			}
 		}
 	}
-	return initialized_sumtype
+	decoder.error('could not decode resolved sumtype (should not happen)')!
+	return initialized_sumtype // suppress compiler error
+}
+
+fn (mut decoder Decoder) check_element_type_valid[T](element T, current_node &Node[ValueInfo]) bool {
+	if current_node == unsafe { nil } {
+		$if element is $array || element is $map {
+			return false
+		}
+		return true
+	}
+
+	$if element is $sumtype { // this will always match the first sumtype array/map
+		return true
+	}
+
+	match current_node.value.value_kind {
+		.string_ {
+			$if element is string {
+				return true
+			} $else $if element is time.Time {
+				return true
+			}
+		}
+		.number {
+			$if element is $float {
+				return true
+			} $else $if element is $int {
+				return true
+			} $else $if element is $enum {
+				return true
+			}
+		}
+		.boolean {
+			$if element is bool {
+				return true
+			}
+		}
+		.null {
+			$if element is $option {
+				return true
+			}
+		}
+		.array {
+			$if element is $array {
+				return decoder.check_array_type_valid(element, current_node.next)
+			}
+		}
+		.object {
+			$if element is $map {
+				if current_node.next != unsafe { nil } {
+					return decoder.check_map_type_valid(element, current_node.next.next)
+				} else {
+					return decoder.check_map_type_valid(element, unsafe { nil })
+				}
+			} $else $if element is $struct {
+				return decoder.check_struct_type_valid(element, current_node)
+			}
+		}
+		else {}
+	}
+
+	return false
+}
+
+fn get_array_element_type[T](arr []T) T {
+	return T{}
+}
+
+fn (mut decoder Decoder) check_array_type_valid[T](arr []T, current_node &Node[ValueInfo]) bool {
+	return decoder.check_element_type_valid(get_array_element_type(arr), current_node)
+}
+
+fn (mut decoder Decoder) get_array_type_workaround[T](initialized_sumtype T) bool {
+	$if initialized_sumtype is $sumtype {
+		$for v in initialized_sumtype.variants {
+			if initialized_sumtype is v {
+				$if initialized_sumtype is $array {
+					return decoder.check_element_type_valid(initialized_sumtype, decoder.current_node)
+				}
+			}
+		}
+	}
+	return false
+}
+
+fn get_map_element_type[U, V](m map[U]V) V {
+	return V{}
+}
+
+fn (mut decoder Decoder) check_map_type_valid[T](m T, current_node &Node[ValueInfo]) bool {
+	element := get_map_element_type(m)
+	return decoder.check_element_type_valid(element, current_node)
+}
+
+fn (mut decoder Decoder) check_map_empty_valid[T](m T) bool {
+	element := get_map_element_type(m)
+	return decoder.check_element_type_valid(element, current_node)
+}
+
+fn (mut decoder Decoder) get_map_type_workaround[T](initialized_sumtype T) bool {
+	$if initialized_sumtype is $sumtype {
+		$for v in initialized_sumtype.variants {
+			if initialized_sumtype is v {
+				$if initialized_sumtype is $map {
+					val := copy_type(initialized_sumtype)
+					if decoder.current_node.next != unsafe { nil } {
+						return decoder.check_map_type_valid(val, decoder.current_node.next.next)
+					} else {
+						return decoder.check_map_type_valid(val, unsafe { nil })
+					}
+				}
+			}
+		}
+	}
+	return false
+}
+
+fn (mut decoder Decoder) check_struct_type_valid[T](s T, current_node Node[ValueInfo]) bool {
+	// find "_type" field in json object
+	mut type_field_node := decoder.current_node.next
+	map_position := current_node.value.position
+	map_end := map_position + current_node.value.length
+
+	type_field := '"_type"'
+
+	for {
+		if type_field_node == unsafe { nil } {
+			break
+		}
+
+		key_info := type_field_node.value
+
+		if key_info.position >= map_end {
+			type_field_node = unsafe { nil }
+			break
+		}
+
+		if unsafe {
+			vmemcmp(decoder.json.str + key_info.position, type_field.str, type_field.len) == 0
+		} {
+			// find type field
+			type_field_node = type_field_node.next
+
+			break
+		} else {
+			type_field_node = type_field_node.next
+		}
+	}
+
+	if type_field_node == unsafe { nil } {
+		return false
+	}
+
+	variant_name := typeof(s).name
+	if type_field_node.value.length - 2 == variant_name.len {
+		unsafe {
+		}
+		if unsafe {
+			vmemcmp(decoder.json.str + type_field_node.value.position + 1, variant_name.str,
+				variant_name.len) == 0
+		} {
+			return true
+		}
+	}
+
+	return false
+}
+
+fn (mut decoder Decoder) get_struct_type_workaround[T](initialized_sumtype T) bool {
+	$if initialized_sumtype is $sumtype {
+		$for v in initialized_sumtype.variants {
+			if initialized_sumtype is v {
+				$if initialized_sumtype is $struct {
+					val := copy_type(initialized_sumtype)
+					return decoder.check_struct_type_valid(val, decoder.current_node)
+				}
+			}
+		}
+	}
+	return false
 }
 
 fn (mut decoder Decoder) init_sumtype_by_value_kind[T](mut val T, value_info ValueInfo) ! {
-	$for v in val.variants {
+	mut failed_struct := false
-		if value_info.value_kind == .string_ {
-			$if v.typ is string {
-				val = T(v)
-				return
-			} $else $if v.typ is time.Time {
-				val = T(v)
-				return
-			}
-		} else if value_info.value_kind == .number {
-			$if v.typ is $float {
-				val = T(v)
-				return
-			} $else $if v.typ is $int {
-				val = T(v)
-				return
-			} $else $if v.typ is $enum {
-				val = T(v)
-				return
-			}
-		} else if value_info.value_kind == .boolean {
-			$if v.typ is bool {
-				val = T(v)
-				return
-			}
-		} else if value_info.value_kind == .object {
-			$if v.typ is $map {
-				val = T(v)
-				return
-			} $else $if v.typ is $struct {
-				// find "_type" field in json object
-				mut type_field_node := decoder.current_node.next
-				map_position := value_info.position
-				map_end := map_position + value_info.length
 
-				type_field := '"_type"'
+	match value_info.value_kind {
-
+		.string_ {
-				for {
+			$for v in val.variants {
-					if type_field_node == unsafe { nil } {
+				$if v.typ is string {
-						break
+					val = T(v)
-					}
+					return
-
+				} $else $if v.typ is time.Time {
-					key_info := type_field_node.value
+					val = T(v)
-
+					return
-					if key_info.position >= map_end {
-						type_field_node = unsafe { nil }
-						break
-					}
-
-					if unsafe {
-						vmemcmp(decoder.json.str + key_info.position, type_field.str,
-							type_field.len) == 0
-					} {
-						// find type field
-						type_field_node = type_field_node.next
-
-						break
-					} else {
-						type_field_node = type_field_node.next
-					}
 				}
-
-				if type_field_node != unsafe { nil } {
-					$for v in val.variants {
-						variant_name := typeof(v.typ).name
-						if type_field_node.value.length - 2 == variant_name.len {
-							unsafe {
-							}
-							if unsafe {
-								vmemcmp(decoder.json.str + type_field_node.value.position + 1,
-									variant_name.str, variant_name.len) == 0
-							} {
-								val = T(v)
-							}
-						}
-					}
-				}
-
-				return
-			}
-		} else if value_info.value_kind == .array {
-			$if v.typ is $array {
-				val = T(v)
-				return
 			}
 		}
+		.number {
+			$for v in val.variants {
+				$if v.typ is $float {
+					val = T(v)
+					return
+				} $else $if v.typ is $int {
+					val = T(v)
+					return
+				} $else $if v.typ is $enum {
+					val = T(v)
+					return
+				}
+			}
+		}
+		.boolean {
+			$for v in val.variants {
+				$if v.typ is bool {
+					val = T(v)
+					return
+				}
+			}
+		}
+		.null {
+			$for v in val.variants {
+				$if v.typ is $option {
+					val = T(v)
+					return
+				}
+			}
+		}
+		.array {
+			$for v in val.variants {
+				$if v.typ is $array {
+					val = T(v)
+
+					if decoder.get_array_type_workaround(val) {
+						return
+					}
+				}
+			}
+		}
+		.object {
+			$for v in val.variants {
+				$if v.typ is $map {
+					val = T(v)
+
+					if decoder.get_map_type_workaround(val) {
+						return
+					}
+				} $else $if v.typ is $struct {
+					val = T(v)
+
+					if decoder.get_struct_type_workaround(val) {
+						return
+					}
+
+					failed_struct = true
+				}
+			}
+		}
+		else {}
 	}
+
+	if failed_struct {
+		decoder.error('could not resolve sumtype `${T.name}`, missing "_type" field?')!
+	}
+
+	decoder.error('could not resolve sumtype `${T.name}`, got ${value_info.value_kind}.')!
 }
 
 fn (mut decoder Decoder) decode_sumtype[T](mut val T) ! {
-	value_info := decoder.current_node.value
+	$if T is $alias {
+		decoder.error('Type aliased sumtypes not supported.')!
+	} $else {
+		value_info := decoder.current_node.value
 
-	decoder.init_sumtype_by_value_kind(mut val, value_info)!
+		decoder.init_sumtype_by_value_kind(mut val, value_info)!
 
-	decoded_sumtype := decoder.get_decoded_sumtype_workaround(val)!
+		val = decoder.get_decoded_sumtype_workaround(val)!
-	unsafe {
-		*val = decoded_sumtype
 	}
 }

vlib/x/json2/decoder2/tests/json_sumtype_test.v

@@ -29,6 +29,21 @@ type Sum = int | string | bool | []string
 
 type StructSumTypes = Stru | Stru2
 
+type Mixed = Cat | int | map[string]int
+
+type Maybes = ?int | ?string
+
+type MultiArray = []int
+	| []bool
+	| [][]string
+	| []map[string]map[string][]int
+	| map[string]json2.Any
+	| string
+
+type StructLists = Cat | []Cat | map[string]Dog
+
+type SumAlias = Sum
+
 pub struct Stru {
 	val  int
 	val2 string
@@ -42,6 +57,8 @@ pub struct Stru2 {
 	steak string
 }
 
+type NewAny = int | string | bool | []NewAny | map[string]NewAny | ?int
+
 fn test_simple_sum_type() {
 	assert json.decode[Sum]('1')! == Sum(1)
 
@@ -86,13 +103,62 @@ fn test_any_sum_type() {
 			'hello2': json2.Any('world')
 		})
 	})
+
+	assert json.decode[NewAny]('{"name": null, "value": "hi"}')! == NewAny({
+		'name':  NewAny(?int(none))
+		'value': NewAny('hi')
+	})
+
+	assert json.decode[json2.Any]('[]')! == json2.Any([]json2.Any{})
+	assert json.decode[json2.Any]('{}')! == json2.Any(map[string]json2.Any{})
 }
 
 fn test_sum_type_struct() {
-	assert json.decode[Animal]('{"cat_name": "Tom"}')! == Animal(Cat{'Tom'})
+	if x := json.decode[Animal]('{"cat_name": "Tom"}') {
-	assert json.decode[Animal]('{"dog_name": "Rex"}')! == Animal(Cat{''})
+		assert false
+	}
+	if x := json.decode[Animal]('{"dog_name": "Rex"}') {
+		assert false
+	}
 	assert json.decode[Animal]('{"dog_name": "Rex", "_type": "Dog"}')! == Animal(Dog{'Rex'})
 
 	// struct sumtype in random order
 	assert json.decode[StructSumTypes]('{"_type": "Stru", "val": 1, "val2": "lala", "val3": {"a": 2, "steak": "leleu"}, "val4": 2147483000, "val5": 2147483000}')! == StructSumTypes(Stru{1, 'lala', Stru2{2, 'leleu'}, 2147483000, 2147483000})
 }
+
+fn test_sum_type_mixed() {
+	assert json.decode[Mixed]('{"key":0}')! == Mixed({
+		'key': 0
+	})
+	assert json.decode[Mixed]('10')! == Mixed(10)
+}
+
+// to be implemented
+fn test_sum_type_options_fail() {
+	assert json.decode[Maybes]('null')! == Maybes(?int(none))
+	if x := json.decode[Maybes]('99') {
+		assert false
+	}
+	if x := json.decode[Maybes]('hi') {
+		assert false
+	}
+	if x := json.decode[Maybes]('true') {
+		assert false
+	}
+}
+
+// to be implemented
+fn test_sum_type_alias_fail() {
+	if x := json.decode[SumAlias]('99') {
+		assert false
+	}
+	if x := json.decode[SumAlias]('true') {
+		assert false
+	}
+	if x := json.decode[SumAlias]('["hi", "bye"]') {
+		assert false
+	}
+	if x := json.decode[SumAlias]('[0, 1]') {
+		assert false
+	}
+}