TheCloud/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2025-09-12 08:57:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

decoder2: fix number decoding and improve errors (#25015)

Browse Source
This commit is contained in:
Larsimusrex 2025-08-01 05:56:39 +02:00 committed by GitHub
parent c49b9da04e
commit f03d800800
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 552 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vlib/x/json2/decoder2/attributes_test.v
View File

@ -90,8 +90,12 @@ fn test_required_attribute() {
mut has_error := false mut has_error := false
json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "b": 3}') or { json.decode[StruWithRequiredAttribute]('{"name": "hola", "a": 2, "b": 3}') or {
if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 31
assert err.message == 'Data: missing required field `skip_and_required`'
}
has_error = true has_error = true
assert err.msg() == 'missing required field `skip_and_required`'
} }
assert has_error, '`required` attribute not working. It should have failed' assert has_error, '`required` attribute not working. It should have failed'

611
vlib/x/json2/decoder2/decode.v
View File

@ -149,34 +149,152 @@ pub enum ValueKind {
null null
} }
// error generates an error message with context from the JSON string. const max_context_lenght = 50
fn (mut checker Decoder) error(message string) ! { const max_extra_charaters = 5
json := if checker.json.len < checker.checker_idx + 5 { const tab_width = 8
checker.json
} else { pub struct JsonDecodeError {
checker.json[0..checker.checker_idx + 5] Error
context string
pub:
message string
line int
character int
}
fn (e JsonDecodeError) msg() string {
return '\n${e.line}:${e.character}: Invalid json: ${e.message}\n${e.context}'
}
// checker_error generates a checker error message showing the position in the json string
fn (mut checker Decoder) checker_error(message string) ! {
position := checker.checker_idx
mut line_number := 0
mut character_number := 0
mut last_newline := 0
for i := position - 1; i >= 0; i-- {
if last_newline == 0 {
if checker.json[i] == `\n` {
last_newline = i + 1
} else if checker.json[i] == `\t` {
character_number += tab_width
} else {
character_number++
}
}
if checker.json[i] == `\n` {
line_number++
}
} }
mut error_message := '\n' cutoff := character_number > max_context_lenght
last_new_line := json.last_index_u8(`\n`)
if last_new_line != -1 {
error_message += json[last_new_line..checker.checker_idx]
} else {
error_message += json[0..checker.checker_idx]
}
error_message += [json[checker.checker_idx]].bytestr()
error_message += '\n' // either start of string, last newline or a limited amount of characters
context_start := if cutoff { position - max_context_lenght } else { last_newline }
if last_new_line != -1 { // print some extra characters
error_message += ' '.repeat(checker.checker_idx - last_new_line) mut context_end := int_min(checker.json.len, position + max_extra_charaters)
} else { context_end_newline := checker.json[position..context_end].index_u8(`\n`)
error_message += ' '.repeat(checker.checker_idx)
if context_end_newline != -1 {
context_end = position + context_end_newline
} }
error_message += '^ ${message}' mut context := ''
return error(error_message) if cutoff {
context += '...'
}
context += checker.json[context_start..position]
context += '\e[31m${checker.json[position].ascii_str()}\e[0m'
context += checker.json[position + 1..context_end]
context += '\n'
if cutoff {
context += ' '.repeat(max_context_lenght + 3)
} else {
context += ' '.repeat(character_number)
}
context += '\e[31m^\e[0m'
return JsonDecodeError{
context: context
message: 'Syntax: ${message}'
line: line_number + 1
character: character_number + 1
}
}
// decode_error generates a decoding error from the decoding stage
fn (mut decoder Decoder) decode_error(message string) ! {
mut error_info := ValueInfo{}
if decoder.current_node != unsafe { nil } {
error_info = decoder.current_node.value
} else {
error_info = decoder.values_info.tail.value
}
start := error_info.position
end := start + int_min(error_info.length, max_context_lenght)
mut line_number := 0
mut character_number := 0
mut last_newline := 0
for i := start - 1; i >= 0; i-- {
if last_newline == 0 {
if decoder.json[i] == `\n` {
last_newline = i + 1
} else if decoder.json[i] == `\t` {
character_number += tab_width
} else {
character_number++
}
}
if decoder.json[i] == `\n` {
line_number++
}
}
cutoff := character_number > max_context_lenght
// either start of string, last newline or a limited amount of characters
context_start := if cutoff { start - max_context_lenght } else { last_newline }
// print some extra characters
mut context_end := int_min(decoder.json.len, end + max_extra_charaters)
context_end_newline := decoder.json[end..context_end].index_u8(`\n`)
if context_end_newline != -1 {
context_end = end + context_end_newline
}
mut context := ''
if cutoff {
context += '...'
}
context += decoder.json[context_start..start]
context += '\e[31m${decoder.json[start..end]}\e[0m'
context += decoder.json[end..context_end]
context += '\n'
if cutoff {
context += ' '.repeat(max_context_lenght + 3)
} else {
context += ' '.repeat(character_number)
}
context += '\e[31m${'~'.repeat(error_info.length)}\e[0m'
return JsonDecodeError{
context: context
message: 'Data: ${message}'
line: line_number + 1
character: character_number + 1
}
} }
// check_json_format checks if the JSON string is valid and updates the decoder state. // check_json_format checks if the JSON string is valid and updates the decoder state.
@ -184,7 +302,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
checker_end := checker.json.len checker_end := checker.json.len
// check if the JSON string is empty // check if the JSON string is empty
if val == '' { if val == '' {
return checker.error('empty string') return checker.checker_error('empty string')
} }
// skip whitespace // skip whitespace
@ -206,12 +324,12 @@ fn (mut checker Decoder) check_json_format(val string) ! {
mut actual_value_info_pointer := checker.values_info.last() mut actual_value_info_pointer := checker.values_info.last()
match value_kind { match value_kind {
.unknown { .unknown {
return checker.error('unknown value kind') return checker.checker_error('unknown value kind')
} }
.null { .null {
// check if the JSON string is a null value // check if the JSON string is a null value
if checker_end - checker.checker_idx <= 3 { if checker_end - checker.checker_idx <= 3 {
return checker.error('EOF error: expecting `null`') return checker.checker_error('EOF error: expecting `null`')
} }
is_not_ok := unsafe { is_not_ok := unsafe {
@ -219,22 +337,17 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
if is_not_ok != 0 { if is_not_ok != 0 {
return checker.error('invalid null value. Got `${checker.json[checker.checker_idx.. return checker.checker_error('invalid null value. Got `${checker.json[checker.checker_idx..
checker.checker_idx + 4]}` instead of `null`') checker.checker_idx + 4]}` instead of `null`')
} }
checker.checker_idx += 3 checker.checker_idx += 3
} }
.object { .object {
if checker_end - checker.checker_idx < 2 { if checker_end - checker.checker_idx < 2 {
return checker.error('EOF error: expecting a complete object after `{`') return checker.checker_error('EOF error: expecting a complete object after `{`')
} }
checker.checker_idx++ checker.checker_idx++
for val[checker.checker_idx] != `}` { for val[checker.checker_idx] != `}` {
// check if the JSON string is an empty object
if checker_end - checker.checker_idx <= 2 {
continue
}
// skip whitespace // skip whitespace
for val[checker.checker_idx] in whitespace_chars { for val[checker.checker_idx] in whitespace_chars {
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
@ -248,7 +361,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
if val[checker.checker_idx] != `"` { if val[checker.checker_idx] != `"` {
return checker.error('Expecting object key') return checker.checker_error('Expecting object key')
} }
// Object key // Object key
@ -256,16 +369,16 @@ fn (mut checker Decoder) check_json_format(val string) ! {
for val[checker.checker_idx] != `:` { for val[checker.checker_idx] != `:` {
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
return checker.error('EOF error: key colon not found') return checker.checker_error('EOF error: key colon not found')
} }
if val[checker.checker_idx] !in whitespace_chars { if val[checker.checker_idx] !in whitespace_chars {
return checker.error('invalid value after object key') return checker.checker_error('invalid value after object key')
} }
checker.checker_idx++ checker.checker_idx++
} }
if val[checker.checker_idx] != `:` { if val[checker.checker_idx] != `:` {
return checker.error('Expecting `:` after object key') return checker.checker_error('Expecting `:` after object key')
} }
// skip `:` // skip `:`
checker.checker_idx++ checker.checker_idx++
@ -286,7 +399,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
break break
} }
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
return checker.error('EOF error: braces are not closed') return checker.checker_error('EOF error: braces are not closed')
} }
if val[checker.checker_idx] == `,` { if val[checker.checker_idx] == `,` {
@ -295,18 +408,18 @@ fn (mut checker Decoder) check_json_format(val string) ! {
checker.checker_idx++ checker.checker_idx++
} }
if val[checker.checker_idx] != `"` { if val[checker.checker_idx] != `"` {
return checker.error('Expecting object key after `,`') return checker.checker_error('Expecting object key after `,`')
} }
} else { } else {
if val[checker.checker_idx] == `}` { if val[checker.checker_idx] == `}` {
break break
} else { } else {
return checker.error('invalid object value') return checker.checker_error('invalid object value')
} }
} }
} }
else { else {
return checker.error('invalid object value') return checker.checker_error('invalid object value')
} }
} }
} }
@ -316,7 +429,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
if checker_end >= checker.checker_idx + 2 { if checker_end >= checker.checker_idx + 2 {
checker.checker_idx++ checker.checker_idx++
} else { } else {
return checker.error('EOF error: There are not enough length for an array') return checker.checker_error('EOF error: There are not enough length for an array')
} }
for val[checker.checker_idx] != `]` { for val[checker.checker_idx] != `]` {
@ -333,7 +446,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
return checker.error('EOF error: array not closed') return checker.checker_error('EOF error: array not closed')
} }
checker.check_json_format(val)! checker.check_json_format(val)!
@ -346,7 +459,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
break break
} }
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
return checker.error('EOF error: braces are not closed') return checker.checker_error('EOF error: braces are not closed')
} }
if val[checker.checker_idx] == `,` { if val[checker.checker_idx] == `,` {
@ -355,14 +468,14 @@ fn (mut checker Decoder) check_json_format(val string) ! {
checker.checker_idx++ checker.checker_idx++
} }
if val[checker.checker_idx] == `]` { if val[checker.checker_idx] == `]` {
return checker.error('Cannot use `,`, before `]`') return checker.checker_error('Cannot use `,`, before `]`')
} }
continue continue
} else { } else {
if val[checker.checker_idx] == `]` { if val[checker.checker_idx] == `]` {
break break
} else { } else {
return checker.error('`]` after value') return checker.checker_error('`]` after value')
} }
} }
} }
@ -371,7 +484,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
// check if the JSON string is a valid string // check if the JSON string is a valid string
if checker.checker_idx >= checker_end - 1 { if checker.checker_idx >= checker_end - 1 {
return checker.error('EOF error: string not closed') return checker.checker_error('EOF error: string not closed')
} }
checker.checker_idx++ checker.checker_idx++
@ -380,7 +493,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
for val[checker.checker_idx] != `"` { for val[checker.checker_idx] != `"` {
if val[checker.checker_idx] == `\\` { if val[checker.checker_idx] == `\\` {
if checker.checker_idx + 1 >= checker_end - 1 { if checker.checker_idx + 1 >= checker_end - 1 {
return checker.error('invalid escape sequence') return checker.checker_error('invalid escape sequence')
} }
escaped_char := val[checker.checker_idx + 1] escaped_char := val[checker.checker_idx + 1]
match escaped_char { match escaped_char {
@ -401,18 +514,17 @@ fn (mut checker Decoder) check_json_format(val string) ! {
checker.checker_idx++ checker.checker_idx++
} }
else { else {
return checker.error('invalid unicode escape sequence') return checker.checker_error('invalid unicode escape sequence')
} }
} }
} }
continue continue
} else { } else {
return checker.error('short unicode escape sequence ${checker.json[checker.checker_idx.. return checker.checker_error('short unicode escape sequence ${checker.json[checker.checker_idx..escaped_char_last_index]}')
escaped_char_last_index + 1]}')
} }
} }
else { else {
return checker.error('unknown escape sequence') return checker.checker_error('unknown escape sequence')
} }
} }
} }
@ -421,50 +533,86 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
.number { .number {
// check if the JSON string is a valid float or integer // check if the JSON string is a valid float or integer
mut is_negative := val[0] == `-`
mut has_dot := false
mut digits_count := 1 if val[0] == `-` {
if is_negative {
checker.checker_idx++ checker.checker_idx++
} }
for checker.checker_idx < checker_end - 1 if checker.checker_idx == checker_end {
&& val[checker.checker_idx + 1] !in [`,`, `}`, `]`, ` `, `\t`, `\n`] checker.checker_idx--
&& checker.checker_idx < checker_end - 1 { return checker.checker_error('expected digit got EOF')
if val[checker.checker_idx] == `.` { }
if has_dot {
return checker.error('invalid float. Multiple dots') // integer part
} if val[checker.checker_idx] == `0` {
has_dot = true checker.checker_idx++
} else if val[checker.checker_idx] >= `1` && val[checker.checker_idx] <= `9` {
checker.checker_idx++
for checker.checker_idx < checker_end && val[checker.checker_idx] >= `0`
&& val[checker.checker_idx] <= `9` {
checker.checker_idx++ checker.checker_idx++
continue }
} else if val[checker.checker_idx] == `-` { } else {
if is_negative { return checker.checker_error('expected digit got ${val[checker.checker_idx].ascii_str()}')
return checker.error('invalid float. Multiple negative signs') }
// fraction part
if checker.checker_idx != checker_end && val[checker.checker_idx] == `.` {
checker.checker_idx++
if checker.checker_idx == checker_end {
checker.checker_idx--
return checker.checker_error('expected digit got EOF')
}
if val[checker.checker_idx] >= `0` && val[checker.checker_idx] <= `9` {
for checker.checker_idx < checker_end && val[checker.checker_idx] >= `0`
&& val[checker.checker_idx] <= `9` {
checker.checker_idx++
} }
checker.checker_idx++
continue
} else { } else {
if val[checker.checker_idx] < `0` || val[checker.checker_idx] > `9` { return checker.checker_error('expected digit got ${val[checker.checker_idx].ascii_str()}')
return checker.error('invalid number') }
}
// exponent part
if checker.checker_idx != checker_end
&& (val[checker.checker_idx] == `e` || val[checker.checker_idx] == `E`) {
checker.checker_idx++
if checker.checker_idx == checker_end {
checker.checker_idx--
return checker.checker_error('expected digit got EOF')
}
if val[checker.checker_idx] == `-` || val[checker.checker_idx] == `+` {
checker.checker_idx++
if checker.checker_idx == checker_end {
checker.checker_idx--
return checker.checker_error('expected digit got EOF')
} }
} }
if digits_count >= 64 { if val[checker.checker_idx] >= `0` && val[checker.checker_idx] <= `9` {
return checker.error('number exceeds 64 digits') for checker.checker_idx < checker_end && val[checker.checker_idx] >= `0`
&& val[checker.checker_idx] <= `9` {
checker.checker_idx++
}
} else {
return checker.checker_error('expected digit got ${val[checker.checker_idx].ascii_str()}')
} }
digits_count++
checker.checker_idx++
} }
checker.checker_idx--
} }
.boolean { .boolean {
// check if the JSON string is a valid boolean // check if the JSON string is a valid boolean
match val[checker.checker_idx] { match val[checker.checker_idx] {
`t` { `t` {
if checker_end - checker.checker_idx <= 3 { if checker_end - checker.checker_idx <= 3 {
return checker.error('EOF error: expecting `true`') return checker.checker_error('EOF error: expecting `true`')
} }
is_not_ok := unsafe { is_not_ok := unsafe {
@ -473,14 +621,14 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
if is_not_ok != 0 { if is_not_ok != 0 {
return checker.error('invalid boolean value. Got `${checker.json[checker.checker_idx.. return checker.checker_error('invalid boolean value. Got `${checker.json[checker.checker_idx..
checker.checker_idx + 4]}` instead of `true`') checker.checker_idx + 4]}` instead of `true`')
} }
checker.checker_idx += 3 checker.checker_idx += 3
} }
`f` { `f` {
if checker_end - checker.checker_idx <= 4 { if checker_end - checker.checker_idx <= 4 {
return checker.error('EOF error: expecting `false`') return checker.checker_error('EOF error: expecting `false`')
} }
is_not_ok := unsafe { is_not_ok := unsafe {
@ -489,14 +637,14 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
if is_not_ok != 0 { if is_not_ok != 0 {
return checker.error('invalid boolean value. Got `${checker.json[checker.checker_idx.. return checker.checker_error('invalid boolean value. Got `${checker.json[checker.checker_idx..
checker.checker_idx + 5]}` instead of `false`') checker.checker_idx + 5]}` instead of `false`')
} }
checker.checker_idx += 4 checker.checker_idx += 4
} }
else { else {
return checker.error('invalid boolean') return checker.checker_error('invalid boolean')
} }
} }
} }
@ -511,7 +659,7 @@ fn (mut checker Decoder) check_json_format(val string) ! {
for checker.checker_idx < checker_end - 1 && val[checker.checker_idx] !in [`,`, `:`, `}`, `]`] { for checker.checker_idx < checker_end - 1 && val[checker.checker_idx] !in [`,`, `:`, `}`, `]`] {
// get trash characters after the value // get trash characters after the value
if val[checker.checker_idx] !in whitespace_chars { if val[checker.checker_idx] !in whitespace_chars {
checker.error('invalid value. Unexpected character after ${value_kind} end')! checker.checker_error('invalid value. Unexpected character after ${value_kind} end')!
} else { } else {
// whitespace // whitespace
} }
@ -519,11 +667,33 @@ fn (mut checker Decoder) check_json_format(val string) ! {
} }
} }
// get_value_kind returns the kind of a JSON value.
fn get_value_kind(value u8) ValueKind {
if value == u8(`"`) {
return .string_
} else if value == u8(`t`) || value == u8(`f`) {
return .boolean
} else if value == u8(`{`) {
return .object
} else if value == u8(`[`) {
return .array
} else if (value >= u8(48) && value <= u8(57)) || value == u8(`-`) {
return .number
} else if value == u8(`n`) {
return .null
}
return .unknown
}
// decode decodes a JSON string into a specified type. // decode decodes a JSON string into a specified type.
@[manualfree] @[manualfree]
pub fn decode[T](val string) !T { pub fn decode[T](val string) !T {
if val == '' { if val == '' {
return error('empty string') return JsonDecodeError{
message: 'empty string'
line: 1
character: 1
}
} }
mut decoder := Decoder{ mut decoder := Decoder{
json: val json: val
@ -611,7 +781,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
val = string_buffer.bytestr() val = string_buffer.bytestr()
} else { } else {
return error('Expected string, but got ${string_info.value_kind}') return decoder.decode_error('Expected string, but got ${string_info.value_kind}')
} }
} $else $if T.unaliased_typ is $sumtype { } $else $if T.unaliased_typ is $sumtype {
decoder.decode_sumtype(mut val)! decoder.decode_sumtype(mut val)!
@ -651,7 +821,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
for attr in field.attrs { for attr in field.attrs {
if attr.starts_with('json:') { if attr.starts_with('json:') {
if attr.len <= 6 { if attr.len <= 6 {
return error('`json` attribute must have an argument') return decoder.decode_error('`json` attribute must have an argument')
} }
json_name_str = unsafe { attr.str + 6 } json_name_str = unsafe { attr.str + 6 }
json_name_len = attr.len - 6 json_name_len = attr.len - 6
@ -756,7 +926,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
if current_field_info.value.is_skip { if current_field_info.value.is_skip {
if current_field_info.value.is_required == false { if current_field_info.value.is_required == false {
return error('This should not happen. Please, file a bug. `skip` field should not be processed here without a `required` attribute') return decoder.decode_error('This should not happen. Please, file a bug. `skip` field should not be processed here without a `required` attribute')
} }
current_field_info.value.decoded_with_value_info_node = decoder.current_node current_field_info.value.decoded_with_value_info_node = decoder.current_node
break break
@ -765,7 +935,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
if current_field_info.value.is_raw { if current_field_info.value.is_raw {
$if field.unaliased_typ is $enum { $if field.unaliased_typ is $enum {
// workaround to avoid the error: enums can only be assigned `int` values // workaround to avoid the error: enums can only be assigned `int` values
return error('`raw` attribute cannot be used with enum fields') return decoder.decode_error('`raw` attribute cannot be used with enum fields')
} $else $if field.typ is ?string { } $else $if field.typ is ?string {
position := decoder.current_node.value.position position := decoder.current_node.value.position
end := position + decoder.current_node.value.length end := position + decoder.current_node.value.length
@ -797,7 +967,7 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
decoder.current_node = decoder.current_node.next decoder.current_node = decoder.current_node.next
} }
} $else { } $else {
return error('`raw` attribute can only be used with string fields') return decoder.decode_error('`raw` attribute can only be used with string fields')
} }
} else { } else {
$if field.typ is $option { $if field.typ is $option {
@ -840,14 +1010,14 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
continue continue
} }
if current_field_info.value.decoded_with_value_info_node == unsafe { nil } { if current_field_info.value.decoded_with_value_info_node == unsafe { nil } {
return error('missing required field `${unsafe { return decoder.decode_error('missing required field `${unsafe {
tos(current_field_info.value.field_name_str, current_field_info.value.field_name_len) tos(current_field_info.value.field_name_str, current_field_info.value.field_name_len)
}}`') }}`')
} }
current_field_info = current_field_info.next current_field_info = current_field_info.next
} }
} else { } else {
return error('Expected object, but got ${struct_info.value_kind}') return decoder.decode_error('Expected object, but got ${struct_info.value_kind}')
} }
unsafe { unsafe {
struct_fields_info.free() struct_fields_info.free()
@ -857,27 +1027,23 @@ fn (mut decoder Decoder) decode_value[T](mut val T) ! {
value_info := decoder.current_node.value value_info := decoder.current_node.value
if value_info.value_kind != .boolean { if value_info.value_kind != .boolean {
return error('Expected boolean, but got ${value_info.value_kind}') return decoder.decode_error('Expected boolean, but got ${value_info.value_kind}')
} }
unsafe { unsafe {
val = vmemcmp(decoder.json.str + value_info.position, true_in_string.str, val = vmemcmp(decoder.json.str + value_info.position, true_in_string.str,
true_in_string.len) == 0 true_in_string.len) == 0
} }
} $else $if T.unaliased_typ in [$float, $int, $enum] { } $else $if T.unaliased_typ is $float || T.unaliased_typ is $int || T.unaliased_typ is $enum {
value_info := decoder.current_node.value value_info := decoder.current_node.value
if value_info.value_kind == .number { if value_info.value_kind == .number {
bytes := unsafe { (decoder.json.str + value_info.position).vbytes(value_info.length) } unsafe { decoder.decode_number(&val)! }
unsafe {
string_buffer_to_generic_number(val, bytes)
}
} else { } else {
return error('Expected number, but got ${value_info.value_kind}') return decoder.decode_error('Expected number, but got ${value_info.value_kind}')
} }
} $else { } $else {
return error('cannot decode value with ${typeof(val).name} type') return decoder.decode_error('cannot decode value with ${typeof(val).name} type')
} }
if decoder.current_node != unsafe { nil } { if decoder.current_node != unsafe { nil } {
@ -907,7 +1073,7 @@ fn (mut decoder Decoder) decode_array[T](mut val []T) ! {
val << array_element val << array_element
} }
} else { } else {
return error('Expected array, but got ${array_info.value_kind}') return decoder.decode_error('Expected array, but got ${array_info.value_kind}')
} }
} }
@ -951,101 +1117,176 @@ fn (mut decoder Decoder) decode_map[K, V](mut val map[K]V) ! {
decoder.decode_value(mut val[key])! decoder.decode_value(mut val[key])!
} }
} else { } else {
return error('Expected object, but got ${map_info.value_kind}') return decoder.decode_error('Expected object, but got ${map_info.value_kind}')
} }
} }
// get_value_kind returns the kind of a JSON value.
fn get_value_kind(value u8) ValueKind {
if value == u8(`"`) {
return .string_
} else if value == u8(`t`) || value == u8(`f`) {
return .boolean
} else if value == u8(`{`) {
return .object
} else if value == u8(`[`) {
return .array
} else if (value >= u8(48) && value <= u8(57)) || value == u8(`-`) {
return .number
} else if value == u8(`n`) {
return .null
}
return .unknown
}
fn create_value_from_optional[T](val ?T) ?T { fn create_value_from_optional[T](val ?T) ?T {
return T{} return T{}
} }
// string_buffer_to_generic_number converts a buffer of bytes (data) into a generic type T and fn get_number_max[T](num T) T {
// stores the result in the provided result pointer. $if num is i8 {
// The function supports conversion to the following types: return max_i8
// - Signed integers: i8, i16, i32, i64 } $else $if num is i16 {
// - Unsigned integers: u8, u16, u32, u64 return max_i16
// - Floating-point numbers: f32, f64 } $else $if num is i32 {
// return max_i32
// For signed integers, the function handles negative numbers by checking for a '-' character. } $else $if num is i64 {
// For floating-point numbers, the function handles decimal points and adjusts the result return max_i64
// accordingly. } $else $if num is u8 {
// return max_u8
// If the type T is not supported, the function will panic with an appropriate error message. } $else $if num is u16 {
// return max_u16
// Parameters: } $else $if num is u32 {
// - data []u8: The buffer of bytes to be converted. return max_u32
// - result &T: A pointer to the variable where the converted result will be stored. } $else $if num is u64 {
// return max_u64
// NOTE: This aims works with not new memory allocated data, to more efficient use `vbytes` before } $else $if num is int {
@[direct_array_access; unsafe] return max_int
pub fn string_buffer_to_generic_number[T](result &T, data []u8) { }
$if T.unaliased_typ is $int { return 0
mut is_negative := false }
for ch in data {
if ch == `-` {
is_negative = true
continue
}
digit := T(ch - `0`)
*result = T(*result * 10 + digit)
}
if is_negative {
*result *= -1
}
} $else $if T.unaliased_typ is $float {
mut is_negative := false
mut decimal_seen := false
mut decimal_divider := T(1)
for ch in data { fn get_number_min[T](num T) T {
if ch == `-` { $if num is i8 {
is_negative = true return min_i8
continue } $else $if num is i16 {
} return min_i16
if ch == `.` { } $else $if num is i32 {
decimal_seen = true return min_i32
continue } $else $if num is i64 {
} return min_i64
} $else $if num is u8 {
return min_u8
} $else $if num is u16 {
return min_u16
} $else $if num is u32 {
return min_u32
} $else $if num is u64 {
return min_u64
} $else $if num is int {
return min_int
}
return 0
}
digit := T(ch - u8(`0`)) fn get_number_digits[T](num T) int {
return $if T.unaliased_typ is i8 || T.unaliased_typ is u8 {
if decimal_seen { 3
decimal_divider *= 10 } $else $if T.unaliased_typ is i16 || T.unaliased_typ is u16 {
*result += T(digit / decimal_divider) 5
} else { } $else $if T.unaliased_typ is i32 || T.unaliased_typ is u32 || T.unaliased_typ is int {
*result = T(*result * 10 + digit) 10
} } $else $if T.unaliased_typ is i64 {
} 19
if is_negative { } $else $if T.unaliased_typ is u64 {
*result *= -1 20
}
} $else $if T.unaliased_typ is $enum {
// Convert the string to an integer
enumeration := 0
for ch in data {
digit := int(ch - `0`)
enumeration = enumeration * 10 + digit
}
*result = T(enumeration)
} $else { } $else {
panic('unsupported type ' + typeof[T]().name) 0
}
}
// use pointer instead of mut so enum cast works
@[unsafe]
fn (mut decoder Decoder) decode_number[T](val &T) ! {
number_info := decoder.current_node.value
$if T.unaliased_typ is $float {
*val = T(strconv.atof_quick(decoder.json[number_info.position..number_info.position +
number_info.length]))
} $else $if T.unaliased_typ is $enum {
mut result := 0
decoder.decode_number(&result)!
*val = T(result)
} $else { // this part is a minefield
mut is_negative := false
mut index := 0
if decoder.json[number_info.position] == `-` {
$if T.unaliased_typ is u8 || T.unaliased_typ is u16 || T.unaliased_typ is u32
|| T.unaliased_typ is u64 || T.unaliased_typ is $enum {
decoder.decode_error('expected positive integer for ${typeof(val).name} but got ${decoder.json[number_info.position..
number_info.position + number_info.length]}')!
}
is_negative = true
index++
}
// doing it like this means the minimum of signed numbers does not overflow before being inverted
if !is_negative {
digit_amount := get_number_digits(*val)
if number_info.length > digit_amount {
decoder.decode_error('overflows ${typeof(val).name}')!
}
for index < int_min(number_info.length, digit_amount - 1) {
digit := T(decoder.json[number_info.position + index] - `0`)
if digit > 9 { // comma, e and E are all smaller 0 in ASCII so they underflow
decoder.decode_error('expected integer but got real number')!
}
*val = *val * 10 + digit
index++
}
if index == digit_amount - 1 {
digit := T(decoder.json[number_info.position + index] - `0`)
if digit > 9 { // comma, e and E are all smaller 0 in ASCII so they underflow
decoder.decode_error('expected integer but got real number')!
}
type_max := get_number_max(*val)
max_digits := type_max / 10
last_digit := type_max % 10
if *val > max_digits || (*val == max_digits && digit > last_digit) {
decoder.decode_error('overflows ${typeof(val).name}s')!
}
*val = *val * 10 + digit
}
} else {
digit_amount := get_number_digits(*val) + 1
if number_info.length > digit_amount {
decoder.decode_error('underflows ${typeof(val).name}')!
}
for index < int_min(number_info.length, digit_amount - 1) {
digit := T(decoder.json[number_info.position + index] - `0`)
if digit > 9 { // comma, e and E are all smaller 0 in ASCII so they underflow
decoder.decode_error('expected integer but got real number')!
}
*val = *val * 10 - digit
index++
}
if index == digit_amount - 1 {
digit := T(decoder.json[number_info.position + index] - `0`)
if digit > 9 { // comma, e and E are all smaller 0 in ASCII so they underflow
decoder.decode_error('expected integer but got real number')!
}
type_min := get_number_min(*val)
min_digits := type_min / 10
last_digit := type_min % 10
if *val < min_digits || (*val == min_digits && -digit < last_digit) {
decoder.decode_error('underflows ${typeof(val).name}')!
}
*val = *val * 10 - digit
}
}
} }
} }

10
vlib/x/json2/decoder2/decode_sumtype.v
View File

@ -19,13 +19,13 @@ fn (mut decoder Decoder) get_decoded_sumtype_workaround[T](initialized_sumtype T
decoder.current_node = decoder.current_node.next decoder.current_node = decoder.current_node.next
return T(initialized_sumtype) return T(initialized_sumtype)
} else { } else {
decoder.error('sumtype option only support decoding null->none (for now)')! decoder.decode_error('sumtype option only support decoding null->none (for now)')!
} }
} }
} }
} }
} }
decoder.error('could not decode resolved sumtype (should not happen)')! decoder.decode_error('could not decode resolved sumtype (should not happen)')!
return initialized_sumtype // suppress compiler error return initialized_sumtype // suppress compiler error
} }
@ -288,15 +288,15 @@ fn (mut decoder Decoder) init_sumtype_by_value_kind[T](mut val T, value_info Val
} }
if failed_struct { if failed_struct {
decoder.error('could not resolve sumtype `${T.name}`, missing "_type" field?')! decoder.decode_error('could not resolve sumtype `${T.name}`, missing "_type" field?')!
} }
decoder.error('could not resolve sumtype `${T.name}`, got ${value_info.value_kind}.')! decoder.decode_error('could not resolve sumtype `${T.name}`, got ${value_info.value_kind}.')!
} }
fn (mut decoder Decoder) decode_sumtype[T](mut val T) ! { fn (mut decoder Decoder) decode_sumtype[T](mut val T) ! {
$if T is $alias { $if T is $alias {
decoder.error('Type aliased sumtypes not supported.')! decoder.decode_error('Type aliased sumtypes not supported.')!
} $else { } $else {
value_info := decoder.current_node.value value_info := decoder.current_node.value

62
vlib/x/json2/decoder2/decode_test.v
View File

@ -5,42 +5,66 @@ fn test_check_if_json_match() {
mut has_error := false mut has_error := false
decode[string]('{"key": "value"}') or { decode[string]('{"key": "value"}') or {
assert err.str() == 'Expected string, but got object' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected string, but got object'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
has_error = false has_error = false
decode[map[string]string]('"value"') or { decode[map[string]string]('"value"') or {
assert err.str() == 'Expected object, but got string_' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected object, but got string_'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
has_error = false has_error = false
decode[[]int]('{"key": "value"}') or { decode[[]int]('{"key": "value"}') or {
assert err.str() == 'Expected array, but got object' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected array, but got object'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
has_error = false has_error = false
decode[string]('[1, 2, 3]') or { decode[string]('[1, 2, 3]') or {
assert err.str() == 'Expected string, but got array' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected string, but got array'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
has_error = false has_error = false
decode[int]('{"key": "value"}') or { decode[int]('{"key": "value"}') or {
assert err.str() == 'Expected number, but got object' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected number, but got object'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
has_error = false has_error = false
decode[bool]('{"key": "value"}') or { decode[bool]('{"key": "value"}') or {
assert err.str() == 'Expected boolean, but got object' if err is JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'Data: Expected boolean, but got object'
}
has_error = true has_error = true
} }
assert has_error, 'Expected error' assert has_error, 'Expected error'
@ -125,43 +149,43 @@ fn test_check_json_format() {
json_and_error_message := [ json_and_error_message := [
{ {
'json': ']' 'json': ']'
'error': '\n]\n^ unknown value kind' 'error': 'Syntax: unknown value kind'
}, },
{ {
'json': '}' 'json': '}'
'error': '\n}\n^ unknown value kind' 'error': 'Syntax: unknown value kind'
}, },
{ {
'json': 'truely' 'json': 'truely'
'error': '\ntruel\n ^ invalid value. Unexpected character after boolean end' 'error': 'Syntax: invalid value. Unexpected character after boolean end'
}, },
{ {
'json': '0[1]' // 'json': '0[1]'
'error': '\n0[\n ^ invalid number' 'error': 'Syntax: invalid value. Unexpected character after number end'
}, },
{ {
'json': '[1, 2, g3]' 'json': '[1, 2, g3]'
'error': '\n[1, 2, g\n ^ unknown value kind' 'error': 'Syntax: unknown value kind'
}, },
{ {
'json': '[1, 2,, 3]' 'json': '[1, 2,, 3]'
'error': '\n[1, 2,,\n ^ unknown value kind' 'error': 'Syntax: unknown value kind'
}, },
{ {
'json': '{"key": 123' 'json': '{"key": 123'
'error': '\n{"key": 123\n ^ EOF error: braces are not closed' 'error': 'Syntax: EOF error: braces are not closed'
}, },
{ {
'json': '{"key": 123,' 'json': '{"key": 123,'
'error': '\n{"key": 123,\n ^ EOF error: braces are not closed' 'error': 'Syntax: EOF error: braces are not closed'
}, },
{ {
'json': '{"key": 123, "key2": 456,}' 'json': '{"key": 123, "key2": 456,}'
'error': '\n{"key": 123, "key2": 456,}\n ^ Expecting object key after `,`' 'error': 'Syntax: Expecting object key after `,`'
}, },
{ {
'json': '[[1, 2, 3], [4, 5, 6],]' 'json': '[[1, 2, 3], [4, 5, 6],]'
'error': '\n[[1, 2, 3], [4, 5, 6],]\n ^ Cannot use `,`, before `]`' 'error': 'Syntax: Cannot use `,`, before `]`'
}, },
] ]
@ -173,7 +197,9 @@ fn test_check_json_format() {
} }
checker.check_json_format(json_and_error['json']) or { checker.check_json_format(json_and_error['json']) or {
assert err.str() == json_and_error['error'] if err is JsonDecodeError {
assert err.message == json_and_error['error']
}
has_error = true has_error = true
} }
assert has_error, 'Expected error ${json_and_error['error']}' assert has_error, 'Expected error ${json_and_error['error']}'

38
vlib/x/json2/decoder2/tests/decode_number_and_boolean_test.v
View File

@ -6,17 +6,11 @@ fn test_number() {
assert json.decode[u8]('1')! == 1 assert json.decode[u8]('1')! == 1
assert json.decode[u8]('201')! == 201 assert json.decode[u8]('201')! == 201
assert json.decode[u8]('-1')! == u8(-1)
assert json.decode[u8]('-127')! == u8(-127)
// Test u16 // Test u16
assert json.decode[u16]('0')! == 0 assert json.decode[u16]('0')! == 0
assert json.decode[u16]('1')! == 1 assert json.decode[u16]('1')! == 1
assert json.decode[u16]('201')! == 201 assert json.decode[u16]('201')! == 201
assert json.decode[u16]('-1')! == u16(-1)
assert json.decode[u16]('-201')! == u16(-201)
// Test u32 // Test u32
assert json.decode[u32]('0')! == 0 assert json.decode[u32]('0')! == 0
assert json.decode[u32]('1')! == 1 assert json.decode[u32]('1')! == 1
@ -90,6 +84,38 @@ fn test_number() {
assert json.decode[f64]('1234567890')! == 1234567890.0 assert json.decode[f64]('1234567890')! == 1234567890.0
assert json.decode[f64]('-1234567890')! == -1234567890.0 assert json.decode[f64]('-1234567890')! == -1234567890.0
assert json.decode[f64]('1e10')! == 10000000000
assert json.decode[f64]('1E10')! == 10000000000
assert json.decode[f64]('1e+10')! == 10000000000
assert json.decode[f64]('1e-10')! == 0.0000000001
assert json.decode[f64]('-1e10')! == -10000000000
assert json.decode[f64]('-1E-10')! == -0.0000000001
assert json.decode[f64]('0.123e3')! - 123 < 0.0000001
assert json.decode[f64]('10.5E+3')! == 10500
// Test Over/Underflow
assert json.decode[i8]('127')! == 127
assert json.decode[i8]('-128')! == -128
if x := json.decode[i8]('128') {
assert false
}
if x := json.decode[i8]('130') {
assert false
}
if x := json.decode[i8]('1000') {
assert false
}
if x := json.decode[i8]('-129') {
assert false
}
if x := json.decode[i8]('-130') {
assert false
}
if x := json.decode[i8]('-1000') {
assert false
}
} }
fn test_boolean() { fn test_boolean() {

12
vlib/x/json2/decoder2/tests/decode_string_test.v
View File

@ -34,7 +34,11 @@ fn test_json_string_invalid_escapes() {
mut has_error := false mut has_error := false
json.decode[string](r'"\x"') or { json.decode[string](r'"\x"') or {
assert err.msg() == '\n"\\\n ^ unknown escape sequence' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 2
assert err.message == 'Syntax: unknown escape sequence'
}
has_error = true has_error = true
} // Invalid escape } // Invalid escape
@ -42,7 +46,11 @@ fn test_json_string_invalid_escapes() {
has_error = false has_error = false
json.decode[string](r'"\u123"') or { json.decode[string](r'"\u123"') or {
assert err.msg() == '\n"\\\n ^ short unicode escape sequence \\u123"' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 2
assert err.message == 'Syntax: short unicode escape sequence \\u123'
}
has_error = true has_error = true
} // Incomplete Unicode } // Incomplete Unicode

12
vlib/x/json2/decoder2/tests/json2_tests/decoder_test.v
View File

@ -57,7 +57,11 @@ fn test_nested_array_object() {
fn test_raw_decode_map_invalid() { fn test_raw_decode_map_invalid() {
json.decode[json2.Any]('{"name","Bob","age":20}') or { json.decode[json2.Any]('{"name","Bob","age":20}') or {
assert err.msg() == '\n{"name",\n ^ invalid value after object key' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 8
assert err.message == 'Syntax: invalid value after object key'
}
return return
} }
@ -66,7 +70,11 @@ fn test_raw_decode_map_invalid() {
fn test_raw_decode_array_invalid() { fn test_raw_decode_array_invalid() {
json.decode[json2.Any]('["Foo", 1,}') or { json.decode[json2.Any]('["Foo", 1,}') or {
assert err.msg() == '\n["Foo", 1,}\n ^ EOF error: array not closed' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 11
assert err.message == 'Syntax: EOF error: array not closed'
}
return return
} }

26
vlib/x/json2/decoder2/tests/json2_tests/json_module_compatibility_test/json_decode_todo_test.v
View File

@ -43,7 +43,11 @@ struct DbConfig {
fn test_decode_error_message_should_have_enough_context_empty() { fn test_decode_error_message_should_have_enough_context_empty() {
json.decode[DbConfig]('') or { json.decode[DbConfig]('') or {
assert err.msg() == 'empty string' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 1
assert err.message == 'empty string'
}
return return
} }
assert false assert false
@ -51,9 +55,11 @@ fn test_decode_error_message_should_have_enough_context_empty() {
fn test_decode_error_message_should_have_enough_context_just_brace() { fn test_decode_error_message_should_have_enough_context_just_brace() {
json.decode[DbConfig]('{') or { json.decode[DbConfig]('{') or {
assert err.msg() == ' if err is json.JsonDecodeError {
{ assert err.line == 1
^ EOF error: expecting a complete object after `{`' assert err.character == 1
assert err.message == 'Syntax: EOF error: expecting a complete object after `{`'
}
return return
} }
assert false assert false
@ -67,7 +73,11 @@ fn test_decode_error_message_should_have_enough_context_trailing_comma_at_end()
}' }'
json.decode[DbConfig](txt) or { json.decode[DbConfig](txt) or {
assert err.msg() == '\n\n}\n ^ Expecting object key after `,`' if err is json.JsonDecodeError {
assert err.line == 5
assert err.character == 1
assert err.message == 'Syntax: Expecting object key after `,`'
}
return return
} }
@ -77,7 +87,11 @@ fn test_decode_error_message_should_have_enough_context_trailing_comma_at_end()
fn test_decode_error_message_should_have_enough_context_in_the_middle() { fn test_decode_error_message_should_have_enough_context_in_the_middle() {
txt := '{"host": "localhost", "dbname": "alex" "user": "alex", "port": "1234"}' txt := '{"host": "localhost", "dbname": "alex" "user": "alex", "port": "1234"}'
json.decode[DbConfig](txt) or { json.decode[DbConfig](txt) or {
assert err.msg() == '\n{"host": "localhost", "dbname": "alex" "\n ^ invalid value. Unexpected character after string_ end' if err is json.JsonDecodeError {
assert err.line == 1
assert err.character == 40
assert err.message == 'Syntax: invalid value. Unexpected character after string_ end'
}
return return
} }
assert false assert false