Jerome Delrieu created AVRO-3810:
------------------------------------
Summary: Incorrect JSON generated by avro::jsonEncoder from
avro::GenericDatum when schema contains as last value a record with 0 field
Key: AVRO-3810
URL: https://issues.apache.org/jira/browse/AVRO-3810
Project: Apache Avro
Issue Type: Bug
Components: c++
Environment: Windows, 64bit, compiled with msvc
Reporter: Jerome Delrieu
When the Schema has a record with 0 field as its last value, the
avro::jsonEncoder doesn't generate the last characters in the JSON string.
The obtained JSON string is malformed and cannot be parsed with a standard JSON
parser.
Note : the avro::{color:#74531f}jsonDecoder {color}accept this string whereas
it is not a valid JSON syntax
small C++ example:
{color:#0000FF}const{color} {color:#0000FF}char{color}
*{color:#1f377f}sSchema{color} =
{color:#e21f1f}"{color}{color:#a31515}{{color}{color:#b776fb}\"{color}{color:#a31515}type{color}{color:#b776fb}\"{color}{color:#a31515}:
{color}{color:#b776fb}\"{color}{color:#a31515}record{color}{color:#b776fb}\"{color}{color:#a31515},{color}{color:#b776fb}\"{color}{color:#a31515}name{color}{color:#b776fb}\"{color}{color:#a31515}
:
{color}{color:#b776fb}\"{color}{color:#a31515}Event{color}{color:#b776fb}\"{color}{color:#a31515},{color}{color:#b776fb}\"{color}{color:#a31515}namespace{color}{color:#b776fb}\"{color}{color:#a31515}
:
{color}{color:#b776fb}\"{color}{color:#a31515}test{color}{color:#b776fb}\"{color}{color:#a31515},{color}{color:#b776fb}\"{color}{color:#a31515}doc{color}{color:#b776fb}\"{color}{color:#a31515}
: {color}{color:#b776fb}\"\"{color}{color:#a31515},
{color}{color:#b776fb}\"{color}{color:#a31515}fields{color}{color:#b776fb}\"{color}{color:#a31515}
: {color}{color:#e21f1f}"{color} {color:#e21f1f}"{color}{color:#a31515}[ {
{color}{color:#b776fb}\"{color}{color:#a31515}name{color}{color:#b776fb}\"{color}{color:#a31515}:
{color}{color:#b776fb}\"{color}{color:#a31515}Event{color}{color:#b776fb}\"{color}{color:#a31515},
{color}{color:#b776fb}\"{color}{color:#a31515}type{color}{color:#b776fb}\"{color}{color:#a31515}
: [ {
{color}{color:#b776fb}\"{color}{color:#a31515}type{color}{color:#b776fb}\"{color}{color:#a31515}:
{color}{color:#b776fb}\"{color}{color:#a31515}record{color}{color:#b776fb}\"{color}{color:#a31515},
{color}{color:#b776fb}\"{color}{color:#a31515}name{color}{color:#b776fb}\"{color}{color:#a31515}
:
{color}{color:#b776fb}\"{color}{color:#a31515}ABEvent{color}{color:#b776fb}\"{color}{color:#a31515},
{color}{color:#b776fb}\"{color}{color:#a31515}doc{color}{color:#b776fb}\"{color}{color:#a31515}
:
{color}{color:#b776fb}\"{color}{color:#a31515}unused{color}{color:#b776fb}\"{color}{color:#a31515},{color}{color:#e21f1f}"{color}
{color:#e21f1f}"{color}{color:#b776fb}\"{color}{color:#a31515}fields{color}{color:#b776fb}\"{color}{color:#a31515}
: [ ] } ] } ] }{color}{color:#e21f1f}"{color};
avro::{color:#2b91af}ValidSchema{color}
{color:#1f377f}schema{color}(avro::{color:#74531f}compileJsonSchemaFromString{color}({color:#1f377f}sSchema{color}));
avro::{color:#2b91af}GenericDatum{color}
{color:#1f377f}datumDecode{color}({color:#1f377f}schema{color});
avro::{color:#2b91af}DecoderPtr{color} {color:#1f377f}decoder{color} =
avro::{color:#74531f}binaryDecoder{color}();
{color:#008000}// only one byte, 0, as source for decoding{color}
{color:#2b91af}uint8_t{color} {color:#1f377f}binarySource{color} = 0;
std::{color:#2b91af}unique_ptr{color}<avro::{color:#2b91af}InputStream{color}>
{color:#1f377f}in{color} =
avro::{color:#74531f}memoryInputStream{color}(({color:#2b91af}uint8_t{color}
*)&{color:#1f377f}binarySource{color}, 1);
{color:#1f377f}decoder{color}{color:#008080}->{color}{color:#74531f}init{color}({color:#008080}*{color}{color:#1f377f}in{color});
avro::{color:#74531f}decode{color}({color:#008080}*{color}{color:#1f377f}decoder{color},
{color:#1f377f}datumDecode{color});
std::{color:#2b91af}unique_ptr{color}<avro::{color:#2b91af}OutputStream{color}>
{color:#1f377f}out{color} = avro::{color:#74531f}memoryOutputStream{color}();
std::{color:#2b91af}shared_ptr{color}<avro::{color:#2b91af}Encoder{color}>
{color:#1f377f}encoder{color} =
avro::{color:#74531f}jsonEncoder{color}({color:#1f377f}schema{color});
{color:#1f377f}encoder{color}{color:#008080}->{color}{color:#74531f}init{color}({color:#008080}*{color}{color:#1f377f}out{color});
avro::{color:#74531f}encode{color}({color:#008080}*{color}{color:#1f377f}encoder{color},
{color:#1f377f}datumDecode{color});
{color:#1f377f}encoder{color}{color:#008080}->{color}{color:#74531f}flush{color}();
{color:#0000FF}const{color}
std::{color:#2b91af}shared_ptr{color}<std::{color:#2b91af}vector{color}<{color:#2b91af}uint8_t{color}>
> &{color:#1f377f}buffer{color} =
avro::{color:#74531f}snapshot{color}(*{color:#1f377f}out{color}.{color:#74531f}get{color}());
std::{color:#2b91af}string{color}
{color:#1f377f}result{color}({color:#0000FF}reinterpret_cast{color}<{color:#0000FF}char{color}
*>({color:#1f377f}buffer{color}.{color:#74531f}get{color}()->{color:#74531f}data{color}()),
{color:#1f377f}buffer{color}.{color:#74531f}get{color}()->{color:#74531f}size{color}());
Example Schema:
Schema:
{
"type": "record",
"name": "Event",
"namespace": "test",
"doc": "",
"fields": [
{
"name": "Event",
"type": [
{
"type": "record",
"name": "ABEvent",
"doc": "unused",
"fields": [
]
}
]
}
]
}
Binary source : 1 byte 00
Result:
{"Event":{"test.ABEvent":
A small patch to lang\c++\impl\parsing\Symbol.hh can solve this issue
Before:
void processImplicitActions() {
for (;;) {
Symbol &s = parsingStack.top();
if (s.isImplicitAction()) {
handler_.handle(s);
parsingStack.pop();
} else if (s.kind() == Symbol::Kind::SkipStart) {
parsingStack.pop();
skip(*decoder_);
} else {
break;
}
}
}
After:
void processImplicitActions() {
for (;;) {
Symbol &s = parsingStack.top();
if (s.isImplicitAction()) {
handler_.handle(s);
parsingStack.pop();
} else if (s.kind() == Symbol::Kind::SkipStart) {
parsingStack.pop();
skip(*decoder_);
} else if (s.kind() == Symbol::Kind::Indirect) {
parsingStack.pop();
parsingStack.push(Symbol::recordEndSymbol());
parsingStack.push(Symbol::recordStartSymbol());
} else {
break;
}
}
}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
