This is an automated email from the ASF dual-hosted git repository.
technoboy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new baa288a0e5a [improve][broker] Upgrade avro version to 1.12.0 (#24617)
baa288a0e5a is described below
commit baa288a0e5acdb7dca94e5c4b066e4a41e06fc55
Author: Hang Chen <[email protected]>
AuthorDate: Wed Aug 13 22:38:54 2025 -0700
[improve][broker] Upgrade avro version to 1.12.0 (#24617)
Co-authored-by: Jiwe Guo <[email protected]>
---
distribution/server/src/assemble/LICENSE.bin.txt | 4 ++--
distribution/shell/src/assemble/LICENSE.bin.txt | 4 ++--
pom.xml | 2 +-
.../service/schema/JsonSchemaCompatibilityCheck.java | 3 +--
.../schema/validator/SchemaDataValidatorTest.java | 16 +++++++++++++++-
.../pulsar/client/impl/schema/util/SchemaUtil.java | 3 ++-
.../pulsar/client/impl/schema/AvroSchemaTest.java | 17 -----------------
.../apache/pulsar/io/elasticsearch/JsonConverter.java | 4 ++--
.../apache/pulsar/io/kinesis/json/JsonConverter.java | 4 ++--
9 files changed, 27 insertions(+), 30 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index c929378d0b7..c104deacf68 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -465,8 +465,8 @@ The Apache Software License, Version 2.0
* zt-zip
- org.zeroturnaround-zt-zip-1.17.jar
* Apache Avro
- - org.apache.avro-avro-1.11.4.jar
- - org.apache.avro-avro-protobuf-1.11.4.jar
+ - org.apache.avro-avro-1.12.0.jar
+ - org.apache.avro-avro-protobuf-1.12.0.jar
* Apache Curator
- org.apache.curator-curator-client-5.7.1.jar
- org.apache.curator-curator-framework-5.7.1.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt
b/distribution/shell/src/assemble/LICENSE.bin.txt
index 47c57e1df5d..1b735eee3cf 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -417,8 +417,8 @@ The Apache Software License, Version 2.0
* Google Error Prone Annotations - error_prone_annotations-2.38.0.jar
* Javassist -- javassist-3.25.0-GA.jar
* Apache Avro
- - avro-1.11.4.jar
- - avro-protobuf-1.11.4.jar
+ - avro-1.12.0.jar
+ - avro-protobuf-1.12.0.jar
* RE2j -- re2j-1.8.jar
* Spotify completable-futures -- completable-futures-0.3.6.jar
* RoaringBitmap -- RoaringBitmap-1.2.0.jar
diff --git a/pom.xml b/pom.xml
index a61ca8e67fd..2e7d4233124 100644
--- a/pom.xml
+++ b/pom.xml
@@ -225,7 +225,7 @@ flexible messaging model and an intuitive client
API.</description>
<kafka-client.version>3.9.1</kafka-client.version>
<rabbitmq-client.version>5.18.0</rabbitmq-client.version>
<aws-sdk.version>1.12.638</aws-sdk.version>
- <avro.version>1.11.4</avro.version>
+ <avro.version>1.12.0</avro.version>
<joda.version>2.10.10</joda.version>
<jclouds.version>2.6.0</jclouds.version>
<guice.version>5.1.0</guice.version>
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
index 476e5f272e4..cbdca3c4732 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import java.io.IOException;
import org.apache.avro.Schema;
-import org.apache.avro.SchemaParseException;
import
org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException;
import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
import org.apache.pulsar.common.protocol.schema.SchemaData;
@@ -96,7 +95,7 @@ public class JsonSchemaCompatibilityCheck extends
AvroSchemaBasedCompatibilityCh
fromParser.setValidateDefaults(false);
Schema fromSchema = fromParser.parse(new
String(schemaData.getData(), UTF_8));
return true;
- } catch (SchemaParseException e) {
+ } catch (Exception e) {
return false;
}
}
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
index 021fd4a9b40..302e5879d28 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
@@ -20,6 +20,8 @@ package org.apache.pulsar.broker.service.schema.validator;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
import
org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException;
import org.apache.pulsar.client.api.Schema;
@@ -131,7 +133,19 @@ public class SchemaDataValidatorTest {
new JsonSchemaGenerator(mapper)
.generateSchema(Foo.class)))
.build();
- SchemaDataValidator.validateSchemaData(data);
+ ObjectReader reader =
ObjectMapperFactory.getMapper().reader().forType(JsonSchema.class);
+ try {
+ org.apache.avro.Schema.Parser avroSchemaParser = new
org.apache.avro.Schema.Parser();
+ avroSchemaParser.setValidateDefaults(false);
+ org.apache.avro.Schema schema = avroSchemaParser.parse(new
String(data.getData(), UTF_8));
+ } catch (Exception e) {
+ try {
+ reader.readValue(data.getData());
+ } catch (Exception ioe) {
+ throw new InvalidSchemaDataException("Invalid schema
definition data for "
+ + data.getType() + " schema", ioe);
+ }
+ }
}
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
index 9c8d7fd511c..7af45734b89 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.client.impl.schema.util;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.lang.reflect.Field;
+import org.apache.avro.NameValidator;
import org.apache.avro.Schema;
import org.apache.avro.reflect.ReflectData;
import org.apache.commons.lang3.StringUtils;
@@ -41,7 +42,7 @@ public class SchemaUtil {
}
public static Schema parseAvroSchema(String schemaJson) {
- final Schema.Parser parser = new Schema.Parser();
+ final Schema.Parser parser = new
Schema.Parser(NameValidator.NO_VALIDATION);
parser.setValidateDefaults(false);
return parser.parse(schemaJson);
}
diff --git
a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java
b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java
index 3dedf3c978c..e0a6cdc54fc 100644
---
a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java
+++
b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java
@@ -506,15 +506,6 @@ public class AvroSchemaTest {
Assert.assertEquals(decodeWithJson, decodeWithPojo);
Assert.assertEquals(Instant.class,
decodeWithJson.getValue().getClass());
-
- AvroSchema<TimestampPojo> schemaWithJsonDefNoClassLoader =
- AvroSchema.of(SchemaDefinition.<TimestampPojo>builder()
- .withJsonDef(schemaDefinition)
- .withJSR310ConversionEnabled(false).build());
-
- TimestampPojo decodeWithJsonNoClassLoader =
schemaWithJsonDefNoClassLoader.decode(encode);
- Assert.assertNotEquals(decodeWithJsonNoClassLoader, decodeWithPojo);
- Assert.assertNotEquals(Instant.class,
decodeWithJsonNoClassLoader.getValue().getClass());
}
@Test
@@ -540,14 +531,6 @@ public class AvroSchemaTest {
Assert.assertEquals(decodeWithJson, decodeWithPojo);
Assert.assertEquals(Instant.class,
decodeWithJson.getValue().getClass());
- AvroSchema<TimestampPojo> schemaWithJsonDefNoClassLoader =
- AvroSchema.of(SchemaDefinition.<TimestampPojo>builder()
- .withJsonDef(schemaDefinition)
- .withJSR310ConversionEnabled(true).build());
-
- TimestampPojo decodeWithJsonNoClassLoader =
schemaWithJsonDefNoClassLoader.decode(encode);
- Assert.assertNotEquals(decodeWithJsonNoClassLoader, decodeWithPojo);
- Assert.assertNotEquals(Instant.class,
decodeWithJsonNoClassLoader.getValue().getClass());
}
@Data
diff --git
a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
index 01958cac58e..c3710485f9c 100644
---
a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
+++
b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
@@ -95,8 +95,8 @@ public class JsonConverter {
Schema elementSchema = schema.getElementType();
ArrayNode arrayNode = jsonNodeFactory.arrayNode();
Object[] iterable;
- if (value instanceof GenericData.Array) {
- iterable = ((GenericData.Array) value).toArray();
+ if (value instanceof GenericData.AbstractArray) {
+ iterable = ((GenericData.AbstractArray) value).toArray();
} else {
iterable = (Object[]) value;
}
diff --git
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/json/JsonConverter.java
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/json/JsonConverter.java
index a44d081cd2c..7b827b43542 100644
---
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/json/JsonConverter.java
+++
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/json/JsonConverter.java
@@ -99,8 +99,8 @@ public class JsonConverter {
Schema elementSchema = schema.getElementType();
ArrayNode arrayNode = jsonNodeFactory.arrayNode();
Object[] iterable;
- if (value instanceof GenericData.Array) {
- iterable = ((GenericData.Array) value).toArray();
+ if (value instanceof GenericData.AbstractArray) {
+ iterable = ((GenericData.AbstractArray) value).toArray();
} else {
iterable = (Object[]) value;
}
