This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 768dee5a7 feat(python): support nested field schema encodings (#3643)
768dee5a7 is described below
commit 768dee5a7e27eb688a6e380dc5fdd98165543ba0
Author: Shawn Yang <[email protected]>
AuthorDate: Sat May 2 12:25:50 2026 +0800
feat(python): support nested field schema encodings (#3643)
## Why?
Python currently loses declared integer encoding aliases inside nested
containers, so schemas such as `Dict[pyfory.fixed_int32,
List[pyfory.tagged_int64]]` can fall back to runtime inference or
incompatible local assignment behavior. This breaks cross-language
schema fidelity for nested map/list/set elements, especially in
compatible mode.
## What does this PR do?
- Preserves declared Python field schema metadata for nested container
keys, values, and elements in both pure Python and Cython serializers.
- Extends primitive container fast paths to write/read fixed, varint,
tagged, signed, unsigned, bool, and float schema-owned primitive
encodings directly.
- Adds compatible-read assignment planning and validation so remote
nested integer encodings are consumed with remote metadata and assigned
only when they satisfy the local schema.
- Updates Python code generation to keep nested integer schema aliases
in generic containers instead of converting those lists to numpy array
hints.
- Adds Python unit coverage, compiler generation coverage, and
Java/Python xlang tests for signed and unsigned nested annotated
containers.
- Documents nested Python collection aliases and compatible-read
behavior in README and Python guide pages.
## Related issues
#1017
#3630
#3625
#3630
#3636
#3639
#3640
#3642
## AI Contribution Checklist
- [ ] Substantial AI assistance was used in this PR: `yes` / `no`
- [ ] If `yes`, I included a completed [AI Contribution
Checklist](https://github.com/apache/fory/blob/main/AI_POLICY.md#9-contributor-checklist-for-ai-assisted-prs)
in this PR description and the required `AI Usage Disclosure`.
- [ ] If `yes`, my PR description includes the required `ai_review`
summary and screenshot evidence of the final clean AI review results
from both fresh reviewers on the current PR diff or current HEAD after
the latest code changes.
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
README.md | 4 +
compiler/fory_compiler/generators/python.py | 39 ++--
.../fory_compiler/tests/test_generated_code.py | 24 ++
docs/guide/python/cross-language.md | 15 +-
docs/guide/python/field-configuration.md | 32 ++-
.../org/apache/fory/xlang/PythonXlangTest.java | 27 +++
.../java/org/apache/fory/xlang/XlangTestBase.java | 74 ++++++
python/README.md | 5 +
python/pyfory/collection.pxi | 116 +++++++++-
python/pyfory/cpp/pyfory.cc | 33 ++-
python/pyfory/cpp/pyfory.h | 12 +-
python/pyfory/meta/typedef.py | 257 ++++++++++++++++++---
python/pyfory/struct.pxi | 132 ++++++++++-
python/pyfory/struct.py | 143 +++++++++---
python/pyfory/tests/test_struct.py | 99 ++++++++
python/pyfory/tests/test_typedef_encoding.py | 27 +++
python/pyfory/tests/xlang_test_main.py | 75 ++++++
17 files changed, 1010 insertions(+), 104 deletions(-)
diff --git a/README.md b/README.md
index 4ce534801..e4d995ae9 100644
--- a/README.md
+++ b/README.md
@@ -310,6 +310,10 @@ result = fory.deserialize(data)
print(result.name, result.age) # Output: chaokunyang 28
```
+Python schema aliases also apply inside declared containers, such as
+`Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]`, so nested keys and
values use the requested
+wire encoding in both pure Python and Cython modes.
+
For detailed Python usage including type hints, compatibility modes, and
advanced features, see [Python Guide](docs/guide/python).
#### Rust Serialization
diff --git a/compiler/fory_compiler/generators/python.py
b/compiler/fory_compiler/generators/python.py
index 2065b8b5f..aaf356505 100644
--- a/compiler/fory_compiler/generators/python.py
+++ b/compiler/fory_compiler/generators/python.py
@@ -114,6 +114,14 @@ class PythonGenerator(BaseGenerator):
PrimitiveKind.FLOAT32: "pyfory.float32_ndarray",
PrimitiveKind.FLOAT64: "pyfory.float64_ndarray",
}
+ LIST_ELEMENT_SCHEMA_PRIMITIVES = {
+ PrimitiveKind.INT32,
+ PrimitiveKind.INT64,
+ PrimitiveKind.TAGGED_INT64,
+ PrimitiveKind.UINT32,
+ PrimitiveKind.UINT64,
+ PrimitiveKind.TAGGED_UINT64,
+ }
# Default values for primitive types
DEFAULT_VALUES = {
@@ -645,6 +653,8 @@ class PythonGenerator(BaseGenerator):
"""Return True if a list should be represented as a numpy array."""
if not isinstance(field_type.element_type, PrimitiveType):
return False
+ if field_type.element_type.kind in self.LIST_ELEMENT_SCHEMA_PRIMITIVES:
+ return False
return (
field_type.element_type.kind in self.ARRAY_TYPE_HINTS
and not element_optional
@@ -718,6 +728,7 @@ class PythonGenerator(BaseGenerator):
element_optional: bool = False,
element_ref: bool = False,
parent_stack: Optional[List[Message]] = None,
+ prefer_numpy_array: bool = True,
) -> str:
"""Generate Python type hint."""
if isinstance(field_type, PrimitiveType):
@@ -744,7 +755,9 @@ class PythonGenerator(BaseGenerator):
if isinstance(field_type.element_type, PrimitiveType):
if not element_optional:
kind = field_type.element_type.kind
- if kind in self.ARRAY_TYPE_HINTS:
+ if prefer_numpy_array and self.uses_numpy_array(
+ field_type, element_optional
+ ):
list_type = self.ARRAY_TYPE_HINTS[kind]
else:
element_type = self.generate_type(
@@ -753,6 +766,7 @@ class PythonGenerator(BaseGenerator):
False,
False,
parent_stack,
+ False,
)
element_type = self.wrap_ref_type(
field_type.element_type,
@@ -769,6 +783,7 @@ class PythonGenerator(BaseGenerator):
False,
False,
parent_stack,
+ False,
)
element_type = self.wrap_ref_type(
field_type.element_type,
@@ -785,6 +800,7 @@ class PythonGenerator(BaseGenerator):
False,
False,
parent_stack,
+ False,
)
element_type = self.wrap_ref_type(
field_type.element_type,
@@ -800,10 +816,10 @@ class PythonGenerator(BaseGenerator):
elif isinstance(field_type, MapType):
key_type = self.generate_type(
- field_type.key_type, False, False, False, parent_stack
+ field_type.key_type, False, False, False, parent_stack, False
)
value_type = self.generate_type(
- field_type.value_type, False, False, False, parent_stack
+ field_type.value_type, False, False, False, parent_stack, False
)
value_type = self.wrap_ref_type(
field_type.value_type,
@@ -943,10 +959,9 @@ class PythonGenerator(BaseGenerator):
elif isinstance(field_type, ListType):
# Use numpy empty array for numeric types
- if isinstance(field_type.element_type, PrimitiveType):
- if field_type.element_type.kind in self.NUMPY_DTYPE_MAP:
- dtype = self.NUMPY_DTYPE_MAP[field_type.element_type.kind]
- return f"None # Use np.array([], dtype={dtype}) to
initialize"
+ if self.uses_numpy_array(field_type, element_optional=False):
+ dtype = self.NUMPY_DTYPE_MAP[field_type.element_type.kind]
+ return f"None # Use np.array([], dtype={dtype}) to initialize"
return "None"
elif isinstance(field_type, MapType):
@@ -971,13 +986,9 @@ class PythonGenerator(BaseGenerator):
elif isinstance(field_type, ListType):
# Add numpy import for primitive arrays
- if isinstance(field_type.element_type, PrimitiveType):
- if (
- field_type.element_type.kind in self.ARRAY_TYPE_HINTS
- and not element_optional
- ):
- imports.add("import numpy as np")
- return
+ if self.uses_numpy_array(field_type, element_optional):
+ imports.add("import numpy as np")
+ return
self.collect_imports(field_type.element_type, imports)
elif isinstance(field_type, MapType):
diff --git a/compiler/fory_compiler/tests/test_generated_code.py
b/compiler/fory_compiler/tests/test_generated_code.py
index e92bf4ead..0197bb7b3 100644
--- a/compiler/fory_compiler/tests/test_generated_code.py
+++ b/compiler/fory_compiler/tests/test_generated_code.py
@@ -608,6 +608,30 @@ def
test_java_nested_integer_annotations_in_generic_containers():
)
+def test_python_nested_integer_schema_aliases_in_generic_containers():
+ schema = parse_fdl(
+ dedent(
+ """
+ package gen;
+
+ message NestedIntegerSchemaAliases {
+ map<fixed_int32, list<tagged_int64>> signed_values = 1;
+ map<fixed_uint32, list<tagged_uint64>> unsigned_values = 2;
+ }
+ """
+ )
+ )
+ python_output = render_files(generate_files(schema, PythonGenerator))
+ assert (
+ "signed_values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]"
+ in python_output
+ )
+ assert (
+ "unsigned_values: Dict[pyfory.fixed_uint32,
List[pyfory.tagged_uint64]]"
+ in python_output
+ )
+
+
def test_cpp_nested_integer_specs_in_generic_containers():
schema = parse_fdl(
dedent(
diff --git a/docs/guide/python/cross-language.md
b/docs/guide/python/cross-language.md
index 306453836..fbe3e90bc 100644
--- a/docs/guide/python/cross-language.md
+++ b/docs/guide/python/cross-language.md
@@ -99,16 +99,23 @@ Use pyfory type annotations for explicit cross-language
type mapping:
```python
from dataclasses import dataclass
+from typing import Dict, List
import pyfory
@dataclass
class TypedData:
- int_value: pyfory.int32 # 32-bit integer
- long_value: pyfory.int64 # 64-bit integer
- float_value: pyfory.float32 # 32-bit float
- double_value: pyfory.float64 # 64-bit float
+ int_value: pyfory.int32 # 32-bit integer
+ long_value: pyfory.int64 # 64-bit integer
+ float_value: pyfory.float32 # 32-bit float
+ double_value: pyfory.float64 # 64-bit float
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]
```
+Nested collection annotations are part of the field schema. For example,
+`Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]` writes map keys as fixed
int32 values and the
+nested list elements as tagged int64 values. Compatible-mode reads consume
bytes with the remote
+schema metadata, then assign only when the decoded value safely satisfies the
local schema.
+
## Reduced-Precision Types
`pyfory.serialization` exports Cython-only carrier types for xlang
reduced-precision values:
diff --git a/docs/guide/python/field-configuration.md
b/docs/guide/python/field-configuration.md
index 487913306..1e4e8f35c 100644
--- a/docs/guide/python/field-configuration.md
+++ b/docs/guide/python/field-configuration.md
@@ -278,6 +278,9 @@ class FloatingPoint:
| `pyfory.int16` | fixed | 2 bytes |
| `pyfory.int32` | varint | 1-5 bytes |
| `pyfory.int64` | varint | 1-10 bytes |
+| `pyfory.fixed_int32` | fixed | 4 bytes |
+| `pyfory.fixed_int64` | fixed | 8 bytes |
+| `pyfory.tagged_int64` | tagged | 1-9 bytes |
| `pyfory.uint8` | fixed | 1 byte |
| `pyfory.uint16` | fixed | 2 bytes |
| `pyfory.uint32` | varint | 1-5 bytes |
@@ -292,7 +295,32 @@ class FloatingPoint:
- `varint`: Best for values that are often small (default for
int32/int64/uint32/uint64)
- `fixed`: Best for values that use full range (e.g., timestamps, hashes)
-- `tagged`: When type information needs to be preserved (uint64 only)
+- `tagged`: When type information needs to be preserved (int64/uint64 only)
+
+## Nested Container Type Annotations
+
+Integer encoding aliases can be used inside declared collection schemas. Fory
uses the declared
+field schema for every nested element, key, and value in both pure Python and
Cython modes:
+
+```python
+from dataclasses import dataclass, field
+from typing import Dict, List
+import pyfory
+
+
+@dataclass
+class Counters:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] =
field(default_factory=dict)
+```
+
+For `values`, map keys are written as fixed-width int32 values and each nested
list element is
+written as tagged int64. Runtime type inference is used only for dynamic or
unknown container
+schemas.
+
+In compatible mode, readers consume field bytes using the remote schema
metadata. Python assigns the
+decoded value only when it can safely satisfy the local declared schema.
Different integer encodings
+in the same signedness and width domain are compatible, and same-signedness
narrowing is assigned
+only after range validation.
## Complete Example
@@ -481,6 +509,8 @@ class User:
| `pyfory.field(dynamic=False)` | Skip type info (use declared
type) |
| `Optional[T]` | Type hint for nullable fields
|
| `pyfory.int32`, `pyfory.int64` | Signed integers (varint
encoding) |
+| `pyfory.fixed_int32`, `pyfory.fixed_int64` | Fixed-size signed
|
+| `pyfory.tagged_int64` | Tagged encoding for int64
|
| `pyfory.uint32`, `pyfory.uint64` | Unsigned integers (varint
encoding) |
| `pyfory.fixed_uint32`, `pyfory.fixed_uint64` | Fixed-size unsigned
|
| `pyfory.tagged_uint64` | Tagged encoding for uint64
|
diff --git
a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java
b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java
index 4bcca3a5e..b7c95fe64 100644
--- a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java
@@ -232,6 +232,33 @@ public class PythonXlangTest extends XlangTestBase {
super.testCollectionElementRefRemoteTracking(enableCodegen);
}
+ @Override
+ @Test(groups = "xlang", dataProvider = "enableCodegenParallel")
+ public void testSignedNestedAnnotatedContainerSchemaConsistent(boolean
enableCodegen)
+ throws IOException {
+ super.testSignedNestedAnnotatedContainerSchemaConsistent(enableCodegen);
+ }
+
+ @Override
+ @Test(groups = "xlang", dataProvider = "enableCodegenParallel")
+ public void testSignedNestedAnnotatedContainerCompatible(boolean
enableCodegen)
+ throws IOException {
+ super.testSignedNestedAnnotatedContainerCompatible(enableCodegen);
+ }
+
+ @Override
+ @Test(groups = "xlang", dataProvider = "enableCodegenParallel")
+ public void testNestedAnnotatedContainerSchemaConsistent(boolean
enableCodegen)
+ throws IOException {
+ super.testNestedAnnotatedContainerSchemaConsistent(enableCodegen);
+ }
+
+ @Override
+ @Test(groups = "xlang", dataProvider = "enableCodegenParallel")
+ public void testNestedAnnotatedContainerCompatible(boolean enableCodegen)
throws IOException {
+ super.testNestedAnnotatedContainerCompatible(enableCodegen);
+ }
+
@Override
@Test(groups = "xlang", dataProvider = "enableCodegenParallel")
public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen)
throws IOException {
diff --git
a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java
b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java
index ba8556e45..4de245b33 100644
--- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java
+++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java
@@ -39,6 +39,8 @@ import org.apache.fory.Fory;
import org.apache.fory.ForyTestBase;
import org.apache.fory.annotation.ForyField;
import org.apache.fory.annotation.ForyStruct;
+import org.apache.fory.annotation.Int32Type;
+import org.apache.fory.annotation.Int64Type;
import org.apache.fory.annotation.Ref;
import org.apache.fory.annotation.UInt16Type;
import org.apache.fory.annotation.UInt32Type;
@@ -1015,6 +1017,78 @@ public abstract class XlangTestBase extends ForyTestBase
{
Map<String, String> data;
}
+ @Data
+ static class SignedNestedAnnotatedContainerSchemaConsistent {
+ Map<
+ @Int32Type(encoding = Int32Encoding.FIXED) Integer,
+ List<@Int64Type(encoding = Int64Encoding.TAGGED) Long>>
+ values;
+ }
+
+ protected void testSignedNestedAnnotatedContainerSchemaConsistent(boolean
enableCodegen)
+ throws java.io.IOException {
+ String caseName =
"test_signed_nested_annotated_container_schema_consistent";
+ Fory fory =
+
Fory.builder().withXlang(true).withCompatible(false).withCodegen(enableCodegen).build();
+ fory.register(SignedNestedAnnotatedContainerSchemaConsistent.class, 803);
+
+ SignedNestedAnnotatedContainerSchemaConsistent obj =
+ new SignedNestedAnnotatedContainerSchemaConsistent();
+ obj.values = new LinkedHashMap<>();
+ obj.values.put(-7, Arrays.asList(7L, -1000000000L));
+ obj.values.put(3, Collections.singletonList(42L));
+
+ serDeCheck(fory, obj);
+
+ MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512);
+ fory.serialize(buffer, obj);
+ ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0,
buffer.writerIndex()));
+ runPeer(ctx);
+
+ MemoryBuffer buffer2 = readBuffer(ctx.dataFile());
+ SignedNestedAnnotatedContainerSchemaConsistent result =
+ (SignedNestedAnnotatedContainerSchemaConsistent)
fory.deserialize(buffer2);
+ Assert.assertEquals(result, obj);
+ }
+
+ @Data
+ static class SignedNestedAnnotatedContainerCompatible {
+ Map<
+ @Int32Type(encoding = Int32Encoding.FIXED) Integer,
+ List<@Int64Type(encoding = Int64Encoding.TAGGED) Long>>
+ values;
+ }
+
+ protected void testSignedNestedAnnotatedContainerCompatible(boolean
enableCodegen)
+ throws java.io.IOException {
+ String caseName = "test_signed_nested_annotated_container_compatible";
+ Fory fory =
+ Fory.builder()
+ .withXlang(true)
+ .withCompatible(true)
+ .withCodegen(enableCodegen)
+ .withMetaCompressor(new NoOpMetaCompressor())
+ .build();
+ fory.register(SignedNestedAnnotatedContainerCompatible.class, 804);
+
+ SignedNestedAnnotatedContainerCompatible obj = new
SignedNestedAnnotatedContainerCompatible();
+ obj.values = new LinkedHashMap<>();
+ obj.values.put(-7, Arrays.asList(7L, -1000000000L));
+ obj.values.put(3, Collections.singletonList(42L));
+
+ serDeCheck(fory, obj);
+
+ MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512);
+ fory.serialize(buffer, obj);
+ ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0,
buffer.writerIndex()));
+ runPeer(ctx);
+
+ MemoryBuffer buffer2 = readBuffer(ctx.dataFile());
+ SignedNestedAnnotatedContainerCompatible result =
+ (SignedNestedAnnotatedContainerCompatible) fory.deserialize(buffer2);
+ Assert.assertEquals(result, obj);
+ }
+
@Data
static class NestedAnnotatedContainerSchemaConsistent {
Map<
diff --git a/python/README.md b/python/README.md
index d92bec3e5..9e3089d3c 100644
--- a/python/README.md
+++ b/python/README.md
@@ -454,6 +454,7 @@ Serialize data in Python and deserialize it in Java, Go,
Rust, or other supporte
**Python (Serializer)**
```python
+from dataclasses import dataclass
import pyfory
# Cross-language mode for interoperability
@@ -472,6 +473,10 @@ binary_data = f.serialize(person)
# binary_data can now be sent to Java, Go, etc.
```
+Nested collection aliases are declared schema. For example,
+`Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]` writes map keys with
fixed int32 encoding and
+nested list elements with tagged int64 encoding in both pure Python and Cython
modes.
+
**Java (Deserializer)**
```java
diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi
index a4b98118f..6a663cac8 100644
--- a/python/pyfory/collection.pxi
+++ b/python/pyfory/collection.pxi
@@ -947,29 +947,79 @@ cdef class MapSerializer(Serializer):
if not key_write_ref or not
ref_writer.write_ref_or_null(c_buffer, key):
if key_cls is str:
write_context.write_string(key)
- elif key_serializer_type is Int64Serializer:
+ elif key_serializer_type is Int64Serializer or
key_serializer_type is Varint64Serializer:
write_context.write_varint64(key)
+ elif key_serializer_type is FixedInt64Serializer:
+ write_context.write_int64(key)
+ elif key_serializer_type is TaggedInt64Serializer:
+ write_context.write_tagged_int64(key)
elif key_serializer_type is Float64Serializer:
write_context.write_double(key)
- elif key_serializer_type is Int32Serializer:
+ elif key_serializer_type is Int32Serializer or
key_serializer_type is Varint32Serializer:
write_context.write_varint32(key)
+ elif key_serializer_type is FixedInt32Serializer:
+ write_context.write_int32(key)
elif key_serializer_type is Float32Serializer:
write_context.write_float(key)
+ elif key_serializer_type is BooleanSerializer:
+ write_context.write_bool(key)
+ elif key_serializer_type is ByteSerializer:
+ write_context.write_int8(key)
+ elif key_serializer_type is Int16Serializer:
+ write_context.write_int16(key)
+ elif key_serializer_type is Uint8Serializer:
+ write_context.write_uint8(key)
+ elif key_serializer_type is Uint16Serializer:
+ write_context.write_uint16(key)
+ elif key_serializer_type is Uint32Serializer:
+ write_context.write_uint32(key)
+ elif key_serializer_type is VarUint32Serializer:
+ write_context.write_var_uint32(key)
+ elif key_serializer_type is Uint64Serializer:
+ write_context.write_uint64(key)
+ elif key_serializer_type is VarUint64Serializer:
+ write_context.write_var_uint64(key)
+ elif key_serializer_type is TaggedUint64Serializer:
+ write_context.write_tagged_uint64(key)
else:
key_serializer.write(write_context, key)
if not value_write_ref or not
ref_writer.write_ref_or_null(c_buffer, value):
if value_cls is str:
write_context.write_string(value)
- elif value_serializer_type is Int64Serializer:
+ elif value_serializer_type is Int64Serializer or
value_serializer_type is Varint64Serializer:
write_context.write_varint64(value)
+ elif value_serializer_type is FixedInt64Serializer:
+ write_context.write_int64(value)
+ elif value_serializer_type is TaggedInt64Serializer:
+ write_context.write_tagged_int64(value)
elif value_serializer_type is Float64Serializer:
write_context.write_double(value)
- elif value_serializer_type is Int32Serializer:
+ elif value_serializer_type is Int32Serializer or
value_serializer_type is Varint32Serializer:
write_context.write_varint32(value)
+ elif value_serializer_type is FixedInt32Serializer:
+ write_context.write_int32(value)
elif value_serializer_type is Float32Serializer:
write_context.write_float(value)
elif value_serializer_type is BooleanSerializer:
write_context.write_bool(value)
+ elif value_serializer_type is ByteSerializer:
+ write_context.write_int8(value)
+ elif value_serializer_type is Int16Serializer:
+ write_context.write_int16(value)
+ elif value_serializer_type is Uint8Serializer:
+ write_context.write_uint8(value)
+ elif value_serializer_type is Uint16Serializer:
+ write_context.write_uint16(value)
+ elif value_serializer_type is Uint32Serializer:
+ write_context.write_uint32(value)
+ elif value_serializer_type is VarUint32Serializer:
+ write_context.write_var_uint32(value)
+ elif value_serializer_type is Uint64Serializer:
+ write_context.write_uint64(value)
+ elif value_serializer_type is VarUint64Serializer:
+ write_context.write_var_uint64(value)
+ elif value_serializer_type is TaggedUint64Serializer:
+ write_context.write_tagged_uint64(value)
else:
value_serializer.write(write_context, value)
chunk_size += 1
@@ -1092,14 +1142,40 @@ cdef class MapSerializer(Serializer):
else:
if key_serializer_type is StringSerializer:
key = read_context.read_string()
- elif key_serializer_type is Int64Serializer:
+ elif key_serializer_type is Int64Serializer or
key_serializer_type is Varint64Serializer:
key = read_context.read_varint64()
+ elif key_serializer_type is FixedInt64Serializer:
+ key = read_context.read_int64()
+ elif key_serializer_type is TaggedInt64Serializer:
+ key = read_context.read_tagged_int64()
elif key_serializer_type is Float64Serializer:
key = read_context.read_double()
- elif key_serializer_type is Int32Serializer:
+ elif key_serializer_type is Int32Serializer or
key_serializer_type is Varint32Serializer:
key = read_context.read_varint32()
+ elif key_serializer_type is FixedInt32Serializer:
+ key = read_context.read_int32()
elif key_serializer_type is Float32Serializer:
key = read_context.read_float()
+ elif key_serializer_type is BooleanSerializer:
+ key = read_context.read_bool()
+ elif key_serializer_type is ByteSerializer:
+ key = read_context.read_int8()
+ elif key_serializer_type is Int16Serializer:
+ key = read_context.read_int16()
+ elif key_serializer_type is Uint8Serializer:
+ key = read_context.read_uint8()
+ elif key_serializer_type is Uint16Serializer:
+ key = read_context.read_uint16()
+ elif key_serializer_type is Uint32Serializer:
+ key = read_context.read_uint32()
+ elif key_serializer_type is VarUint32Serializer:
+ key = read_context.read_var_uint32()
+ elif key_serializer_type is Uint64Serializer:
+ key = read_context.read_uint64()
+ elif key_serializer_type is VarUint64Serializer:
+ key = read_context.read_var_uint64()
+ elif key_serializer_type is TaggedUint64Serializer:
+ key = read_context.read_tagged_uint64()
else:
key = read_context.read_non_ref(key_serializer)
if track_value_ref:
@@ -1114,16 +1190,40 @@ cdef class MapSerializer(Serializer):
else:
if value_serializer_type is StringSerializer:
value = read_context.read_string()
- elif value_serializer_type is Int64Serializer:
+ elif value_serializer_type is Int64Serializer or
value_serializer_type is Varint64Serializer:
value = read_context.read_varint64()
+ elif value_serializer_type is FixedInt64Serializer:
+ value = read_context.read_int64()
+ elif value_serializer_type is TaggedInt64Serializer:
+ value = read_context.read_tagged_int64()
elif value_serializer_type is Float64Serializer:
value = read_context.read_double()
- elif value_serializer_type is Int32Serializer:
+ elif value_serializer_type is Int32Serializer or
value_serializer_type is Varint32Serializer:
value = read_context.read_varint32()
+ elif value_serializer_type is FixedInt32Serializer:
+ value = read_context.read_int32()
elif value_serializer_type is Float32Serializer:
value = read_context.read_float()
elif value_serializer_type is BooleanSerializer:
value = read_context.read_bool()
+ elif value_serializer_type is ByteSerializer:
+ value = read_context.read_int8()
+ elif value_serializer_type is Int16Serializer:
+ value = read_context.read_int16()
+ elif value_serializer_type is Uint8Serializer:
+ value = read_context.read_uint8()
+ elif value_serializer_type is Uint16Serializer:
+ value = read_context.read_uint16()
+ elif value_serializer_type is Uint32Serializer:
+ value = read_context.read_uint32()
+ elif value_serializer_type is VarUint32Serializer:
+ value = read_context.read_var_uint32()
+ elif value_serializer_type is Uint64Serializer:
+ value = read_context.read_uint64()
+ elif value_serializer_type is VarUint64Serializer:
+ value = read_context.read_var_uint64()
+ elif value_serializer_type is TaggedUint64Serializer:
+ value = read_context.read_tagged_uint64()
else:
value = read_context.read_non_ref(value_serializer)
map_[key] = value
diff --git a/python/pyfory/cpp/pyfory.cc b/python/pyfory/cpp/pyfory.cc
index 0f35f8a64..06b587db8 100644
--- a/python/pyfory/cpp/pyfory.cc
+++ b/python/pyfory/cpp/pyfory.cc
@@ -582,10 +582,19 @@ static bool can_use_list_sequence_fastpath(PyObject
**items, Py_ssize_t size,
}
return true;
case TypeId::VARINT64:
+ case TypeId::INT64:
+ case TypeId::TAGGED_INT64:
case TypeId::VARINT32:
+ case TypeId::INT32:
+ case TypeId::VAR_UINT64:
+ case TypeId::UINT64:
+ case TypeId::TAGGED_UINT64:
+ case TypeId::VAR_UINT32:
+ case TypeId::UINT32:
+ case TypeId::UINT8:
case TypeId::INT8:
+ case TypeId::UINT16:
case TypeId::INT16:
- case TypeId::INT32:
for (Py_ssize_t i = 0; i < size; ++i) {
if (!PyLong_CheckExact(items[i])) {
return false;
@@ -599,6 +608,7 @@ static bool can_use_list_sequence_fastpath(PyObject
**items, Py_ssize_t size,
}
}
return true;
+ case TypeId::FLOAT32:
case TypeId::FLOAT64:
for (Py_ssize_t i = 0; i < size; ++i) {
if (!PyFloat_CheckExact(items[i])) {
@@ -1086,9 +1096,13 @@ static int write_primitive_sequence(PyObject **items,
Py_ssize_t size,
return 0;
}
default:
- PyErr_Format(PyExc_ValueError, "unsupported primitive fastpath type id:
%u",
- static_cast<unsigned>(type_id));
- return -1;
+ for (Py_ssize_t i = 0; i < size; ++i) {
+ if (FORY_PREDICT_FALSE(write_primitive_item(buffer, items[i], type_id) !=
+ 0)) {
+ return -1;
+ }
+ }
+ return 0;
}
}
@@ -1434,9 +1448,14 @@ static int read_primitive_sequence_indexed(Buffer
*buffer, Py_ssize_t size,
}
return 0;
default:
- PyErr_Format(PyExc_ValueError, "unsupported primitive fastpath type id:
%u",
- static_cast<unsigned>(type_id));
- return -1;
+ for (Py_ssize_t i = 0; i < size; ++i) {
+ PyObject *item = read_primitive_item(buffer, type_id);
+ if (FORY_PREDICT_FALSE(item == nullptr)) {
+ return -1;
+ }
+ set_item(i, item);
+ }
+ return 0;
}
}
diff --git a/python/pyfory/cpp/pyfory.h b/python/pyfory/cpp/pyfory.h
index 3801929c9..619ad9b77 100644
--- a/python/pyfory/cpp/pyfory.h
+++ b/python/pyfory/cpp/pyfory.h
@@ -49,12 +49,22 @@ inline constexpr bool
Fory_CanUsePrimitiveCollectionFastpath(uint8_t type_id) {
switch (static_cast<TypeId>(type_id)) {
case TypeId::STRING:
case TypeId::VARINT64:
+ case TypeId::INT64:
+ case TypeId::TAGGED_INT64:
case TypeId::VARINT32:
+ case TypeId::INT32:
+ case TypeId::VAR_UINT64:
+ case TypeId::UINT64:
+ case TypeId::TAGGED_UINT64:
+ case TypeId::VAR_UINT32:
+ case TypeId::UINT32:
case TypeId::BOOL:
+ case TypeId::FLOAT32:
case TypeId::FLOAT64:
+ case TypeId::UINT8:
case TypeId::INT8:
+ case TypeId::UINT16:
case TypeId::INT16:
- case TypeId::INT32:
return true;
default:
return false;
diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py
index 759fbabdc..521cbf8de 100644
--- a/python/pyfory/meta/typedef.py
+++ b/python/pyfory/meta/typedef.py
@@ -71,16 +71,18 @@ class TypeDef:
self.encoded = encoded
self.is_compressed = is_compressed
- def create_fields_serializer(self, resolver, resolved_field_names=None):
+ def create_fields_serializer(self, resolver, resolved_field_names=None,
local_field_types=None):
"""Create serializers for each field.
Args:
resolver: The type resolver
resolved_field_names: Optional list of resolved field names (for
TAG_ID encoding).
If None, uses field_info.name directly.
+ local_field_types: Optional precomputed local field type map.
"""
- field_nullable = resolver.field_nullable
- field_types = infer_field_types(self.cls,
field_nullable=field_nullable)
+ field_types = local_field_types
+ if field_types is None:
+ field_types = infer_field_types(self.cls,
field_nullable=resolver.field_nullable)
serializers = []
for i, field_info in enumerate(self.fields):
# Use resolved name if provided, otherwise use original name
@@ -155,33 +157,56 @@ class TypeDef:
return resolver.get_type_info_by_name(self.namespace,
self.typename).serializer
from pyfory.struct import DataClassSerializer
+ from pyfory.struct import FieldInfo as StructFieldInfo
+ from pyfory.type_util import get_type_hints, unwrap_optional
# Resolve actual field names from TAG_ID encoding if needed
field_names = self._resolve_field_names_from_tag_ids()
- # Build nullable_fields using resolved field names
- nullable_fields = {}
+ local_field_infos = build_field_infos(resolver, self.cls)
+ local_infos_by_name = {field_info.name: field_info for field_info in
local_field_infos}
+ local_infos_by_tag = {field_info.tag_id: field_info for field_info in
local_field_infos if field_info.tag_id >= 0}
+ local_field_types = infer_field_types(self.cls,
field_nullable=resolver.field_nullable)
+ type_hints = get_type_hints(self.cls)
+ serializers = self.create_fields_serializer(resolver, field_names,
local_field_types)
+ runtime_field_infos = []
for i, field_info in enumerate(self.fields):
resolved_name = field_names[i]
- nullable_fields[resolved_name] = field_info.field_type.is_nullable
-
- dynamic_fields = {}
- ref_fields = {}
- for i, field_info in enumerate(self.fields):
- resolved_name = field_names[i]
- type_id = field_info.field_type.type_id
- ref_fields[resolved_name] = field_info.field_type.is_tracking_ref
- if is_polymorphic_type(type_id):
- dynamic_fields[resolved_name] = True
+ local_info = None
+ if field_info.tag_id >= 0:
+ local_info = local_infos_by_tag.get(field_info.tag_id)
+ if local_info is None:
+ local_info = local_infos_by_name.get(resolved_name)
+ can_assign, validation_field_type = plan_field_assignment(
+ field_info.field_type,
+ local_info.field_type if local_info is not None else None,
+ )
+ type_hint = type_hints.get(resolved_name, typing.Any)
+ unwrapped_type, _ = unwrap_optional(type_hint,
field_nullable=resolver.field_nullable)
+ runtime_field_infos.append(
+ StructFieldInfo(
+ name=resolved_name,
+ index=i,
+ type_hint=type_hint,
+ tag_id=field_info.tag_id,
+ nullable=field_info.field_type.is_nullable,
+ ref=field_info.field_type.is_tracking_ref,
+ dynamic=is_polymorphic_type(field_info.field_type.type_id),
+ runtime_ref_tracking=field_info.field_type.is_tracking_ref,
+ type_id=field_info.field_type.type_id,
+ serializer=serializers[i],
+ unwrapped_type=unwrapped_type,
+ field_type=field_info.field_type,
+ assign=can_assign,
+ validation_field_type=validation_field_type,
+ )
+ )
return DataClassSerializer(
resolver,
self.cls,
- field_names=field_names,
- serializers=self.create_fields_serializer(resolver, field_names),
- nullable_fields=nullable_fields,
- dynamic_fields=dynamic_fields,
- ref_fields=ref_fields,
+ field_infos=runtime_field_infos,
+ fields_from_typedef=True,
)
def __repr__(self):
@@ -442,12 +467,126 @@ class DynamicFieldType(FieldType):
"Union fields don't write field type info, \
they are not dynamic field types"
)
+ if self.type_id != TypeId.UNKNOWN:
+ return FieldType.create_serializer(self, resolver, type_)
return None
def __repr__(self):
return f"DynamicFieldType(type_id={self.type_id},
is_monomorphic={self.is_monomorphic}, is_nullable={self.is_nullable},
is_tracking_ref={self.is_tracking_ref})"
+_SIGNED_INT32_TYPE_IDS = frozenset((TypeId.INT32, TypeId.VARINT32))
+_SIGNED_INT64_TYPE_IDS = frozenset((TypeId.INT64, TypeId.VARINT64,
TypeId.TAGGED_INT64))
+_UNSIGNED_INT32_TYPE_IDS = frozenset((TypeId.UINT32, TypeId.VAR_UINT32))
+_UNSIGNED_INT64_TYPE_IDS = frozenset((TypeId.UINT64, TypeId.VAR_UINT64,
TypeId.TAGGED_UINT64))
+_INT_TYPE_DOMAINS = {type_id: (True, 32) for type_id in _SIGNED_INT32_TYPE_IDS}
+_INT_TYPE_DOMAINS.update({type_id: (True, 64) for type_id in
_SIGNED_INT64_TYPE_IDS})
+_INT_TYPE_DOMAINS.update({type_id: (False, 32) for type_id in
_UNSIGNED_INT32_TYPE_IDS})
+_INT_TYPE_DOMAINS.update({type_id: (False, 64) for type_id in
_UNSIGNED_INT64_TYPE_IDS})
+_INT_RANGES = {
+ (True, 32): (-(1 << 31), (1 << 31) - 1),
+ (True, 64): (-(1 << 63), (1 << 63) - 1),
+ (False, 32): (0, (1 << 32) - 1),
+ (False, 64): (0, (1 << 64) - 1),
+}
+
+
+def _requires_nullable_validation(remote_field_type: FieldType,
local_field_type: FieldType) -> bool:
+ return remote_field_type.is_nullable and not local_field_type.is_nullable
+
+
+def _field_type_assignment(remote_field_type: FieldType, local_field_type:
FieldType) -> typing.Tuple[bool, bool]:
+ if local_field_type is None:
+ return False, False
+ needs_validation = _requires_nullable_validation(remote_field_type,
local_field_type)
+ remote_type_id = remote_field_type.type_id
+ local_type_id = local_field_type.type_id
+ if local_type_id == TypeId.UNKNOWN:
+ return True, needs_validation
+ if remote_type_id == TypeId.UNKNOWN:
+ return True, True
+ if remote_type_id in (TypeId.LIST, TypeId.SET):
+ if local_type_id != remote_type_id:
+ return False, False
+ child_assignable, child_needs_validation = _field_type_assignment(
+ remote_field_type.element_type,
+ local_field_type.element_type,
+ )
+ return child_assignable, needs_validation or child_needs_validation
+ if remote_type_id == TypeId.MAP:
+ if local_type_id != TypeId.MAP:
+ return False, False
+ key_assignable, key_needs_validation = _field_type_assignment(
+ remote_field_type.key_type,
+ local_field_type.key_type,
+ )
+ value_assignable, value_needs_validation = _field_type_assignment(
+ remote_field_type.value_type,
+ local_field_type.value_type,
+ )
+ return key_assignable and value_assignable, needs_validation or
key_needs_validation or value_needs_validation
+ remote_int_domain = _INT_TYPE_DOMAINS.get(remote_type_id)
+ local_int_domain = _INT_TYPE_DOMAINS.get(local_type_id)
+ if remote_int_domain is not None or local_int_domain is not None:
+ if remote_int_domain is None or local_int_domain is None:
+ return False, False
+ remote_signed, remote_width = remote_int_domain
+ local_signed, local_width = local_int_domain
+ if remote_signed != local_signed:
+ return False, False
+ return True, needs_validation or remote_width > local_width
+ if remote_type_id == local_type_id:
+ return True, needs_validation
+ return False, False
+
+
+def plan_field_assignment(
+ remote_field_type: FieldType, local_field_type: typing.Optional[FieldType]
+) -> typing.Tuple[bool, typing.Optional[FieldType]]:
+ assignable, needs_validation = _field_type_assignment(remote_field_type,
local_field_type)
+ if not assignable:
+ return False, None
+ return True, local_field_type if needs_validation else None
+
+
+def _validate_int_value(value, type_id: int) -> bool:
+ domain = _INT_TYPE_DOMAINS.get(type_id)
+ if domain is None:
+ return True
+ if type(value) is not int:
+ return False
+ min_value, max_value = _INT_RANGES[domain]
+ return min_value <= value <= max_value
+
+
+def is_value_assignable(value, local_field_type: FieldType) -> bool:
+ if value is None:
+ return local_field_type.is_nullable
+ type_id = local_field_type.type_id
+ if type_id == TypeId.UNKNOWN:
+ return True
+ if type_id in (TypeId.LIST, TypeId.SET):
+ if not isinstance(value, (list, tuple, set)):
+ return False
+ return all(is_value_assignable(element, local_field_type.element_type)
for element in value)
+ if type_id == TypeId.MAP:
+ if not isinstance(value, dict):
+ return False
+ return all(
+ is_value_assignable(key, local_field_type.key_type) and
is_value_assignable(map_value, local_field_type.value_type)
+ for key, map_value in value.items()
+ )
+ if type_id in _INT_TYPE_DOMAINS:
+ return _validate_int_value(value, type_id)
+ if type_id == TypeId.BOOL:
+ return type(value) is bool
+ if type_id in (TypeId.FLOAT32, TypeId.FLOAT64):
+ return type(value) is float
+ if type_id == TypeId.STRING:
+ return type(value) is str
+ return True
+
+
def build_field_infos(type_resolver, cls):
"""Build field information for the class.
@@ -511,7 +650,7 @@ def build_field_infos(type_resolver, cls):
field_info = FieldInfo(field_name, field_type, cls.__name__, tag_id)
field_infos.append(field_info)
- field_types = infer_field_types(cls)
+ field_types = infer_field_types(cls, field_nullable=field_nullable)
serializers = [field_info.field_type.create_serializer(type_resolver,
field_types.get(field_info.name, None)) for field_info in field_infos]
# Get just the field names for sorting
@@ -559,6 +698,7 @@ def build_field_type_from_type_ids_with_ref(
):
"""Build field type from type IDs with explicit ref tracking control."""
from pyfory.type_util import unwrap_ref
+ from pyfory.type_util import unwrap_optional
type_id = type_ids[0]
if type_id is None:
@@ -571,6 +711,7 @@ def build_field_type_from_type_ids_with_ref(
morphic = not is_polymorphic_type(type_id)
if type_id in [TypeId.SET, TypeId.LIST]:
elem_hint = None
+ elem_nullable = False
elem_ref_override = None
if type_hint is not None:
origin = typing.get_origin(type_hint) if hasattr(typing,
"get_origin") else getattr(type_hint, "__origin__", None)
@@ -578,19 +719,21 @@ def build_field_type_from_type_ids_with_ref(
args = typing.get_args(type_hint) if hasattr(typing,
"get_args") else getattr(type_hint, "__args__", ())
if args:
elem_hint, elem_ref_override = unwrap_ref(args[0])
+ elem_hint, elem_nullable = unwrap_optional(elem_hint)
elif origin in (tuple, typing.Tuple):
tuple_elem_hint = get_homogeneous_tuple_elem_type(type_hint)
if tuple_elem_hint is not None:
elem_hint, elem_ref_override = unwrap_ref(tuple_elem_hint)
+ elem_hint, elem_nullable = unwrap_optional(elem_hint)
elem_tracking_ref = is_tracking_ref
if elem_ref_override is not None:
- elem_tracking_ref = elem_ref_override and is_tracking_ref
+ elem_tracking_ref = elem_ref_override and type_resolver.track_ref
elem_type = build_field_type_from_type_ids_with_ref(
type_resolver,
field_name,
type_ids[1],
visitor,
- is_nullable=False,
+ is_nullable=elem_nullable,
is_tracking_ref=elem_tracking_ref,
type_hint=elem_hint,
)
@@ -600,6 +743,8 @@ def build_field_type_from_type_ids_with_ref(
elif type_id == TypeId.MAP:
key_hint = None
value_hint = None
+ key_nullable = False
+ value_nullable = False
key_ref_override = None
value_ref_override = None
if type_hint is not None:
@@ -608,19 +753,21 @@ def build_field_type_from_type_ids_with_ref(
args = typing.get_args(type_hint) if hasattr(typing,
"get_args") else getattr(type_hint, "__args__", ())
if len(args) >= 2:
key_hint, key_ref_override = unwrap_ref(args[0])
+ key_hint, key_nullable = unwrap_optional(key_hint)
value_hint, value_ref_override = unwrap_ref(args[1])
+ value_hint, value_nullable = unwrap_optional(value_hint)
key_tracking_ref = is_tracking_ref
if key_ref_override is not None:
- key_tracking_ref = key_ref_override and is_tracking_ref
+ key_tracking_ref = key_ref_override and type_resolver.track_ref
value_tracking_ref = is_tracking_ref
if value_ref_override is not None:
- value_tracking_ref = value_ref_override and is_tracking_ref
+ value_tracking_ref = value_ref_override and type_resolver.track_ref
key_type = build_field_type_from_type_ids_with_ref(
type_resolver,
field_name,
type_ids[1],
visitor,
- is_nullable=False,
+ is_nullable=key_nullable,
is_tracking_ref=key_tracking_ref,
type_hint=key_hint,
)
@@ -629,7 +776,7 @@ def build_field_type_from_type_ids_with_ref(
field_name,
type_ids[2],
visitor,
- is_nullable=False,
+ is_nullable=value_nullable,
is_tracking_ref=value_tracking_ref,
type_hint=value_hint,
)
@@ -664,6 +811,8 @@ def build_field_type(type_resolver, field_name: str,
type_hint, visitor, is_null
def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids,
visitor, is_nullable=False, type_hint=None):
+ from pyfory.type_util import unwrap_optional, unwrap_ref
+
tracking_ref = type_resolver.track_ref
type_id = type_ids[0]
if type_id is None:
@@ -675,11 +824,59 @@ def build_field_type_from_type_ids(type_resolver,
field_name: str, type_ids, vis
assert type_id >= 0, f"Unknown type: {type_id} for field: {field_name}"
morphic = not is_polymorphic_type(type_id)
if type_id in [TypeId.SET, TypeId.LIST]:
- elem_type = build_field_type_from_type_ids(type_resolver, field_name,
type_ids[1], visitor, is_nullable=False)
+ elem_hint = None
+ elem_nullable = False
+ if type_hint is not None:
+ origin = typing.get_origin(type_hint) if hasattr(typing,
"get_origin") else getattr(type_hint, "__origin__", None)
+ if origin in (list, typing.List, set, typing.Set):
+ args = typing.get_args(type_hint) if hasattr(typing,
"get_args") else getattr(type_hint, "__args__", ())
+ if args:
+ elem_hint, _ = unwrap_ref(args[0])
+ elem_hint, elem_nullable = unwrap_optional(elem_hint)
+ elif origin in (tuple, typing.Tuple):
+ tuple_elem_hint = get_homogeneous_tuple_elem_type(type_hint)
+ if tuple_elem_hint is not None:
+ elem_hint, _ = unwrap_ref(tuple_elem_hint)
+ elem_hint, elem_nullable = unwrap_optional(elem_hint)
+ elem_type = build_field_type_from_type_ids(
+ type_resolver,
+ field_name,
+ type_ids[1],
+ visitor,
+ is_nullable=elem_nullable,
+ type_hint=elem_hint,
+ )
return CollectionFieldType(type_id, morphic, is_nullable,
tracking_ref, elem_type)
elif type_id == TypeId.MAP:
- key_type = build_field_type_from_type_ids(type_resolver, field_name,
type_ids[1], visitor, is_nullable=False)
- value_type = build_field_type_from_type_ids(type_resolver, field_name,
type_ids[2], visitor, is_nullable=False)
+ key_hint = None
+ value_hint = None
+ key_nullable = False
+ value_nullable = False
+ if type_hint is not None:
+ origin = typing.get_origin(type_hint) if hasattr(typing,
"get_origin") else getattr(type_hint, "__origin__", None)
+ if origin in (dict, typing.Dict):
+ args = typing.get_args(type_hint) if hasattr(typing,
"get_args") else getattr(type_hint, "__args__", ())
+ if len(args) >= 2:
+ key_hint, _ = unwrap_ref(args[0])
+ key_hint, key_nullable = unwrap_optional(key_hint)
+ value_hint, _ = unwrap_ref(args[1])
+ value_hint, value_nullable = unwrap_optional(value_hint)
+ key_type = build_field_type_from_type_ids(
+ type_resolver,
+ field_name,
+ type_ids[1],
+ visitor,
+ is_nullable=key_nullable,
+ type_hint=key_hint,
+ )
+ value_type = build_field_type_from_type_ids(
+ type_resolver,
+ field_name,
+ type_ids[2],
+ visitor,
+ is_nullable=value_nullable,
+ type_hint=value_hint,
+ )
return MapFieldType(type_id, morphic, is_nullable, tracking_ref,
key_type, value_type)
elif type_id in [
TypeId.UNKNOWN,
diff --git a/python/pyfory/struct.pxi b/python/pyfory/struct.pxi
index 7b4ee4259..129a438f0 100644
--- a/python/pyfory/struct.pxi
+++ b/python/pyfory/struct.pxi
@@ -30,8 +30,10 @@ cdef struct FieldRuntimeInfo:
uint8_t track_ref
uint8_t is_dynamic
uint8_t field_exists
+ uint8_t assign
PyObject *field_name
PyObject *serializer
+ PyObject *validation_field_type
@cython.final
@@ -42,6 +44,7 @@ cdef class DataClassSerializer(Serializer):
cdef public bint _has_slots
cdef public bint _fields_from_typedef
cdef public bint _has_missing_fields
+ cdef bint _has_validation_fields
cdef public list _field_names
cdef public list _serializers
cdef public dict _nullable_fields
@@ -53,8 +56,12 @@ cdef class DataClassSerializer(Serializer):
cdef public int32_t _hash
cdef public tuple _field_name_interned
cdef tuple _serializer_owner
+ cdef tuple _validation_field_type_owner
cdef public dict _default_values_factory
cdef tuple _missing_field_defaults
+ cdef public object _assign_fields
+ cdef public object _assigned_field_names
+ cdef public object _value_assignable_checker
cdef vector[FieldRuntimeInfo] _field_runtime_infos
def __init__(
@@ -66,6 +73,8 @@ cdef class DataClassSerializer(Serializer):
nullable_fields: dict = None,
dynamic_fields: dict = None,
ref_fields: dict = None,
+ field_infos: list = None,
+ fields_from_typedef: bint = False,
):
super().__init__(type_resolver, clz)
@@ -83,8 +92,16 @@ cdef class DataClassSerializer(Serializer):
self._type_hints = get_type_hints(clz)
self._has_slots = hasattr(clz, "__slots__")
- self._fields_from_typedef = field_names is not None and serializers is
not None
- if self._fields_from_typedef:
+ self._fields_from_typedef = fields_from_typedef or (field_names is not
None and serializers is not None)
+ if field_infos is not None:
+ self._field_infos = list(field_infos)
+ self._field_names = [fi.name for fi in self._field_infos]
+ self._serializers = [fi.serializer for fi in self._field_infos]
+ self._nullable_fields = {fi.name: fi.nullable for fi in
self._field_infos}
+ self._ref_fields = {fi.name: fi.runtime_ref_tracking for fi in
self._field_infos}
+ self._dynamic_fields = {fi.name: fi.dynamic for fi in
self._field_infos}
+ self._field_metas = {}
+ elif self._fields_from_typedef:
self._field_names = list(field_names)
self._serializers = list(serializers)
self._nullable_fields = dict(nullable_fields) if nullable_fields
is not None else {}
@@ -157,6 +174,13 @@ cdef class DataClassSerializer(Serializer):
self._build_fastpath_metadata()
self._build_missing_field_defaults()
+ if self._has_validation_fields:
+ from pyfory.meta.typedef import is_value_assignable
+
+ self._value_assignable_checker = is_value_assignable
+ else:
+ self._value_assignable_checker = None
+
cdef object _intern_field_name(self, str name):
cdef bytes encoded = name.encode("utf-8")
cdef const char *ptr = encoded
@@ -233,12 +257,22 @@ cdef class DataClassSerializer(Serializer):
cdef bint is_dynamic
cdef bint is_nullable
cdef bint is_tracking_ref
+ cdef bint assign
cdef FieldRuntimeInfo runtime_info
+ cdef list validation_field_types
+ cdef object validation_field_type
self._field_runtime_infos.clear()
self._has_missing_fields = False
+ self._has_validation_fields = False
current_fields = set(self._get_field_names(self.type_))
self._field_runtime_infos.reserve(len(self._field_names))
+ self._assign_fields = [
+ bool(getattr(self._field_infos[i], "assign", True)) if i <
len(self._field_infos) else True
+ for i in range(len(self._field_names))
+ ]
+ self._assigned_field_names = set()
+ validation_field_types = []
for i in range(len(self._field_names)):
field_name = self._field_names[i]
@@ -246,16 +280,30 @@ cdef class DataClassSerializer(Serializer):
is_nullable = bool(self._nullable_fields.get(field_name, False))
is_tracking_ref = bool(self._ref_fields.get(field_name, False))
is_dynamic = bool(self._dynamic_fields.get(field_name, False))
+ assign = bool(self._assign_fields[i])
+ validation_field_type = (
+ getattr(self._field_infos[i], "validation_field_type", None)
+ if i < len(self._field_infos)
+ else None
+ )
+ validation_field_types.append(validation_field_type)
+ if validation_field_type is not None:
+ self._has_validation_fields = True
runtime_info.basic_type_id =
self._resolve_basic_type_id(serializer, is_dynamic)
runtime_info.is_nullable = 1 if is_nullable else 0
runtime_info.track_ref = 1 if is_tracking_ref else 0
runtime_info.is_dynamic = 1 if is_dynamic else 0
runtime_info.field_exists = 1 if field_name in current_fields else 0
- if runtime_info.field_exists == 0:
+ runtime_info.assign = 1 if assign else 0
+ if runtime_info.field_exists == 0 or runtime_info.assign == 0:
self._has_missing_fields = True
+ if runtime_info.field_exists != 0 and runtime_info.assign != 0:
+ self._assigned_field_names.add(field_name)
runtime_info.field_name = <PyObject *>self._field_name_interned[i]
runtime_info.serializer = <PyObject *>serializer
+ runtime_info.validation_field_type = <PyObject
*>validation_field_type if validation_field_type is not None else NULL
self._field_runtime_infos.push_back(runtime_info)
+ self._validation_field_type_owner = tuple(validation_field_types)
cdef void _build_missing_field_defaults(self):
cdef object missing_fields
@@ -267,7 +315,7 @@ cdef class DataClassSerializer(Serializer):
if not self.type_resolver.compatible or not
self._default_values_factory:
return
- missing_fields = set(self._get_field_names(self.type_)) -
set(self._field_names)
+ missing_fields = set(self._get_field_names(self.type_)) -
self._assigned_field_names
if not missing_fields:
return
@@ -389,21 +437,45 @@ cdef class DataClassSerializer(Serializer):
cdef FieldRuntimeInfo *field_info
if not self._has_missing_fields:
+ if not self._has_validation_fields:
+ for i in range(field_count):
+ field_info = &self._field_runtime_infos[i]
+ field_value = self._read_field_value(read_context,
field_info)
+ field_name = <object>field_info.field_name
+ obj_dict[field_name] = field_value
+ return
for i in range(field_count):
field_info = &self._field_runtime_infos[i]
field_value = self._read_field_value(read_context, field_info)
field_name = <object>field_info.field_name
+ if field_info.validation_field_type == NULL:
+ obj_dict[field_name] = field_value
+ else:
+ obj_dict[field_name] =
self._validate_or_default(field_name, field_value, field_info)
+ return
+
+ if not self._has_validation_fields:
+ for i in range(field_count):
+ field_info = &self._field_runtime_infos[i]
+ if field_info.field_exists == 0 or field_info.assign == 0:
+ self._read_missing_field_value(read_context, field_info)
+ continue
+ field_value = self._read_field_value(read_context, field_info)
+ field_name = <object>field_info.field_name
obj_dict[field_name] = field_value
return
for i in range(field_count):
field_info = &self._field_runtime_infos[i]
- if field_info.field_exists == 0:
+ if field_info.field_exists == 0 or field_info.assign == 0:
self._read_missing_field_value(read_context, field_info)
continue
field_value = self._read_field_value(read_context, field_info)
field_name = <object>field_info.field_name
- obj_dict[field_name] = field_value
+ if field_info.validation_field_type == NULL:
+ obj_dict[field_name] = field_value
+ else:
+ obj_dict[field_name] = self._validate_or_default(field_name,
field_value, field_info)
cdef inline void _read_slots(self, ReadContext read_context, object obj):
cdef Py_ssize_t i
@@ -413,21 +485,53 @@ cdef class DataClassSerializer(Serializer):
cdef FieldRuntimeInfo *field_info
if not self._has_missing_fields:
+ if not self._has_validation_fields:
+ for i in range(field_count):
+ field_info = &self._field_runtime_infos[i]
+ field_value = self._read_field_value(read_context,
field_info)
+ field_name = <object>field_info.field_name
+ PyObject_SetAttr(obj, field_name, field_value)
+ return
for i in range(field_count):
field_info = &self._field_runtime_infos[i]
field_value = self._read_field_value(read_context, field_info)
field_name = <object>field_info.field_name
+ if field_info.validation_field_type == NULL:
+ PyObject_SetAttr(obj, field_name, field_value)
+ else:
+ PyObject_SetAttr(
+ obj,
+ field_name,
+ self._validate_or_default(field_name, field_value,
field_info),
+ )
+ return
+
+ if not self._has_validation_fields:
+ for i in range(field_count):
+ field_info = &self._field_runtime_infos[i]
+ if field_info.field_exists == 0 or field_info.assign == 0:
+ self._read_missing_field_value(read_context, field_info)
+ continue
+ field_value = self._read_field_value(read_context, field_info)
+ field_name = <object>field_info.field_name
PyObject_SetAttr(obj, field_name, field_value)
return
for i in range(field_count):
field_info = &self._field_runtime_infos[i]
- if field_info.field_exists == 0:
+ if field_info.field_exists == 0 or field_info.assign == 0:
self._read_missing_field_value(read_context, field_info)
continue
field_value = self._read_field_value(read_context, field_info)
field_name = <object>field_info.field_name
- PyObject_SetAttr(obj, field_name, field_value)
+ if field_info.validation_field_type == NULL:
+ PyObject_SetAttr(obj, field_name, field_value)
+ else:
+ PyObject_SetAttr(
+ obj,
+ field_name,
+ self._validate_or_default(field_name, field_value,
field_info),
+ )
cdef inline object _read_missing_field_value(self, ReadContext
read_context, FieldRuntimeInfo *field_info):
cdef object resolver = self.type_resolver.resolver
@@ -461,6 +565,18 @@ cdef class DataClassSerializer(Serializer):
return read_context.read_no_ref()
return read_context.read_non_ref(serializer)
+ cdef inline object _default_field_value(self, object field_name):
+ cdef object default_factory =
self._default_values_factory.get(field_name)
+ if default_factory is None:
+ return None
+ return default_factory()
+
+ cdef inline object _validate_or_default(self, object field_name, object
field_value, FieldRuntimeInfo *field_info):
+ if field_info.validation_field_type != NULL:
+ if not self._value_assignable_checker(field_value,
<object>field_info.validation_field_type):
+ return self._default_field_value(field_name)
+ return field_value
+
cdef inline void _apply_missing_defaults_dict(self, dict obj_dict):
cdef object field_name
cdef object default_factory
diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py
index 857a2e3a2..14572e36e 100644
--- a/python/pyfory/struct.py
+++ b/python/pyfory/struct.py
@@ -60,6 +60,7 @@ from pyfory.type_util import (
_get_args,
_get_origin,
infer_field,
+ infer_field_types,
get_homogeneous_tuple_elem_type,
is_subclass,
get_type_hints,
@@ -138,6 +139,9 @@ class FieldInfo:
type_id: int # Fory TypeId
serializer: Serializer # Field serializer
unwrapped_type: type # Type with Optional unwrapped
+ field_type: object = None # Recursive TypeDef schema node for this field
+ assign: bool = True # Whether a remote field value may be assigned locally
+ validation_field_type: object = None # Local schema used for
compatible-read validation
def _is_abstract_type(type_hint: type) -> bool:
@@ -234,7 +238,10 @@ def _extract_field_infos(
# Build FieldInfo list
field_infos: List[FieldInfo] = []
- visitor = StructFieldSerializerVisitor(fory)
+ from pyfory.meta.typedef import build_field_type_with_ref
+
+ visitor = StructTypeIdVisitor(fory, clz)
+ field_types = infer_field_types(clz, field_nullable=global_field_nullable)
global_ref_tracking = fory.track_ref
for index, (field_name, dc_field) in enumerate(active_fields):
@@ -248,21 +255,28 @@ def _extract_field_infos(
# Compute runtime ref tracking: field.ref AND global config
runtime_ref = meta.ref and global_ref_tracking
- # Infer serializer
- serializer = infer_field(field_name, unwrapped_type, visitor,
types_path=[])
+ field_type = build_field_type_with_ref(
+ fory,
+ field_name,
+ unwrapped_type,
+ visitor,
+ effective_nullable,
+ runtime_ref,
+ )
+ serializer = field_type.create_serializer(fory,
field_types.get(field_name, None))
- # Get type_id from serializer
- if serializer is not None:
- type_id = fory.get_type_info(serializer.type_).type_id
- else:
- type_id = TypeId.UNKNOWN
+ type_id = field_type.type_id
# Compute effective dynamic based on type.
# - Abstract classes: always True (type info must be written)
- # - If explicitly set (not None): use that value
+ # - Compatible ref-tracked struct fields carry type info for xlang
+ # parity with other runtimes
+ # - If explicitly set (not None): use that value for non-ref fields
# - Otherwise: write type info for polymorphic types that are not
registered by id
is_abstract = _is_abstract_type(unwrapped_type)
- if is_abstract:
+ if fory.compatible and runtime_ref and is_polymorphic_type(type_id):
+ effective_dynamic = True
+ elif is_abstract:
# Abstract classes always need type info
effective_dynamic = True
elif meta.dynamic is not None:
@@ -284,6 +298,7 @@ def _extract_field_infos(
type_id=type_id,
serializer=serializer,
unwrapped_type=unwrapped_type,
+ field_type=field_type,
)
field_infos.append(field_info)
@@ -367,14 +382,24 @@ class DataClassSerializer(Serializer):
nullable_fields: Dict[str, bool] = None,
dynamic_fields: Dict[str, bool] = None,
ref_fields: Dict[str, bool] = None,
+ field_infos: List[FieldInfo] = None,
+ fields_from_typedef: bool = False,
):
super().__init__(type_resolver, clz)
self._type_hints = get_type_hints(clz)
self._has_slots = hasattr(clz, "__slots__")
- self._fields_from_typedef = field_names is not None and serializers is
not None
- if self._fields_from_typedef:
+ self._fields_from_typedef = fields_from_typedef or (field_names is not
None and serializers is not None)
+ if field_infos is not None:
+ self._field_infos = list(field_infos)
+ self._field_names = [fi.name for fi in self._field_infos]
+ self._serializers = [fi.serializer for fi in self._field_infos]
+ self._nullable_fields = {fi.name: fi.nullable for fi in
self._field_infos}
+ self._ref_fields = {fi.name: fi.runtime_ref_tracking for fi in
self._field_infos}
+ self._dynamic_fields = {fi.name: fi.dynamic for fi in
self._field_infos}
+ self._field_metas = {}
+ elif self._fields_from_typedef:
self._field_names = list(field_names)
self._serializers = list(serializers)
self._nullable_fields = nullable_fields or {}
@@ -426,7 +451,23 @@ class DataClassSerializer(Serializer):
self._field_name_interned = {name: sys.intern(name) for name in
self._field_names}
self._current_class_field_names =
set(self._get_field_names(self.type_))
- self._has_missing_fields = any(field_name not in
self._current_class_field_names for field_name in self._field_names)
+ self._assign_fields = [
+ bool(getattr(self._field_infos[index], "assign", True)) if index <
len(self._field_infos) else True
+ for index in range(len(self._field_names))
+ ]
+ self._validation_field_types = [
+ getattr(self._field_infos[index], "validation_field_type", None)
if index < len(self._field_infos) else None
+ for index in range(len(self._field_names))
+ ]
+ self._has_validation_fields = any(field_type is not None for
field_type in self._validation_field_types)
+ self._assigned_field_names = {
+ field_name
+ for index, field_name in enumerate(self._field_names)
+ if self._assign_fields[index] and field_name in
self._current_class_field_names
+ }
+ self._has_missing_fields = any(
+ field_name not in self._current_class_field_names or not
self._assign_fields[index] for index, field_name in enumerate(self._field_names)
+ )
self._default_values_factory = (
build_default_values_factory(self.type_resolver, self._type_hints,
dataclasses.fields(self.type_))
if dataclasses.is_dataclass(self.type_)
@@ -456,7 +497,7 @@ class DataClassSerializer(Serializer):
def _build_missing_field_defaults(self):
if not self.type_resolver.compatible or not
self._default_values_factory:
return []
- missing_fields = self._current_class_field_names -
set(self._field_names)
+ missing_fields = self._current_class_field_names -
self._assigned_field_names
if not missing_fields:
return []
return [(field_name, default_factory) for field_name, default_factory
in self._default_values_factory.items() if field_name in missing_fields]
@@ -500,6 +541,24 @@ class DataClassSerializer(Serializer):
return read_context.read_no_ref()
return read_context.read_no_ref(serializer=serializer)
+ def _default_field_value(self, field_name):
+ default_factory = self._default_values_factory.get(field_name)
+ if default_factory is None:
+ return None
+ return default_factory()
+
+ def _assign_read_field_value(self, obj, obj_dict, field_name, field_value,
validation_field_type):
+ if validation_field_type is not None:
+ from pyfory.meta.typedef import is_value_assignable
+
+ if not is_value_assignable(field_value, validation_field_type):
+ field_value = self._default_field_value(field_name)
+ interned_name = self._field_name_interned[field_name]
+ if obj_dict is not None:
+ obj_dict[interned_name] = field_value
+ else:
+ setattr(obj, interned_name, field_value)
+
def write(self, write_context: Buffer, value):
if not self.type_resolver.compatible:
write_context.write_int32(self._hash)
@@ -567,28 +626,50 @@ class DataClassSerializer(Serializer):
is_dynamic = self._dynamic_fields.get(field_name, False)
is_tracking_ref = self._ref_fields.get(field_name, False)
is_basic = self._basic_field_flags[index]
- if field_name not in self._current_class_field_names:
+ if field_name not in self._current_class_field_names or not
self._assign_fields[index]:
self._read_missing_field_value(read_context, serializer,
is_nullable, is_dynamic, is_basic, is_tracking_ref)
continue
field_value = self._read_field_value(read_context, serializer,
is_nullable, is_dynamic, is_basic, is_tracking_ref)
- interned_name = self._field_name_interned[field_name]
- if obj_dict is not None:
- obj_dict[interned_name] = field_value
+ validation_field_type = self._validation_field_types[index]
+ if validation_field_type is None:
+ interned_name = self._field_name_interned[field_name]
+ if obj_dict is not None:
+ obj_dict[interned_name] = field_value
+ else:
+ setattr(obj, interned_name, field_value)
else:
- setattr(obj, interned_name, field_value)
+ self._assign_read_field_value(obj, obj_dict, field_name,
field_value, validation_field_type)
else:
- for index, field_name in enumerate(self._field_names):
- serializer = self._serializers[index]
- is_nullable = self._nullable_fields.get(field_name, False)
- is_dynamic = self._dynamic_fields.get(field_name, False)
- is_tracking_ref = self._ref_fields.get(field_name, False)
- is_basic = self._basic_field_flags[index]
- field_value = self._read_field_value(read_context, serializer,
is_nullable, is_dynamic, is_basic, is_tracking_ref)
- interned_name = self._field_name_interned[field_name]
- if obj_dict is not None:
- obj_dict[interned_name] = field_value
- else:
- setattr(obj, interned_name, field_value)
+ if not self._has_validation_fields:
+ for index, field_name in enumerate(self._field_names):
+ serializer = self._serializers[index]
+ is_nullable = self._nullable_fields.get(field_name, False)
+ is_dynamic = self._dynamic_fields.get(field_name, False)
+ is_tracking_ref = self._ref_fields.get(field_name, False)
+ is_basic = self._basic_field_flags[index]
+ field_value = self._read_field_value(read_context,
serializer, is_nullable, is_dynamic, is_basic, is_tracking_ref)
+ interned_name = self._field_name_interned[field_name]
+ if obj_dict is not None:
+ obj_dict[interned_name] = field_value
+ else:
+ setattr(obj, interned_name, field_value)
+ else:
+ for index, field_name in enumerate(self._field_names):
+ serializer = self._serializers[index]
+ is_nullable = self._nullable_fields.get(field_name, False)
+ is_dynamic = self._dynamic_fields.get(field_name, False)
+ is_tracking_ref = self._ref_fields.get(field_name, False)
+ is_basic = self._basic_field_flags[index]
+ field_value = self._read_field_value(read_context,
serializer, is_nullable, is_dynamic, is_basic, is_tracking_ref)
+ validation_field_type = self._validation_field_types[index]
+ if validation_field_type is None:
+ interned_name = self._field_name_interned[field_name]
+ if obj_dict is not None:
+ obj_dict[interned_name] = field_value
+ else:
+ setattr(obj, interned_name, field_value)
+ else:
+ self._assign_read_field_value(obj, obj_dict,
field_name, field_value, validation_field_type)
if self._missing_field_defaults:
for field_name, default_factory in self._missing_field_defaults:
diff --git a/python/pyfory/tests/test_struct.py
b/python/pyfory/tests/test_struct.py
index 23e24e7f9..699756f34 100644
--- a/python/pyfory/tests/test_struct.py
+++ b/python/pyfory/tests/test_struct.py
@@ -36,6 +36,14 @@ def ser_de(fory, obj):
return fory.deserialize(binary)
+def compat_ser_de(remote_cls, local_cls, value, type_id):
+ writer = Fory(xlang=True, compatible=True, ref=False)
+ reader = Fory(xlang=True, compatible=True, ref=False)
+ writer.register_type(remote_cls, type_id=type_id)
+ reader.register_type(local_cls, type_id=type_id)
+ return reader.deserialize(writer.serialize(value))
+
+
@dataclass
class SimpleObject:
f1: Optional[Dict[pyfory.int32, pyfory.float64]] = None
@@ -87,6 +95,97 @@ def test_struct():
assert ser_de(fory, ComplexObject(f6=2**64)) == ComplexObject(f6=2**64)
+@dataclass
+class NestedDeclaredEncodingObject:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] =
dataclasses.field(default_factory=dict)
+ flags: Set[pyfory.fixed_uint32] = dataclasses.field(default_factory=set)
+
+
+def test_nested_declared_field_type_drives_local_roundtrip():
+ fory = Fory(xlang=True, compatible=True, ref=False)
+ fory.register_type(NestedDeclaredEncodingObject, type_id=701)
+ serializer =
fory.type_resolver.get_serializer(NestedDeclaredEncodingObject)
+ field_infos = {field_info.name: field_info for field_info in
serializer._field_infos}
+
+ values_type = field_infos["values"].field_type
+ assert values_type.type_id == TypeId.MAP
+ assert values_type.key_type.type_id == TypeId.INT32
+ assert values_type.value_type.type_id == TypeId.LIST
+ assert values_type.value_type.element_type.type_id == TypeId.TAGGED_INT64
+ assert field_infos["flags"].field_type.element_type.type_id ==
TypeId.UINT32
+
+ value = NestedDeclaredEncodingObject(values={1: [2, -3], -4: [5]},
flags={1, 2**32 - 1})
+ assert ser_de(fory, value) == value
+
+
+@dataclass
+class RemoteNestedFixedTagged:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] =
dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class LocalNestedVarint:
+ values: Dict[pyfory.int32, List[pyfory.int64]] =
dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class RemoteNestedWide:
+ values: Dict[pyfory.fixed_int64, List[pyfory.tagged_int64]] =
dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class LocalNestedNarrow:
+ values: Dict[pyfory.fixed_int32, List[pyfory.int32]] =
dataclasses.field(default_factory=lambda: {7: [8]})
+
+
+@dataclass
+class RemoteNestedUnsigned:
+ values: Dict[pyfory.fixed_uint32, List[pyfory.tagged_uint64]] =
dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class LocalNestedSignedDefault:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] =
dataclasses.field(default_factory=lambda: {-1: [-1]})
+
+
+def test_compatible_read_accepts_nested_same_domain_integer_encoding():
+ result = compat_ser_de(
+ RemoteNestedFixedTagged,
+ LocalNestedVarint,
+ RemoteNestedFixedTagged(values={1: [2, -3], -4: [5]}),
+ 702,
+ )
+ assert result == LocalNestedVarint(values={1: [2, -3], -4: [5]})
+
+
+def test_compatible_read_validates_nested_integer_narrowing():
+ result = compat_ser_de(
+ RemoteNestedWide,
+ LocalNestedNarrow,
+ RemoteNestedWide(values={1: [2, -3]}),
+ 703,
+ )
+ assert result == LocalNestedNarrow(values={1: [2, -3]})
+
+ result = compat_ser_de(
+ RemoteNestedWide,
+ LocalNestedNarrow,
+ RemoteNestedWide(values={1: [1 << 40]}),
+ 704,
+ )
+ assert result == LocalNestedNarrow()
+
+
+def test_compatible_read_skips_nested_signed_unsigned_mismatch():
+ result = compat_ser_de(
+ RemoteNestedUnsigned,
+ LocalNestedSignedDefault,
+ RemoteNestedUnsigned(values={1: [2]}),
+ 705,
+ )
+ assert result == LocalNestedSignedDefault()
+
+
@dataclass
class SuperClass1:
f1: Optional[Any] = None
diff --git a/python/pyfory/tests/test_typedef_encoding.py
b/python/pyfory/tests/test_typedef_encoding.py
index 7496b9f9b..6a68c4849 100644
--- a/python/pyfory/tests/test_typedef_encoding.py
+++ b/python/pyfory/tests/test_typedef_encoding.py
@@ -21,6 +21,7 @@ Tests for xlang TypeDef implementation.
from dataclasses import dataclass
from typing import List, Dict
+import pyfory
from pyfory.serialization import Buffer
from pyfory.meta.typedef import (
TypeDef,
@@ -53,6 +54,13 @@ class SimpleTypeDef:
value: int
+@dataclass
+class NestedEncodingTypeDef:
+ """TypeDef with nested primitive encoding overrides."""
+
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]
+
+
def test_collection_field_type():
"""Test collection field type creation and serialization."""
element_type = FieldType(TypeId.INT32, True, True, False)
@@ -146,6 +154,25 @@ def test_encode_decode_typedef():
assert field.field_type.is_nullable ==
typedef.fields[i].field_type.is_nullable
+def test_nested_container_typedef_preserves_declared_encoding():
+ fory = Fory(xlang=True)
+ fory.register(NestedEncodingTypeDef, namespace="example",
typename="NestedEncodingTypeDef")
+
+ typedef = encode_typedef(fory.type_resolver, NestedEncodingTypeDef)
+ values_field = next(field for field in typedef.fields if field.name ==
"values")
+ assert values_field.field_type.type_id == TypeId.MAP
+ assert values_field.field_type.key_type.type_id == TypeId.INT32
+ assert values_field.field_type.value_type.type_id == TypeId.LIST
+ assert values_field.field_type.value_type.element_type.type_id ==
TypeId.TAGGED_INT64
+
+ decoded_typedef = decode_typedef(Buffer(typedef.encoded),
fory.type_resolver)
+ decoded_values_field = next(field for field in decoded_typedef.fields if
field.name == "values")
+ assert decoded_values_field.field_type.type_id == TypeId.MAP
+ assert decoded_values_field.field_type.key_type.type_id == TypeId.INT32
+ assert decoded_values_field.field_type.value_type.type_id == TypeId.LIST
+ assert decoded_values_field.field_type.value_type.element_type.type_id ==
TypeId.TAGGED_INT64
+
+
if __name__ == "__main__":
test_collection_field_type()
test_map_field_type()
diff --git a/python/pyfory/tests/xlang_test_main.py
b/python/pyfory/tests/xlang_test_main.py
index 8ada5dea6..2a1afed53 100644
--- a/python/pyfory/tests/xlang_test_main.py
+++ b/python/pyfory/tests/xlang_test_main.py
@@ -230,6 +230,26 @@ class RefOverrideContainer:
map_field: Dict[str, Ref[RefOverrideElement]] = None
+@dataclass
+class SignedNestedAnnotatedContainerSchemaConsistent:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] = None
+
+
+@dataclass
+class SignedNestedAnnotatedContainerCompatible:
+ values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] = None
+
+
+@dataclass
+class NestedAnnotatedContainerSchemaConsistent:
+ values: Dict[pyfory.fixed_uint32, List[pyfory.tagged_uint64]] = None
+
+
+@dataclass
+class NestedAnnotatedContainerCompatible:
+ values: Dict[pyfory.fixed_uint32, List[pyfory.tagged_uint64]] = None
+
+
# ============================================================================
# Nullable Field Test Types
# ============================================================================
@@ -1424,6 +1444,61 @@ def test_collection_element_ref_remote_tracking():
f.write(fory.serialize(outer))
+def _test_nested_annotated_container(cls, type_id: int, expected, compatible:
bool):
+ data_file = get_data_file()
+ with open(data_file, "rb") as f:
+ data_bytes = f.read()
+
+ kwargs = {"xlang": True, "compatible": compatible}
+ if compatible:
+ kwargs["meta_compressor"] = NoOpMetaCompressor()
+ fory = pyfory.Fory(**kwargs)
+ fory.register_type(cls, type_id=type_id)
+
+ obj = fory.deserialize(data_bytes)
+ debug_print(f"Deserialized nested annotated container: {obj}")
+ assert obj == expected, f"Mismatch: {obj} != {expected}"
+
+ with open(data_file, "wb") as f:
+ f.write(fory.serialize(obj))
+
+
+def test_signed_nested_annotated_container_schema_consistent():
+ _test_nested_annotated_container(
+ SignedNestedAnnotatedContainerSchemaConsistent,
+ 803,
+ SignedNestedAnnotatedContainerSchemaConsistent(values={-7: [7,
-1000000000], 3: [42]}),
+ compatible=False,
+ )
+
+
+def test_signed_nested_annotated_container_compatible():
+ _test_nested_annotated_container(
+ SignedNestedAnnotatedContainerCompatible,
+ 804,
+ SignedNestedAnnotatedContainerCompatible(values={-7: [7, -1000000000],
3: [42]}),
+ compatible=True,
+ )
+
+
+def test_nested_annotated_container_schema_consistent():
+ _test_nested_annotated_container(
+ NestedAnnotatedContainerSchemaConsistent,
+ 801,
+ NestedAnnotatedContainerSchemaConsistent(values={4000000000: [7,
1000000000], 3: [42]}),
+ compatible=False,
+ )
+
+
+def test_nested_annotated_container_compatible():
+ _test_nested_annotated_container(
+ NestedAnnotatedContainerCompatible,
+ 802,
+ NestedAnnotatedContainerCompatible(values={4000000000: [7,
1000000000], 3: [42]}),
+ compatible=True,
+ )
+
+
# ============================================================================
# Circular Reference Test Types
# ============================================================================
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
