This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/fory-site.git
commit 6bb384cb921ff581885cfc07063e92ec8986ec5a Author: chaokunyang <[email protected]> AuthorDate: Mon May 4 13:02:28 2026 +0000 ๐ synced local 'docs/guide/' with remote 'docs/guide/' --- docs/guide/cpp/cross-language.md | 30 ++++++- docs/guide/cpp/polymorphism.md | 5 +- docs/guide/cpp/row-format.md | 12 +-- docs/guide/cpp/supported-types.md | 2 +- docs/guide/csharp/cross-language.md | 22 +++++ docs/guide/csharp/field-configuration.md | 13 +-- docs/guide/csharp/supported-types.md | 4 +- docs/guide/dart/cross-language.md | 22 +++++ docs/guide/dart/supported-types.md | 11 +-- docs/guide/dart/web-platform-support.md | 4 +- docs/guide/go/cross-language.md | 29 +++++- docs/guide/go/struct-tags.md | 16 ++-- docs/guide/go/supported-types.md | 9 +- docs/guide/java/cross-language.md | 35 +++++++- docs/guide/java/field-configuration.md | 71 ++++++--------- docs/guide/javascript/basic-serialization.md | 2 +- docs/guide/javascript/cross-language.md | 23 +++++ docs/guide/javascript/supported-types.md | 36 ++++---- docs/guide/python/cross-language.md | 109 ++++++++++++++-------- docs/guide/python/field-configuration.md | 130 +++++++++++++-------------- docs/guide/python/row-format.md | 38 ++++++-- docs/guide/rust/basic-serialization.md | 2 +- docs/guide/rust/cross-language.md | 32 +++++-- docs/guide/swift/cross-language.md | 23 +++++ docs/guide/xlang/field-nullability.md | 2 +- docs/guide/xlang/getting-started.md | 4 +- docs/guide/xlang/index.md | 2 +- docs/guide/xlang/serialization.md | 28 +++--- docs/guide/xlang/troubleshooting.md | 8 +- 29 files changed, 482 insertions(+), 242 deletions(-) diff --git a/docs/guide/cpp/cross-language.md b/docs/guide/cpp/cross-language.md index f9540c5e94..b4d05f4757 100644 --- a/docs/guide/cpp/cross-language.md +++ b/docs/guide/cpp/cross-language.md @@ -145,8 +145,8 @@ print(f"Timestamp: {msg.timestamp}") | `int64_t` | `long` | `int` | `int64` | `i64` | | `float` | `float` | `float` | `float32` | `f32` | | `double` | `double` | `float` | `float64` | `f64` | -| `fory::float16_t` | `Float16` | `pyfory.float16` | `float16.Float16` | `Float16` | -| `fory::bfloat16_t` | `BFloat16` | `pyfory.bfloat16` | `bfloat16.BFloat16` | `BFloat16` | +| `fory::float16_t` | `Float16` | `pyfory.Float16` | `float16.Float16` | `Float16` | +| `fory::bfloat16_t` | `BFloat16` | `pyfory.BFloat16` | `bfloat16.BFloat16` | `BFloat16` | ### String Types @@ -159,11 +159,33 @@ print(f"Timestamp: {msg.timestamp}") | C++ Type | Java Type | Python Type | Go Type | Rust Type | | ------------------------------- | -------------- | --------------- | --------------------- | --------------- | | `std::vector<T>` | `List<T>` | `list` | `[]T` | `Vec<T>` | -| `std::vector<fory::float16_t>` | `Float16List` | `float16array` | `[]float16.Float16` | `Vec<Float16>` | -| `std::vector<fory::bfloat16_t>` | `BFloat16List` | `bfloat16array` | `[]bfloat16.BFloat16` | `Vec<BFloat16>` | +| `std::vector<fory::float16_t>` | `Float16List` | `Float16Array` | `[]float16.Float16` | `Vec<Float16>` | +| `std::vector<fory::bfloat16_t>` | `BFloat16List` | `BFloat16Array` | `[]bfloat16.BFloat16` | `Vec<BFloat16>` | | `std::set<T>` | `Set<T>` | `set` | `map[T]struct{}` | `HashSet<T>` | | `std::map<K,V>` | `Map<K,V>` | `dict` | `map[K]V` | `HashMap<K,V>` | +### Lists and Dense Arrays + +`std::vector<T>` maps to Fory `list<T>` by default in handwritten C++ structs. +Use the field metadata DSL's array node when the schema is dense `array<T>`. + +| Fory schema | C++ metadata sketch | +| ----------------- | ---------------------------------------- | +| `list<int32>` | `fory::F(id).list(fory::T::int32())` | +| `array<bool>` | `fory::F(id).array(fory::T::bool_())` | +| `array<int8>` | `fory::F(id).array(fory::T::int8())` | +| `array<int16>` | `fory::F(id).array(fory::T::int16())` | +| `array<int32>` | `fory::F(id).array(fory::T::int32())` | +| `array<int64>` | `fory::F(id).array(fory::T::int64())` | +| `array<uint8>` | `fory::F(id).array(fory::T::uint8())` | +| `array<uint16>` | `fory::F(id).array(fory::T::uint16())` | +| `array<uint32>` | `fory::F(id).array(fory::T::uint32())` | +| `array<uint64>` | `fory::F(id).array(fory::T::uint64())` | +| `array<float16>` | `fory::F(id).array(fory::T::float16())` | +| `array<bfloat16>` | `fory::F(id).array(fory::T::bfloat16())` | +| `array<float32>` | `fory::F(id).array(fory::T::float32())` | +| `array<float64>` | `fory::F(id).array(fory::T::float64())` | + ### Temporal Types | C++ Type | Java Type | Python Type | Go Type | diff --git a/docs/guide/cpp/polymorphism.md b/docs/guide/cpp/polymorphism.md index 4465553eff..f72ebab1ec 100644 --- a/docs/guide/cpp/polymorphism.md +++ b/docs/guide/cpp/polymorphism.md @@ -89,8 +89,9 @@ int main() { assert(decoded.star_animal->age == 3); auto* dog_ptr = dynamic_cast<Dog*>(decoded.star_animal.get()); - assert(dog_ptr != nullptr); - assert(dog_ptr->breed == "Labrador"); + if (dog_ptr == nullptr || dog_ptr->breed != "Labrador") { + return 1; + } } ``` diff --git a/docs/guide/cpp/row-format.md b/docs/guide/cpp/row-format.md index 0ae72f6cb9..d1551ab5f5 100644 --- a/docs/guide/cpp/row-format.md +++ b/docs/guide/cpp/row-format.md @@ -70,13 +70,9 @@ int main() { auto row = encoder.get_writer().to_row(); // Random access to fields - int32_t id = row->get_int32(0); - std::string name = row->get_string(1); - float score = row->get_float(2); - - assert(id == 1); - assert(name == "Alice"); - assert(score == 95.5f); + assert(row->get_int32(0) == 1); + assert(row->get_string(1) == "Alice"); + assert(row->get_float(2) == 95.5f); return 0; } @@ -222,7 +218,7 @@ public: ### ArrayData Class -The `ArrayData` class provides access to list/array elements: +The `ArrayData` class provides access to nested sequence elements: ```cpp class ArrayData { diff --git a/docs/guide/cpp/supported-types.md b/docs/guide/cpp/supported-types.md index c2ca398e09..49917d608e 100644 --- a/docs/guide/cpp/supported-types.md +++ b/docs/guide/cpp/supported-types.md @@ -72,7 +72,7 @@ assert(text == decoded); `std::vector<T>` for any serializable element type: -`std::vector<fory::bfloat16_t>` is the dense array carrier for xlang `bfloat16_array`. +`std::vector<fory::bfloat16_t>` can be used as the dense carrier when the field metadata declares `array<bfloat16>` schema. ```cpp std::vector<int32_t> numbers{1, 2, 3, 4, 5}; diff --git a/docs/guide/csharp/cross-language.md b/docs/guide/csharp/cross-language.md index 2520c7dea9..c712ed0c32 100644 --- a/docs/guide/csharp/cross-language.md +++ b/docs/guide/csharp/cross-language.md @@ -95,6 +95,28 @@ See [xlang guide](../xlang/index.md) for complete mapping. For reduced-precision numeric payloads, use `Half` / `Half[]` or `List<Half>` for xlang `float16`, and `BFloat16` / `BFloat16[]` or `List<BFloat16>` for xlang `bfloat16`. +## Lists and Dense Arrays + +C# `List<T>` maps to Fory `list<T>`. Use the schema marker +`Apache.Fory.Schema.Types.Array<T>` when a field is dense `array<T>`. + +| Fory schema | C# schema marker sketch | +| ----------------- | ----------------------- | +| `list<int32>` | `S.List<S.Int32>` | +| `array<bool>` | `S.Array<S.Bool>` | +| `array<int8>` | `S.Array<S.Int8>` | +| `array<int16>` | `S.Array<S.Int16>` | +| `array<int32>` | `S.Array<S.Int32>` | +| `array<int64>` | `S.Array<S.Int64>` | +| `array<uint8>` | `S.Array<S.UInt8>` | +| `array<uint16>` | `S.Array<S.UInt16>` | +| `array<uint32>` | `S.Array<S.UInt32>` | +| `array<uint64>` | `S.Array<S.UInt64>` | +| `array<float16>` | `S.Array<S.Float16>` | +| `array<bfloat16>` | `S.Array<S.BFloat16>` | +| `array<float32>` | `S.Array<S.Float32>` | +| `array<float64>` | `S.Array<S.Float64>` | + ## Best Practices 1. Keep type IDs stable and documented. diff --git a/docs/guide/csharp/field-configuration.md b/docs/guide/csharp/field-configuration.md index 0e53eacf27..c68eeac4a1 100644 --- a/docs/guide/csharp/field-configuration.md +++ b/docs/guide/csharp/field-configuration.md @@ -35,7 +35,7 @@ public sealed class Metrics [ForyField(Type = typeof(S.UInt32))] public uint Count { get; set; } - [ForyField(Type = typeof(S.TaggedUInt64))] + [ForyField(Type = typeof(S.Tagged<S.UInt64>))] public ulong TraceId { get; set; } public long LatencyMicros { get; set; } @@ -51,7 +51,7 @@ using S = Apache.Fory.Schema.Types; [ForyObject] public sealed class NestedMetrics { - [ForyField(Type = typeof(S.Map<S.UInt32, S.List<S.TaggedUInt64>>))] + [ForyField(Type = typeof(S.Map<S.Fixed<S.UInt32>, S.List<S.Tagged<S.UInt64>>>))] public Dictionary<uint, List<ulong?>?> Values { get; set; } = []; [ForyField(3, Type = typeof(S.UInt64))] @@ -65,18 +65,19 @@ Schema descriptors live under `Apache.Fory.Schema.Types` and are metadata only. Common scalar descriptors include: -- `S.Int32`, `S.VarInt32`, `S.UInt32`, `S.VarUInt32` -- `S.Int64`, `S.VarInt64`, `S.TaggedInt64` -- `S.UInt64`, `S.VarUInt64`, `S.TaggedUInt64` +- `S.Int32`, `S.UInt32` +- `S.Int64`, `S.UInt64` - `S.Float16`, `S.BFloat16`, `S.Float32`, `S.Float64` Container descriptors are composable: +- `S.Fixed<TScalar>` and `S.Tagged<TScalar>` for scalar integer encodings - `S.List<TElement>` - `S.Set<TElement>` - `S.Map<TKey, TValue>` +- `S.Array<TElement>` -Packed array descriptors such as `S.Int32Array`, `S.UInt32Array`, `S.Float16Array`, and `S.BFloat16Array` are available when the field should use the packed array wire type. +Dense array fields use `S.Array<TElement>`, for example `S.Array<S.Int32>` or `S.Array<S.BFloat16>`. Nullability comes from the C# carrier type. Use `List<ulong?>` for nullable list elements and `NullableKeyDictionary<TKey, TValue>` when a map needs nullable keys. diff --git a/docs/guide/csharp/supported-types.md b/docs/guide/csharp/supported-types.md index e1a92ac0f4..da5d4ced40 100644 --- a/docs/guide/csharp/supported-types.md +++ b/docs/guide/csharp/supported-types.md @@ -37,8 +37,8 @@ This page summarizes built-in and generated type support in Apache Foryโข C#. ## Arrays - Primitive numeric arrays (`bool[]`, `int[]`, `ulong[]`, etc.) -- `Half[]`, `List<Half>` for `float16_array` -- `BFloat16[]`, `List<BFloat16>` for `bfloat16_array` +- `Half[]`, `List<Half>` with `S.Array<S.Float16>` for `array<float16>` +- `BFloat16[]`, `List<BFloat16>` with `S.Array<S.BFloat16>` for `array<bfloat16>` - `byte[]` - General arrays (`T[]`) through collection serializers diff --git a/docs/guide/dart/cross-language.md b/docs/guide/dart/cross-language.md index f5d7b5476c..843e0cf38d 100644 --- a/docs/guide/dart/cross-language.md +++ b/docs/guide/dart/cross-language.md @@ -177,6 +177,28 @@ Because Dart `int` is not itself a promise about the exact xlang wire width, pre - `Float16List` and `Bfloat16List` for 16-bit floating-point array payloads - `Timestamp`, `LocalDate`, and `Duration` for explicit temporal semantics +### Lists and Dense Arrays + +`List<T>` always represents Fory `list<T>` unless a field has explicit array +metadata. Use `array<T>` only for dense one-dimensional bool or numeric data. + +| Fory schema | Dart field carrier and annotation | +| ----------------- | --------------------------------------------------- | +| `list<bool>` | `List<bool>` | +| `array<bool>` | `@ArrayField(element: BoolType()) BoolList` | +| `array<int8>` | `@ArrayField(element: Int8Type()) Int8List` | +| `array<int16>` | `@ArrayField(element: Int16Type()) Int16List` | +| `array<int32>` | `@ArrayField(element: Int32Type()) Int32List` | +| `array<int64>` | `@ArrayField(element: Int64Type()) Int64List` | +| `array<uint8>` | `@ArrayField(element: Uint8Type()) Uint8List` | +| `array<uint16>` | `@ArrayField(element: Uint16Type()) Uint16List` | +| `array<uint32>` | `@ArrayField(element: Uint32Type()) Uint32List` | +| `array<uint64>` | `@ArrayField(element: Uint64Type()) Uint64List` | +| `array<float16>` | `@ArrayField(element: Float16Type()) Float16List` | +| `array<bfloat16>` | `@ArrayField(element: Bfloat16Type()) Bfloat16List` | +| `array<float32>` | `@ArrayField(element: Float32Type()) Float32List` | +| `array<float64>` | `@ArrayField(element: Float64Type()) Float64List` | + See [Supported Types](supported-types.md) and [xlang type mapping](../../specification/xlang_type_mapping.md). ## Validation diff --git a/docs/guide/dart/supported-types.md b/docs/guide/dart/supported-types.md index 4b43d10e85..186de6938d 100644 --- a/docs/guide/dart/supported-types.md +++ b/docs/guide/dart/supported-types.md @@ -86,8 +86,8 @@ floating-point values, use an explicit wrapper: - `Bfloat16` โ brain floating point, useful when interoperating with ML-oriented payloads For contiguous 16-bit floating-point arrays, use `Float16List` and -`Bfloat16List` rather than `Uint16List` when the wire type is `float16_array` -or `bfloat16_array`. +`Bfloat16List` rather than `Uint16List` when the schema is `array<float16>` +or `array<bfloat16>`. ## Time and Date Types @@ -136,10 +136,11 @@ See [Code Generation](code-generation.md). Fory supports `List<T>`, `Set<T>`, and `Map<K, V>`. Element and key types must also be serializable types. Avoid using mutable objects as map keys. -Generic `List<int>` with primitive element metadata still uses the `list` wire -type. Dedicated array wire kinds come from dedicated carriers: +Generic `List<int>` with primitive element metadata still uses `list<T>` schema. +Dedicated dense array schema comes from dedicated carriers: -- `List<bool>` for `bool_array` +- `BoolList` plus `@ArrayField(element: BoolType())` for `array<bool>`. + Plain `List<bool>` maps to `list<bool>`. - `Int8List`, `Int16List`, `Int32List`, `Int64List` - `Uint8List`, `Uint16List`, `Uint32List`, `Uint64List` - `Float16List`, `Bfloat16List`, `Float32List`, `Float64List` diff --git a/docs/guide/dart/web-platform-support.md b/docs/guide/dart/web-platform-support.md index 4cb63da47f..b1105e9810 100644 --- a/docs/guide/dart/web-platform-support.md +++ b/docs/guide/dart/web-platform-support.md @@ -172,8 +172,8 @@ final class OffsetSerializer extends Serializer<StorageExtent> { `List`, `Set`, `Map`, `Uint8List`, numeric typed arrays, `Int64List`, and `Uint64List` are supported on web. The `Int64List` and `Uint64List` implementations preserve 64-bit values without depending on JavaScript integer -precision. Use the Fory wrapper list types when the wire type is `int64_array` -or `uint64_array`. +precision. Use the Fory wrapper list types when the schema is `array<int64>` +or `array<uint64>`. ## Testing Browser Builds diff --git a/docs/guide/go/cross-language.md b/docs/guide/go/cross-language.md index ecf44025fc..44d2e56543 100644 --- a/docs/guide/go/cross-language.md +++ b/docs/guide/go/cross-language.md @@ -66,7 +66,7 @@ import pyfory @dataclass class User: - id: pyfory.Int64Type + id: pyfory.Int64 name: str fory = pyfory.Fory(xlang=True) @@ -157,9 +157,9 @@ import pyfory @dataclass class Message: - id: pyfory.Int64Type + id: pyfory.Int64 content: str - timestamp: pyfory.Int64Type + timestamp: pyfory.Int64 fory = pyfory.Fory(xlang=True) fory.register(Message, type_id=1) @@ -189,6 +189,29 @@ fmt.Println(msg.Content) // "Hello from Python" Cross-language nested structures require all types to be registered: +## Lists and Dense Arrays + +Go slices are ordinary `list<T>` carriers unless a field tag explicitly requests +the dense `array<T>` schema. Use `array<T>` only for one-dimensional bool or +numeric data. + +| Fory schema | Go carrier and tag sketch | +| ----------------- | ------------------------------------------------------ | +| `list<int32>` | `[]int32` / `fory:"type=list(element=int32)"` | +| `array<bool>` | `[]bool` / `fory:"type=array(element=bool)"` | +| `array<int8>` | `[]int8` / `fory:"type=array(element=int8)"` | +| `array<int16>` | `[]int16` / `fory:"type=array(element=int16)"` | +| `array<int32>` | `[]int32` / `fory:"type=array(element=int32)"` | +| `array<int64>` | `[]int64` / `fory:"type=array(element=int64)"` | +| `array<uint8>` | `[]uint8` / `fory:"type=array(element=uint8)"` | +| `array<uint16>` | `[]uint16` / `fory:"type=array(element=uint16)"` | +| `array<uint32>` | `[]uint32` / `fory:"type=array(element=uint32)"` | +| `array<uint64>` | `[]uint64` / `fory:"type=array(element=uint64)"` | +| `array<float16>` | `[]float16.Float16` / `type=array(element=float16)` | +| `array<bfloat16>` | `[]bfloat16.BFloat16` / `type=array(element=bfloat16)` | +| `array<float32>` | `[]float32` / `fory:"type=array(element=float32)"` | +| `array<float64>` | `[]float64` / `fory:"type=array(element=float64)"` | + **Go**: ```go diff --git a/docs/guide/go/struct-tags.md b/docs/guide/go/struct-tags.md index b973769eb2..00c8dc7826 100644 --- a/docs/guide/go/struct-tags.md +++ b/docs/guide/go/struct-tags.md @@ -158,7 +158,7 @@ Use `type=` to override inferred carrier semantics or nested value encoding: ```go type Foo struct { - // Force general LIST protocol instead of inferred packed INT32_ARRAY + // Force general list protocol. Values []int32 `fory:"type=list"` // Override inner integer encoding for a general list @@ -167,17 +167,19 @@ type Foo struct { // Override nested map/list integer encoding Nested map[string][]*uint64 `fory:"type=map(value=list(element=uint64(encoding=tagged)))"` - // Preserve the inferred packed inner carrier and only override descendants - Packed map[string][]int32 `fory:"type=map(value=_(element=int32(encoding=fixed)))"` + // Declare dense numeric array schema explicitly. + Dense []int32 `fory:"type=array(element=int32)"` + + // Use array schema inside a map value. + Packed map[string][]int32 `fory:"type=map(value=array(element=int32))"` } ``` **Notes**: -- `list(...)`, `set(...)`, and `map(...)` are explicit container overrides -- `_` keeps the inferred Go branch at that position and only overrides nested descendants -- There is no public `array(...)` tag. Packed primitive arrays are inferred from Go `[]T` / `[N]T` when the final element spec is packable -- Explicit `type=list(...)` always stays general LIST protocol and never collapses into a packed primitive array +- `list(...)`, `array(...)`, `set(...)`, and `map(...)` are explicit container overrides +- `list(...)` always uses list schema and never collapses into dense array schema +- `array(element=...)` requires a bool or numeric element domain and rejects nullable elements and scalar encoding modifiers ## Combining Tags diff --git a/docs/guide/go/supported-types.md b/docs/guide/go/supported-types.md index 93b8959499..1020d627f3 100644 --- a/docs/guide/go/supported-types.md +++ b/docs/guide/go/supported-types.md @@ -61,7 +61,12 @@ data, _ := f.Serialize(i64) // Uses varint encoding ## Collection Types -### Slices +### Root And Dynamic Slices + +When a slice is serialized as a root or dynamic value, primitive slices use the +dense array wire tags for compact transport. In struct fields, unannotated Go +slices are logical `list<T>` fields; use `fory:"type=array(element=...)"` when a +struct field is dense numeric `array<T>` data. | Go Type | Fory TypeId | Notes | | --------------- | ------------- | --------------------- | @@ -79,7 +84,7 @@ data, _ := f.Serialize(i64) // Uses varint encoding ```go f := fory.New() -// Primitive slices (optimized) +// Primitive root slice (optimized dense array payload) ints := []int32{1, 2, 3, 4, 5} data, _ := f.Serialize(ints) diff --git a/docs/guide/java/cross-language.md b/docs/guide/java/cross-language.md index e01f8258f1..5be83ce9a9 100644 --- a/docs/guide/java/cross-language.md +++ b/docs/guide/java/cross-language.md @@ -109,7 +109,7 @@ from dataclasses import dataclass @dataclass class Person: name: str - age: pyfory.int32 + age: pyfory.Int32 # Create Fory in xlang mode fory = pyfory.Fory(xlang=True, ref=True) @@ -159,10 +159,41 @@ Not all Java types have equivalents in other languages. When using xlang mode: - Use **primitive types** (`int`, `long`, `double`, `String`) for maximum compatibility - Use **standard collections** (`List`, `Map`, `Set`) instead of language-specific ones - Use **reduced-precision carriers** (`Float16`, `BFloat16`, `Float16List`, `BFloat16List`) for 16-bit float payloads -- Treat `Float16[]` and `BFloat16[]` as xlang `list` carriers; use `Float16List` and `BFloat16List` when the wire type must be `float16_array` or `bfloat16_array` +- Treat `Float16[]`, `BFloat16[]`, `Float16List`, and `BFloat16List` as `list<T>` carriers by default; use `@ArrayType` when the schema must be `array<float16>` or `array<bfloat16>` - Avoid **Java-specific types** like `Optional`, `BigDecimal` (unless the target language supports them) - See [Type Mapping Guide](../../specification/xlang_type_mapping.md) for complete compatibility matrix +### Lists and Dense Arrays + +Java primitive arrays are dense `array<T>` carriers, except plain `byte[]`, +which defaults to `bytes`. General Java collections and Fory primitive-list +carriers such as `Int32List`, `Float16List`, and `BFloat16List` use +`list<T>` unless the field has explicit `@ArrayType` metadata. + +| Fory schema | Java field shape | +| ----------------- | ------------------------------------------ | +| `list<int32>` | `List<Integer>` or `Int32List` | +| `array<bool>` | `boolean[]` | +| `array<int8>` | `@Int8Type byte[]` type-use | +| `array<int16>` | `short[]` | +| `array<int32>` | `int[]` | +| `array<int64>` | `long[]` | +| `array<uint8>` | `@UInt8Type byte[]` type-use | +| `array<uint16>` | `@UInt16Type short[]` type-use | +| `array<uint32>` | `@UInt32Type int[]` type-use | +| `array<uint64>` | `@UInt64Type long[]` type-use | +| `array<float16>` | `Float16Array` or `@Float16Type short[]` | +| `array<bfloat16>` | `BFloat16Array` or `@BFloat16Type short[]` | +| `array<float32>` | `float[]` | +| `array<float64>` | `double[]` | + +Prefer type-use syntax for primitive-array annotations: + +```java +private @UInt32Type int[] ids; +private @BFloat16Type short[] values; +``` + ### Compatible Types ```java diff --git a/docs/guide/java/field-configuration.md b/docs/guide/java/field-configuration.md index 48df5f0400..ef8bebd1bb 100644 --- a/docs/guide/java/field-configuration.md +++ b/docs/guide/java/field-configuration.md @@ -259,6 +259,8 @@ public class User { ## Integer Type Annotations Fory provides annotations to control integer encoding for cross-language compatibility. +Integer schema annotations are Java type-use annotations. Put them on the field type, after any +field modifiers and alongside `@ForyField` when both are present. ### Signed 32-bit Integer (`@Int32Type`) @@ -268,12 +270,10 @@ import org.apache.fory.config.Int32Encoding; public class MyStruct { // Variable-length encoding (default) - compact for small values - @Int32Type(encoding = Int32Encoding.VARINT) - private int compactId; + private @Int32Type(encoding = Int32Encoding.VARINT) int compactId; // Fixed 4-byte encoding - consistent size - @Int32Type(encoding = Int32Encoding.FIXED) - private int fixedId; + private @Int32Type(encoding = Int32Encoding.FIXED) int fixedId; } ``` @@ -285,16 +285,13 @@ import org.apache.fory.config.Int64Encoding; public class MyStruct { // Variable-length encoding (default) - @Int64Type(encoding = Int64Encoding.VARINT) - private long compactId; + private @Int64Type(encoding = Int64Encoding.VARINT) long compactId; // Fixed 8-byte encoding - @Int64Type(encoding = Int64Encoding.FIXED) - private long fixedTimestamp; + private @Int64Type(encoding = Int64Encoding.FIXED) long fixedTimestamp; // Tagged encoding (4 bytes for small values, 9 bytes otherwise) - @Int64Type(encoding = Int64Encoding.TAGGED) - private long taggedValue; + private @Int64Type(encoding = Int64Encoding.TAGGED) long taggedValue; } ``` @@ -310,30 +307,23 @@ import org.apache.fory.config.Int64Encoding; public class UnsignedStruct { // Unsigned 8-bit [0, 255] - @UInt8Type - private int flags; + private @UInt8Type int flags; // Unsigned 16-bit [0, 65535] - @UInt16Type - private int port; + private @UInt16Type int port; // Unsigned 32-bit with varint encoding (default) - @UInt32Type(encoding = Int32Encoding.VARINT) - private long compactCount; + private @UInt32Type(encoding = Int32Encoding.VARINT) long compactCount; // Unsigned 32-bit with fixed encoding - @UInt32Type(encoding = Int32Encoding.FIXED) - private long fixedCount; + private @UInt32Type(encoding = Int32Encoding.FIXED) long fixedCount; // Unsigned 64-bit with various encodings - @UInt64Type(encoding = Int64Encoding.VARINT) - private long varintU64; + private @UInt64Type(encoding = Int64Encoding.VARINT) long varintU64; - @UInt64Type(encoding = Int64Encoding.FIXED) - private long fixedU64; + private @UInt64Type(encoding = Int64Encoding.FIXED) long fixedU64; - @UInt64Type(encoding = Int64Encoding.TAGGED) - private long taggedU64; + private @UInt64Type(encoding = Int64Encoding.TAGGED) long taggedU64; } ``` @@ -384,19 +374,18 @@ public class NestedStruct { } ``` -Specialized unsigned list carriers still use packed-array wire types when the -field or nested element encoding is fixed. If a `UInt32List` or `UInt64List` -field is annotated with a variable or tagged encoding, Fory serializes it with -the collection protocol so the element type metadata is preserved. +Specialized unsigned list carriers use the `list<T>` schema by default, so +their element annotations are preserved in list metadata. Add `@ArrayType` only +when the field should use dense `array<T>` schema. -Primitive unsigned arrays can use element annotations for packed-array metadata: +Primitive unsigned arrays can use scalar element annotations for dense +`array<T>` metadata: ```java -import org.apache.fory.annotation.UInt32Elements; +import org.apache.fory.annotation.UInt32Type; public class IdBatch { - @UInt32Elements - private int[] ids; + private @UInt32Type int[] ids; } ``` @@ -438,16 +427,13 @@ public class Document { // Integer with different encodings @ForyField(id = 6) - @UInt64Type(encoding = Int64Encoding.VARINT) - private long viewCount; // varint encoding + private @UInt64Type(encoding = Int64Encoding.VARINT) long viewCount; // varint encoding @ForyField(id = 7) - @UInt64Type(encoding = Int64Encoding.FIXED) - private long fileSize; // fixed encoding + private @UInt64Type(encoding = Int64Encoding.FIXED) long fileSize; // fixed encoding @ForyField(id = 8) - @UInt64Type(encoding = Int64Encoding.TAGGED) - private long checksum; // tagged encoding + private @UInt64Type(encoding = Int64Encoding.TAGGED) long checksum; // tagged encoding // Reference-tracked field for shared/circular references @ForyField(id = 9, ref = true, nullable = true) @@ -492,16 +478,13 @@ When serializing data to be read by other languages (Python, Rust, C++, Go), use public class CrossLangData { // Use field IDs for cross-language compatibility @ForyField(id = 0) - @Int32Type(encoding = Int32Encoding.VARINT) - private int intVar; + private @Int32Type(encoding = Int32Encoding.VARINT) int intVar; @ForyField(id = 1) - @UInt64Type(encoding = Int64Encoding.FIXED) - private long longFixed; + private @UInt64Type(encoding = Int64Encoding.FIXED) long longFixed; @ForyField(id = 2) - @UInt64Type(encoding = Int64Encoding.TAGGED) - private long longTagged; + private @UInt64Type(encoding = Int64Encoding.TAGGED) long longTagged; @ForyField(id = 3, nullable = true) private String optionalValue; diff --git a/docs/guide/javascript/basic-serialization.md b/docs/guide/javascript/basic-serialization.md index ced490fd29..1ef847e507 100644 --- a/docs/guide/javascript/basic-serialization.md +++ b/docs/guide/javascript/basic-serialization.md @@ -123,7 +123,7 @@ Dynamic root serialization (calling `fory.serialize(someNumber)` without a schem ```ts const inventoryType = Type.struct("example.inventory", { - tags: Type.array(Type.string()), + tags: Type.list(Type.string()), counts: Type.map(Type.string(), Type.int32()), labels: Type.set(Type.string()), }); diff --git a/docs/guide/javascript/cross-language.md b/docs/guide/javascript/cross-language.md index d4fb159aa2..0e1c700347 100644 --- a/docs/guide/javascript/cross-language.md +++ b/docs/guide/javascript/cross-language.md @@ -86,6 +86,29 @@ JavaScript `number` is a 64-bit float, which does not map cleanly to every integ - `Type.int64()` with `bigint` values for 64-bit integers (Java `long`, Go `int64`) - `Type.float32()` or `Type.float64()` for floating-point values +## Lists and Dense Arrays + +Use `Type.list(T)` for ordinary JavaScript `Array<T>` values and Fory +`list<T>` schema. Dense bool/numeric vectors use the explicit array builders +listed below. + +| Fory schema | JavaScript/TypeScript schema builder | +| ----------------- | ------------------------------------ | +| `list<int32>` | `Type.list(Type.int32())` | +| `array<bool>` | `Type.boolArray()` | +| `array<int8>` | `Type.int8Array()` | +| `array<int16>` | `Type.int16Array()` | +| `array<int32>` | `Type.int32Array()` | +| `array<int64>` | `Type.int64Array()` | +| `array<uint8>` | `Type.uint8Array()` | +| `array<uint16>` | `Type.uint16Array()` | +| `array<uint32>` | `Type.uint32Array()` | +| `array<uint64>` | `Type.uint64Array()` | +| `array<float16>` | `Type.float16Array()` | +| `array<bfloat16>` | `Type.bfloat16Array()` | +| `array<float32>` | `Type.float32Array()` | +| `array<float64>` | `Type.float64Array()` | + ## Date and Time - `Type.timestamp()` โ a point in time; round-trips as a JavaScript `Date` diff --git a/docs/guide/javascript/supported-types.md b/docs/guide/javascript/supported-types.md index d61f135e5b..a932a35b3c 100644 --- a/docs/guide/javascript/supported-types.md +++ b/docs/guide/javascript/supported-types.md @@ -27,7 +27,7 @@ This page lists the JavaScript and TypeScript types supported by Fory, and expla | --------------------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------- | | `boolean` | `Type.bool()` | | | `number` | `Type.int8()` / `Type.int16()` / `Type.int32()` / `Type.float32()` / `Type.float64()` | Pick the width that matches the peer language | -| `bigint` | `Type.int64()` / `Type.varInt64()` / `Type.uint64()` | Use `bigint` for 64-bit integers | +| `bigint` | `Type.int64()` / `Type.uint64()` | Use `bigint` for 64-bit integers | | `string` | `Type.string()` | | | `Uint8Array` | `Type.binary()` | Binary blob | | `Date` | `Type.timestamp()` | Serializes/deserializes as `Date` | @@ -43,18 +43,18 @@ JavaScript `number` is a 64-bit float. It cannot safely represent all 64-bit int ```ts Type.int8(); // -128 to 127 Type.int16(); // -32,768 to 32,767 -Type.int32(); // matches Java int, Go int32, C# int -Type.varInt32(); // variable-length encoding; smaller for small values -Type.int64(); // use with bigint; matches Java long, Go int64 -Type.varInt64(); -Type.sliInt64(); +Type.int32(); // variable-length int32; default for semantic int32 +Type.int32({ encoding: "fixed" }); +Type.int64(); // variable-length int64; use with bigint +Type.int64({ encoding: "fixed" }); +Type.int64({ encoding: "tagged" }); Type.uint8(); Type.uint16(); -Type.uint32(); -Type.varUInt32(); -Type.uint64(); // use with bigint -Type.varUInt64(); -Type.taggedUInt64(); +Type.uint32(); // variable-length uint32 +Type.uint32({ encoding: "fixed" }); +Type.uint64(); // variable-length uint64; use with bigint +Type.uint64({ encoding: "fixed" }); +Type.uint64({ encoding: "tagged" }); ``` **Rule of thumb**: anything that maps to a 64-bit integer in another language should use `Type.int64()` or `Type.uint64()` on the JavaScript side and be passed as a `bigint` value. @@ -72,22 +72,22 @@ Type.bfloat16(); ## Arrays and Typed Arrays -### Generic arrays +### Lists ```ts -Type.array(Type.string()); -Type.array( +Type.list(Type.string()); +Type.list( Type.struct("example.item", { id: Type.int64(), }), ); ``` -These map to JavaScript arrays. +These map to JavaScript arrays and use the Fory `list<T>` schema. ## Optimized Numeric Arrays -For arrays of numbers, use the specialized typed array schemas. They are more compact and map to native typed arrays: +For dense arrays of bools and numbers, use the element-specific array builders. They are more compact and map to native typed arrays where JavaScript has one: ```ts Type.boolArray(); // boolean[] in JS @@ -100,7 +100,7 @@ Type.float16Array(); // number[] Type.bfloat16Array(); // BFloat16[] ``` -For non-numeric or struct arrays, use `Type.array(elementType)` instead. +Use `Type.list(elementType)` for non-numeric, struct, nullable-element, or ref-tracked ordered collections. ## Maps and Sets @@ -117,7 +117,7 @@ These map to JavaScript `Map` and `Set` values. Type.struct("example.user", { id: Type.int64(), name: Type.string(), - tags: Type.array(Type.string()), + tags: Type.list(Type.string()), }); ``` diff --git a/docs/guide/python/cross-language.md b/docs/guide/python/cross-language.md index fbe3e90bc3..8b857f44e5 100644 --- a/docs/guide/python/cross-language.md +++ b/docs/guide/python/cross-language.md @@ -45,7 +45,7 @@ f = pyfory.Fory(xlang=True, ref=True) @dataclass class Person: name: str - age: pyfory.int32 + age: pyfory.Int32 f.register(Person, typename="example.Person") @@ -104,56 +104,95 @@ import pyfory @dataclass class TypedData: - int_value: pyfory.int32 # 32-bit integer - long_value: pyfory.int64 # 64-bit integer - float_value: pyfory.float32 # 32-bit float - double_value: pyfory.float64 # 64-bit float - values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] + int_value: pyfory.Int32 # 32-bit integer + long_value: pyfory.Int64 # 64-bit integer + float_value: pyfory.Float32 # 32-bit float + double_value: pyfory.Float64 # 64-bit float + values: Dict[pyfory.Int32, List[pyfory.Int64]] ``` -Nested collection annotations are part of the field schema. For example, -`Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]]` writes map keys as fixed int32 values and the -nested list elements as tagged int64 values. Compatible-mode reads consume bytes with the remote -schema metadata, then assign only when the decoded value safely satisfies the local schema. +Nested collection annotations are part of the field schema. Compatible-mode +reads consume bytes with the remote schema metadata, then assign only when the +decoded value safely satisfies the local schema. ## Reduced-Precision Types -`pyfory.serialization` exports Cython-only carrier types for xlang reduced-precision values: +`pyfory.Float16` and `pyfory.BFloat16` are reserved annotation markers for xlang +reduced-precision fields. They are not runtime value classes; scalar values deserialize as native +Python `float`. -- `float16` and `float16array` -- `bfloat16` and `bfloat16array` - -These names are compiled into the `pyfory.serialization` extension and re-exported from `pyfory`. There is no pure-Python fallback module for them. - -The scalar wrappers behave like reduced-precision numeric value types. They support arithmetic and -ordering with Python numeric operands, and each operation quantizes the result back to the wrapper's -own format (`pyfory.float16` or `pyfory.bfloat16`). - -The array wrappers are value-oriented public APIs. Construct them from Python numeric values with -`pyfory.float16array([...])`, `pyfory.float16array.from_values([...])`, -`pyfory.bfloat16array([...])`, or `pyfory.bfloat16array.from_values([...])`. Use -`from_buffer(...)` and `to_buffer()` only when you already need packed little-endian `uint16` -storage and want the raw-buffer fast path. Both array carriers also implement the CPython buffer -protocol, so `memoryview(pyfory.float16array(...))` and `memoryview(pyfory.bfloat16array(...))` -expose the packed `uint16` storage directly. +Dense reduced-precision arrays use public dense wrappers with list-like sequence behavior. Construct them from Python +numeric values with `pyfory.Float16Array.from_values([...])` or +`pyfory.BFloat16Array.from_values([...])`. Use `from_buffer(...)` and `to_buffer()` only when you +already need packed little-endian `uint16` storage and want the raw-buffer fast path. ## Type Mapping -| Python | Java | Rust | Go | +| Python marker/carrier | Java | Rust | Go | | ---------------------- | -------------- | --------------- | --------------------- | | `str` | `String` | `String` | `string` | | `int` | `long` | `i64` | `int64` | -| `pyfory.int32` | `int` | `i32` | `int32` | -| `pyfory.int64` | `long` | `i64` | `int64` | +| `pyfory.Int32` | `int` | `i32` | `int32` | +| `pyfory.Int64` | `long` | `i64` | `int64` | | `float` | `double` | `f64` | `float64` | -| `pyfory.float32` | `float` | `f32` | `float32` | -| `pyfory.float16` | `Float16` | `Float16` | `float16.Float16` | -| `pyfory.bfloat16` | `BFloat16` | `BFloat16` | `bfloat16.BFloat16` | -| `pyfory.float16array` | `Float16List` | `Vec<Float16>` | `[]float16.Float16` | -| `pyfory.bfloat16array` | `BFloat16List` | `Vec<BFloat16>` | `[]bfloat16.BFloat16` | +| `pyfory.Float32` | `float` | `f32` | `float32` | +| `pyfory.Float16` | `Float16` | `Float16` | `float16.Float16` | +| `pyfory.BFloat16` | `BFloat16` | `BFloat16` | `bfloat16.BFloat16` | +| `pyfory.Float16Array` | `Float16List` | `Vec<Float16>` | `[]float16.Float16` | +| `pyfory.BFloat16Array` | `BFloat16List` | `Vec<BFloat16>` | `[]bfloat16.BFloat16` | | `list` | `List` | `Vec` | `[]T` | | `dict` | `Map` | `HashMap` | `map[K]V` | +### Lists and Dense Arrays + +Python `List[T]` maps to Fory `list<T>`. Use `pyfory.Array[T]`, +`pyfory.NDArray[T]`, or `pyfory.PyArray[T]` only when the schema is the dense +one-dimensional `array<T>` kind. + +| Fory schema | Python annotation and default carrier | +| ----------------- | -------------------------------------------------- | +| `list<int32>` | `List[pyfory.Int32]` | +| `array<bool>` | `pyfory.Array[bool]` -> `BoolArray` | +| `array<int8>` | `pyfory.Array[pyfory.Int8]` -> `Int8Array` | +| `array<int16>` | `pyfory.Array[pyfory.Int16]` -> `Int16Array` | +| `array<int32>` | `pyfory.Array[pyfory.Int32]` -> `Int32Array` | +| `array<int64>` | `pyfory.Array[pyfory.Int64]` -> `Int64Array` | +| `array<uint8>` | `pyfory.Array[pyfory.UInt8]` -> `UInt8Array` | +| `array<uint16>` | `pyfory.Array[pyfory.UInt16]` -> `UInt16Array` | +| `array<uint32>` | `pyfory.Array[pyfory.UInt32]` -> `UInt32Array` | +| `array<uint64>` | `pyfory.Array[pyfory.UInt64]` -> `UInt64Array` | +| `array<float16>` | `pyfory.Array[pyfory.Float16]` -> `Float16Array` | +| `array<bfloat16>` | `pyfory.Array[pyfory.BFloat16]` -> `BFloat16Array` | +| `array<float32>` | `pyfory.Array[pyfory.Float32]` -> `Float32Array` | +| `array<float64>` | `pyfory.Array[pyfory.Float64]` -> `Float64Array` | + +The `pyfory.*Array` wrappers accept iterable constructors such as +`pyfory.Float32Array([1, 2, 3])` and expose list-like sequence behavior over +dense owned storage. + +`pyfory.Array[T]`, `pyfory.NDArray[T]`, and `pyfory.PyArray[T]` all describe +the same Fory `array<T>` schema. They differ only in the Python carrier +contract: + +| Python field annotation | Value accepted for that field | Deserialized carrier | +| ----------------------- | ------------------------------------------------------- | -------------------- | +| `pyfory.Array[T]` | `pyfory.*Array`, `numpy.ndarray`, `array.array`, `list` | `pyfory.*Array` | +| `pyfory.NDArray[T]` | `numpy.ndarray` | `numpy.ndarray` | +| `pyfory.PyArray[T]` | Python `array.array` | Python `array.array` | + +In compatible mode, a writer and reader can use different Python carriers for +the same named field as long as both annotations lower to the same Fory +`array<T>` schema. For example, a writer field declared as +`pyfory.Array[pyfory.Int32]` can be read by a Python class whose matching field +is declared as `pyfory.NDArray[pyfory.Int32]`, and the reader receives a NumPy +`int32` ndarray. The reverse pattern also works for `pyfory.PyArray[T]`; that +name always means Python `array.array`. + +PyArrow is a separate row/columnar format surface, not a `pyfory.PyArray` +carrier. Use `pyfory.format.from_arrow_schema(...)` and +`pyfory.format.to_arrow_schema(...)` to convert between PyArrow schemas and +Fory row-format schemas. + ## Differences from Python Native Mode The binary protocol and API are similar to `pyfory`'s python-native mode, but Python-native mode can serialize any Python objectโincluding global functions, local functions, lambdas, local classes, and types with customized serialization using `__getstate__/__reduce__/__reduce_ex__`, which are **not allowed** in xlang mode. diff --git a/docs/guide/python/field-configuration.md b/docs/guide/python/field-configuration.md index 1e4e8f35ca..e223d5de21 100644 --- a/docs/guide/python/field-configuration.md +++ b/docs/guide/python/field-configuration.md @@ -50,7 +50,7 @@ import pyfory @dataclass class Person: name: str = pyfory.field(id=0) - age: pyfory.int32 = pyfory.field(id=1, default=0) + age: pyfory.Int32 = pyfory.field(id=1, default=0) nickname: Optional[str] = pyfory.field(id=2, nullable=True, default=None) ``` @@ -61,7 +61,7 @@ Use `pyfory.field()` to configure field-level metadata: ```python @dataclass class User: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") email: Optional[str] = pyfory.field(id=2, nullable=True, default=None) friends: List["User"] = pyfory.field(id=3, ref=True, default_factory=list) @@ -87,9 +87,9 @@ Assigns a numeric ID to a field to minimize struct field meta size overhead: ```python @dataclass class User: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") - age: pyfory.int32 = pyfory.field(id=2, default=0) + age: pyfory.Int32 = pyfory.field(id=2, default=0) ``` **Benefits**: @@ -111,7 +111,7 @@ class User: ```python @dataclass class User: - id: pyfory.int64 = 0 + id: pyfory.Int64 = 0 name: str = "" ``` @@ -128,7 +128,7 @@ class Record: optional_name: Optional[str] = pyfory.field(id=0, nullable=True, default=None) # Nullable integer field - optional_count: Optional[pyfory.int32] = pyfory.field(id=1, nullable=True, default=None) + optional_count: Optional[pyfory.Int32] = pyfory.field(id=1, nullable=True, default=None) ``` **Notes**: @@ -172,7 +172,7 @@ Exclude fields from serialization: ```python @dataclass class User: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") # Not serialized _cache: dict = pyfory.field(ignore=True, default_factory=dict) @@ -234,10 +234,10 @@ Fory provides type annotations to control integer encoding: ```python @dataclass class SignedIntegers: - byte_val: pyfory.int8 = 0 # 8-bit signed - short_val: pyfory.int16 = 0 # 16-bit signed - int_val: pyfory.int32 = 0 # 32-bit signed (varint encoding) - long_val: pyfory.int64 = 0 # 64-bit signed (varint encoding) + byte_val: pyfory.Int8 = 0 # 8-bit signed + short_val: pyfory.Int16 = 0 # 16-bit signed + int_val: pyfory.Int32 = 0 # 32-bit signed (varint encoding) + long_val: pyfory.Int64 = 0 # 64-bit signed (varint encoding) ``` ### Unsigned Integers @@ -246,19 +246,19 @@ class SignedIntegers: @dataclass class UnsignedIntegers: # Fixed-size encoding - u8_val: pyfory.uint8 = 0 # 8-bit unsigned (fixed) - u16_val: pyfory.uint16 = 0 # 16-bit unsigned (fixed) + u8_val: pyfory.UInt8 = 0 # 8-bit unsigned (fixed) + u16_val: pyfory.UInt16 = 0 # 16-bit unsigned (fixed) # Variable-length encoding (default for u32/u64) - u32_var: pyfory.uint32 = 0 # 32-bit unsigned (varint) - u64_var: pyfory.uint64 = 0 # 64-bit unsigned (varint) + u32_var: pyfory.UInt32 = 0 # 32-bit unsigned (varint) + u64_var: pyfory.UInt64 = 0 # 64-bit unsigned (varint) # Explicit fixed-size encoding - u32_fixed: pyfory.fixed_uint32 = 0 # 32-bit unsigned (fixed 4 bytes) - u64_fixed: pyfory.fixed_uint64 = 0 # 64-bit unsigned (fixed 8 bytes) + u32_fixed: pyfory.FixedUInt32 = 0 # 32-bit unsigned (fixed 4 bytes) + u64_fixed: pyfory.FixedUInt64 = 0 # 64-bit unsigned (fixed 8 bytes) # Tagged encoding (includes type tag) - u64_tagged: pyfory.tagged_uint64 = 0 # 64-bit unsigned (tagged) + u64_tagged: pyfory.TaggedUInt64 = 0 # 64-bit unsigned (tagged) ``` ### Floating Point @@ -266,30 +266,30 @@ class UnsignedIntegers: ```python @dataclass class FloatingPoint: - float_val: pyfory.float32 = 0.0 # 32-bit float - double_val: pyfory.float64 = 0.0 # 64-bit double + float_val: pyfory.Float32 = 0.0 # 32-bit float + double_val: pyfory.Float64 = 0.0 # 64-bit double ``` ### Encoding Summary -| Type | Encoding | Size | -| ---------------------- | -------- | ---------- | -| `pyfory.int8` | fixed | 1 byte | -| `pyfory.int16` | fixed | 2 bytes | -| `pyfory.int32` | varint | 1-5 bytes | -| `pyfory.int64` | varint | 1-10 bytes | -| `pyfory.fixed_int32` | fixed | 4 bytes | -| `pyfory.fixed_int64` | fixed | 8 bytes | -| `pyfory.tagged_int64` | tagged | 1-9 bytes | -| `pyfory.uint8` | fixed | 1 byte | -| `pyfory.uint16` | fixed | 2 bytes | -| `pyfory.uint32` | varint | 1-5 bytes | -| `pyfory.uint64` | varint | 1-10 bytes | -| `pyfory.fixed_uint32` | fixed | 4 bytes | -| `pyfory.fixed_uint64` | fixed | 8 bytes | -| `pyfory.tagged_uint64` | tagged | 1-9 bytes | -| `pyfory.float32` | fixed | 4 bytes | -| `pyfory.float64` | fixed | 8 bytes | +| Type | Encoding | Size | +| --------------------- | -------- | ---------- | +| `pyfory.Int8` | fixed | 1 byte | +| `pyfory.Int16` | fixed | 2 bytes | +| `pyfory.Int32` | varint | 1-5 bytes | +| `pyfory.Int64` | varint | 1-10 bytes | +| `pyfory.FixedInt32` | fixed | 4 bytes | +| `pyfory.FixedInt64` | fixed | 8 bytes | +| `pyfory.TaggedInt64` | tagged | 1-9 bytes | +| `pyfory.UInt8` | fixed | 1 byte | +| `pyfory.UInt16` | fixed | 2 bytes | +| `pyfory.UInt32` | varint | 1-5 bytes | +| `pyfory.UInt64` | varint | 1-10 bytes | +| `pyfory.FixedUInt32` | fixed | 4 bytes | +| `pyfory.FixedUInt64` | fixed | 8 bytes | +| `pyfory.TaggedUInt64` | tagged | 1-9 bytes | +| `pyfory.Float32` | fixed | 4 bytes | +| `pyfory.Float64` | fixed | 8 bytes | **When to Use**: @@ -310,7 +310,7 @@ import pyfory @dataclass class Counters: - values: Dict[pyfory.fixed_int32, List[pyfory.tagged_int64]] = field(default_factory=dict) + values: Dict[pyfory.FixedInt32, List[pyfory.TaggedInt64]] = field(default_factory=dict) ``` For `values`, map keys are written as fixed-width int32 values and each nested list element is @@ -334,7 +334,7 @@ import pyfory class Document: # Fields with tag IDs (recommended for compatible mode) title: str = pyfory.field(id=0, default="") - version: pyfory.int32 = pyfory.field(id=1, default=0) + version: pyfory.Int32 = pyfory.field(id=1, default=0) # Nullable field description: Optional[str] = pyfory.field(id=2, nullable=True, default=None) @@ -345,9 +345,9 @@ class Document: categories: Set[str] = pyfory.field(id=5, default_factory=set) # Unsigned integers with different encodings - view_count: pyfory.uint64 = pyfory.field(id=6, default=0) # varint encoding - file_size: pyfory.fixed_uint64 = pyfory.field(id=7, default=0) # fixed encoding - checksum: pyfory.tagged_uint64 = pyfory.field(id=8, default=0) # tagged encoding + view_count: pyfory.UInt64 = pyfory.field(id=6, default=0) # varint encoding + file_size: pyfory.FixedUInt64 = pyfory.field(id=7, default=0) # fixed encoding + checksum: pyfory.TaggedUInt64 = pyfory.field(id=8, default=0) # tagged encoding # Reference-tracked field for shared/circular references parent: Optional["Document"] = pyfory.field(id=9, ref=True, nullable=True, default=None) @@ -394,9 +394,9 @@ When serializing data to be read by other languages (Java, Rust, C++, Go), use f @dataclass class CrossLangData: # Use field IDs for cross-language compatibility - int_var: pyfory.int32 = pyfory.field(id=0, default=0) - long_fixed: pyfory.fixed_uint64 = pyfory.field(id=1, default=0) - long_tagged: pyfory.tagged_uint64 = pyfory.field(id=2, default=0) + int_var: pyfory.Int32 = pyfory.field(id=0, default=0) + long_fixed: pyfory.FixedUInt64 = pyfory.field(id=1, default=0) + long_tagged: pyfory.TaggedUInt64 = pyfory.field(id=2, default=0) optional_value: Optional[str] = pyfory.field(id=3, nullable=True, default=None) ``` @@ -408,14 +408,14 @@ Compatible mode supports schema evolution. It is recommended to configure field # Version 1 @dataclass class DataV1: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") # Version 2: Added new field @dataclass class DataV2: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") email: Optional[str] = pyfory.field(id=2, nullable=True, default=None) # New field ``` @@ -427,7 +427,7 @@ Alternatively, field IDs can be omitted (field names will be used in metadata wi ```python @dataclass class Data: - id: pyfory.int64 = 0 + id: pyfory.Int64 = 0 name: str = "" ``` @@ -474,7 +474,7 @@ In xlang mode, you **need to configure fields** when: # Xlang mode: explicit configuration required for nullable/ref fields @dataclass class User: - id: pyfory.int64 = pyfory.field(id=0, default=0) + id: pyfory.Int64 = pyfory.field(id=0, default=0) name: str = pyfory.field(id=1, default="") email: Optional[str] = pyfory.field(id=2, nullable=True, default=None) # Must declare nullable friend: Optional["User"] = pyfory.field(id=3, ref=True, nullable=True, default=None) # Must declare ref @@ -499,21 +499,21 @@ class User: ## Options Reference -| Configuration | Description | -| -------------------------------------------- | ------------------------------------ | -| `pyfory.field(id=N)` | Field tag ID to reduce metadata size | -| `pyfory.field(nullable=True)` | Mark field as nullable | -| `pyfory.field(ref=True)` | Enable reference tracking | -| `pyfory.field(ignore=True)` | Exclude field from serialization | -| `pyfory.field(dynamic=True)` | Force type info to be written | -| `pyfory.field(dynamic=False)` | Skip type info (use declared type) | -| `Optional[T]` | Type hint for nullable fields | -| `pyfory.int32`, `pyfory.int64` | Signed integers (varint encoding) | -| `pyfory.fixed_int32`, `pyfory.fixed_int64` | Fixed-size signed | -| `pyfory.tagged_int64` | Tagged encoding for int64 | -| `pyfory.uint32`, `pyfory.uint64` | Unsigned integers (varint encoding) | -| `pyfory.fixed_uint32`, `pyfory.fixed_uint64` | Fixed-size unsigned | -| `pyfory.tagged_uint64` | Tagged encoding for uint64 | +| Configuration | Description | +| ------------------------------------------ | ------------------------------------ | +| `pyfory.field(id=N)` | Field tag ID to reduce metadata size | +| `pyfory.field(nullable=True)` | Mark field as nullable | +| `pyfory.field(ref=True)` | Enable reference tracking | +| `pyfory.field(ignore=True)` | Exclude field from serialization | +| `pyfory.field(dynamic=True)` | Force type info to be written | +| `pyfory.field(dynamic=False)` | Skip type info (use declared type) | +| `Optional[T]` | Type hint for nullable fields | +| `pyfory.Int32`, `pyfory.Int64` | Signed integers (varint encoding) | +| `pyfory.FixedInt32`, `pyfory.FixedInt64` | Fixed-size signed | +| `pyfory.TaggedInt64` | Tagged encoding for int64 | +| `pyfory.UInt32`, `pyfory.UInt64` | Unsigned integers (varint encoding) | +| `pyfory.FixedUInt32`, `pyfory.FixedUInt64` | Fixed-size unsigned | +| `pyfory.TaggedUInt64` | Tagged encoding for uint64 | ## Related Topics diff --git a/docs/guide/python/row-format.md b/docs/guide/python/row-format.md index 3fd86f9f3e..a6784b350e 100644 --- a/docs/guide/python/row-format.md +++ b/docs/guide/python/row-format.md @@ -38,21 +38,21 @@ Row format drastically reduces overhead when working with large objects where on ## Basic Usage ```python -import pyfory -import pyarrow as pa from dataclasses import dataclass -from typing import List, Dict +from typing import Dict, List + +import pyfory @dataclass class Bar: f1: str - f2: List[pa.int64] + f2: List[pyfory.Int64] @dataclass class Foo: - f1: pa.int32 - f2: List[pa.int32] - f3: Dict[str, pa.int32] + f1: pyfory.Int32 + f2: List[pyfory.Int32] + f3: Dict[str, pyfory.Int32] f4: List[Bar] # Create encoder for row format @@ -76,6 +76,30 @@ print(foo_row.f4[100000].f1) # Access nested field directly print(foo_row.f4[200000].f2[5]) # Access deeply nested field directly ``` +## PyArrow Schema Conversion + +Row format can convert PyArrow schemas through `pyfory.format` when the +`format` optional dependency is installed: + +```python +import pyarrow as pa +from pyfory.format import from_arrow_schema, to_arrow_schema + +arrow_schema = pa.schema( + [ + pa.field("id", pa.int32(), nullable=False), + pa.field("scores", pa.list_(pa.float64())), + ] +) + +fory_schema = from_arrow_schema(arrow_schema) +roundtrip_arrow_schema = to_arrow_schema(fory_schema) +``` + +This PyArrow conversion surface is separate from cross-language dense-array +field annotations. In object serialization, `pyfory.PyArray[T]` means the +standard-library Python `array.array` carrier, not PyArrow. + ## Cross-Language Compatibility Row format works seamlessly across languages. The same binary data can be accessed from Java and C++. diff --git a/docs/guide/rust/basic-serialization.md b/docs/guide/rust/basic-serialization.md index 6298ed4268..2d2386f5a6 100644 --- a/docs/guide/rust/basic-serialization.md +++ b/docs/guide/rust/basic-serialization.md @@ -103,7 +103,7 @@ assert_eq!(person, decoded); | `BinaryHeap<T>` | Binary heap | | `Option<T>` | Optional value | -`Vec<BFloat16>` is the dense xlang carrier for `bfloat16_array` payloads. +`Vec<BFloat16>` is the dense carrier when the schema is `array<bfloat16>`. ### Smart Pointers diff --git a/docs/guide/rust/cross-language.md b/docs/guide/rust/cross-language.md index f5f0d24b11..c067f1315b 100644 --- a/docs/guide/rust/cross-language.md +++ b/docs/guide/rust/cross-language.md @@ -119,7 +119,7 @@ from dataclasses import dataclass @dataclass class Person: name: str - age: pyfory.int32 + age: pyfory.Int32 fory = pyfory.Fory(xlang=True, ref=True) fory.register_type(Person, type_id=100) # Same ID as Rust @@ -143,13 +143,35 @@ See [xlang_type_mapping.md](../../specification/xlang_type_mapping.md) for compl | `BFloat16` | `BFloat16` | `bfloat16` | | `String` | `String` | `str` | | `Vec<T>` | `List<T>` | `List[T]` | -| `Vec<Float16>` | `Float16List` | `float16array` | -| `Vec<BFloat16>` | `BFloat16List` | `bfloat16array` | -| `[Float16; N]` | `Float16List` | `float16array` | -| `[BFloat16; N]` | `BFloat16List` | `bfloat16array` | +| `Vec<Float16>` | `Float16List` | `Float16Array` | +| `Vec<BFloat16>` | `BFloat16List` | `BFloat16Array` | +| `[Float16; N]` | `Float16List` | `Float16Array` | +| `[BFloat16; N]` | `BFloat16List` | `BFloat16Array` | | `HashMap<K,V>` | `Map<K,V>` | `Dict[K,V]` | | `Option<T>` | nullable `T` | `Optional[T]` | +### Lists and Dense Arrays + +Rust `Vec<T>` maps to Fory `list<T>` by default for manual structs. Use an +explicit array field attribute when the schema is dense `array<T>`. + +| Fory schema | Rust carrier and metadata | +| ----------------- | ------------------------------ | +| `list<int32>` | `Vec<i32>` | +| `array<bool>` | `#[fory(array)] Vec<bool>` | +| `array<int8>` | `#[fory(array)] Vec<i8>` | +| `array<int16>` | `#[fory(array)] Vec<i16>` | +| `array<int32>` | `#[fory(array)] Vec<i32>` | +| `array<int64>` | `#[fory(array)] Vec<i64>` | +| `array<uint8>` | `#[fory(array)] Vec<u8>` | +| `array<uint16>` | `#[fory(array)] Vec<u16>` | +| `array<uint32>` | `#[fory(array)] Vec<u32>` | +| `array<uint64>` | `#[fory(array)] Vec<u64>` | +| `array<float16>` | `#[fory(array)] Vec<Float16>` | +| `array<bfloat16>` | `#[fory(array)] Vec<BFloat16>` | +| `array<float32>` | `#[fory(array)] Vec<f32>` | +| `array<float64>` | `#[fory(array)] Vec<f64>` | + ## Best Practices 1. **Use consistent type IDs** across all languages diff --git a/docs/guide/swift/cross-language.md b/docs/guide/swift/cross-language.md index 2c9f4c6fde..8d32c60523 100644 --- a/docs/guide/swift/cross-language.md +++ b/docs/guide/swift/cross-language.md @@ -54,6 +54,29 @@ try fory.register(Order.self, namespace: "com.example", name: "Order") - Use compatible mode when independently evolving schemas - Register all user-defined concrete types used by dynamic fields (`Any`, `any Serializer`) +## Lists and Dense Arrays + +Swift `Array<T>` fields map to Fory `list<T>` unless field metadata explicitly +requests dense `array<T>`. Use `array<T>` only for one-dimensional bool or +numeric data. + +| Fory schema | Swift field metadata sketch | +| ----------------- | ---------------------------------------------------------- | +| `list<int32>` | `@ListField(element: .int32()) var ids: [Int32]` | +| `array<bool>` | `@ArrayField(element: .bool()) var flags: [Bool]` | +| `array<int8>` | `@ArrayField(element: .int8()) var values: [Int8]` | +| `array<int16>` | `@ArrayField(element: .int16()) var values: [Int16]` | +| `array<int32>` | `@ArrayField(element: .int32()) var values: [Int32]` | +| `array<int64>` | `@ArrayField(element: .int64()) var values: [Int64]` | +| `array<uint8>` | `@ArrayField(element: .uint8()) var values: [UInt8]` | +| `array<uint16>` | `@ArrayField(element: .uint16()) var values: [UInt16]` | +| `array<uint32>` | `@ArrayField(element: .uint32()) var values: [UInt32]` | +| `array<uint64>` | `@ArrayField(element: .uint64()) var values: [UInt64]` | +| `array<float16>` | `@ArrayField(element: .float16()) var values: [Float16]` | +| `array<bfloat16>` | `@ArrayField(element: .bfloat16()) var values: [BFloat16]` | +| `array<float32>` | `@ArrayField(element: .float32()) var values: [Float]` | +| `array<float64>` | `@ArrayField(element: .float64()) var values: [Double]` | + ## Swift IDL Workflow Generate Swift models directly from Fory IDL/Proto/FBS inputs: diff --git a/docs/guide/xlang/field-nullability.md b/docs/guide/xlang/field-nullability.md index de7f9dc2b4..5740344714 100644 --- a/docs/guide/xlang/field-nullability.md +++ b/docs/guide/xlang/field-nullability.md @@ -123,7 +123,7 @@ import pyfory class Person: # Non-nullable by default name: str # Must have a value - age: pyfory.int32 # Primitive + age: pyfory.Int32 # Primitive tags: List[str] # Must not be None # Optional makes it nullable diff --git a/docs/guide/xlang/getting-started.md b/docs/guide/xlang/getting-started.md index 6cd8ca3a40..23a4cdcdf4 100644 --- a/docs/guide/xlang/getting-started.md +++ b/docs/guide/xlang/getting-started.md @@ -270,7 +270,7 @@ from dataclasses import dataclass @dataclass class Person: name: str - age: pyfory.Int32Type + age: pyfory.Int32 fory = pyfory.Fory(xlang=True) fory.register_type(Person, typename="example.Person") @@ -288,7 +288,7 @@ print(f"Name: {person.name}, Age: {person.age}") 1. **Use consistent type names**: Ensure all languages use the same type name or ID 2. **Enable reference tracking**: If your data has circular or shared references 3. **Reuse Fory instances**: Creating Fory is expensive; reuse instances -4. **Use type annotations**: In Python, use `pyfory.Int32Type` etc. for precise type mapping +4. **Use type annotations**: In Python, use markers such as `pyfory.Int32` for precise type mapping 5. **Test cross-language**: Verify serialization works across all target languages ## Next Steps diff --git a/docs/guide/xlang/index.md b/docs/guide/xlang/index.md index 4d322a0ac2..8ddc120942 100644 --- a/docs/guide/xlang/index.md +++ b/docs/guide/xlang/index.md @@ -89,7 +89,7 @@ from dataclasses import dataclass @dataclass class Person: name: str - age: pyfory.Int32Type + age: pyfory.Int32 fory = pyfory.Fory(xlang=True) fory.register_type(Person, typename="example.Person") diff --git a/docs/guide/xlang/serialization.md b/docs/guide/xlang/serialization.md index dda0fc6961..ca82a69ea8 100644 --- a/docs/guide/xlang/serialization.md +++ b/docs/guide/xlang/serialization.md @@ -27,10 +27,10 @@ Common types can be serialized automatically without registration: primitive num Reduced-precision floating-point values are also part of the built-in xlang type system: -- `float16` and `float16_array` -- `bfloat16` and `bfloat16_array` +- `float16` and `array<float16>` +- `bfloat16` and `array<bfloat16>` -Use the language-specific carrier types documented in the type mapping reference. Python exposes the Cython-only `float16`, `float16array`, `bfloat16`, and `bfloat16array` names from `pyfory.serialization`; the Python array carriers are constructed from Python numeric values, while `from_buffer(...)` is reserved for packed raw storage. Go uses the `float16` and `bfloat16` packages for scalar, slice, and array carriers; JavaScript uses `number` / `number[]` for `float16` and `BFloat16` / [...] +Use the language-specific carrier types documented in the type mapping reference. Python uses `pyfory.Float16` and `pyfory.BFloat16` as annotation markers only; scalar values are native Python `float`, and dense reduced-precision arrays use `pyfory.Float16Array` and `pyfory.BFloat16Array`. Go uses the `float16` and `bfloat16` packages for scalar, slice, and array carriers; JavaScript uses `number` / `number[]` for `float16` and `BFloat16` / `BFloat16Array` for `bfloat16`; Java uses `@Arr [...] ### Java @@ -222,7 +222,7 @@ import pyfory, array @dataclass class SomeClass1: f1: Any - f2: Dict[pyfory.Int8Type, pyfory.Int32Type] + f2: Dict[pyfory.Int8, pyfory.Int32] @dataclass @@ -230,18 +230,18 @@ class SomeClass2: f1: Any = None f2: str = None f3: List[str] = None - f4: Dict[pyfory.Int8Type, pyfory.Int32Type] = None - f5: pyfory.Int8Type = None - f6: pyfory.Int16Type = None - f7: pyfory.Int32Type = None - # int type will be taken as `pyfory.Int64Type`. - # use `pyfory.Int32Type` for type hint if peer uses more narrow type. + f4: Dict[pyfory.Int8, pyfory.Int32] = None + f5: pyfory.Int8 = None + f6: pyfory.Int16 = None + f7: pyfory.Int32 = None + # int type will be taken as `pyfory.Int64`. + # use `pyfory.Int32` for type hint if peer uses more narrow type. f8: int = None - f9: pyfory.Float32Type = None - # float type will be taken as `pyfory.Float64Type` + f9: pyfory.Float32 = None + # float type will be taken as `pyfory.Float64` f10: float = None - f11: pyfory.Int16ArrayType = None - f12: List[pyfory.Int16Type] = None + f11: pyfory.Array[pyfory.Int16] = None + f12: List[pyfory.Int16] = None if __name__ == "__main__": diff --git a/docs/guide/xlang/troubleshooting.md b/docs/guide/xlang/troubleshooting.md index 2ccb2071f3..b3931eeb06 100644 --- a/docs/guide/xlang/troubleshooting.md +++ b/docs/guide/xlang/troubleshooting.md @@ -90,7 +90,7 @@ fory.register_type(Address, type_id=101) ```python @dataclass class Data: - value: pyfory.Int32Type # Not just 'int' + value: pyfory.Int32 # Not just 'int' ``` 2. Ensure integer ranges are compatible: @@ -111,7 +111,7 @@ fory.register_type(Address, type_id=101) ```python @dataclass class Data: - value: pyfory.Float32Type # Explicit 32-bit float + value: pyfory.Float32 # Explicit 32-bit float ``` 2. Be aware that Python's `float` maps to `float64` by default @@ -159,7 +159,7 @@ public class Person { @dataclass class Person: name: str - age: pyfory.Int32Type + age: pyfory.Int32 email: str ``` @@ -291,7 +291,7 @@ python deserializer.py data.bin 1. **Not registering types**: Always register custom types before use 2. **Inconsistent type names/IDs**: Use the same names/IDs across all languages 3. **Forgetting xlang mode**: Use `.withXlang(true)` in Java -4. **Wrong type annotations**: Use `pyfory.Int32Type` etc. in Python +4. **Wrong type annotations**: Use markers such as `pyfory.Int32` in Python 5. **Ignoring reference tracking**: Enable for circular/shared references ## See Also --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
