2
0
mirror of https://github.com/esiur/esiur-dotnet.git synced 2026-03-31 18:38:22 +00:00

comparision test brought here

This commit is contained in:
2026-03-19 15:26:42 +03:00
parent ee3fbd116d
commit e300173bdd
24 changed files with 2613 additions and 424 deletions

View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Apache.Avro" Version="1.12.1" />
<PackageReference Include="AvroConvert" Version="3.4.16" />
<PackageReference Include="FlatSharp" Version="6.3.5" />
<PackageReference Include="Google.Protobuf" Version="3.34.0" />
<PackageReference Include="MessagePack" Version="3.1.4" />
<PackageReference Include="MongoDB.Bson" Version="3.7.1" />
<PackageReference Include="PeterO.Cbor" Version="4.5.5" />
<PackageReference Include="protobuf-net" Version="3.2.56" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Esiur\Esiur.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,353 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Esiur.Tests.Serialization;
public static class IntArrayGenerator
{
private static readonly Random rng = new Random(24241564);
public static long[] GenerateInt32Run(int length)
{
var data = new long[length];
int i = 0;
var inSmallRange = true;
var inShortRange = false;
var inLargeRange = false;
var inLongRange = false;
long range = 30;
while (i < length)
{
// stay same range
if (rng.NextDouble() < 0.9)
{
if (inSmallRange)
data[i++] = rng.Next(-64, 65);
else if (inShortRange)
data[i++] = rng.NextInt64(range - 100, range + 100);
else if (inLargeRange)
data[i++] = rng.NextInt64(range - 1000, range + 1000);
else if (inLongRange)
data[i++] = rng.NextInt64(range - 10000, range + 10000);
}
else
{
// switch range
var rand = rng.NextDouble();
if (rand < 0.25)
{
inSmallRange = true;
inShortRange = false;
inLargeRange = false;
inLongRange = false;
data[i++] = rng.Next(-64, 65);
}
else if (rand < 0.50)
{
inSmallRange = false;
inShortRange = true;
inLargeRange = false;
inLongRange = false;
range = rng.NextInt64(1000, short.MaxValue);
data[i++] = rng.NextInt64(range - 100, range + 100);
}
else if (rand < 0.75)
{
inSmallRange = false;
inShortRange = false;
inLargeRange = true;
inLongRange = false;
range = rng.NextInt64(1000, int.MaxValue);
data[i++] = rng.NextInt64(range - 1000, range + 1000);
}
else
{
inSmallRange = false;
inShortRange = false;
inLargeRange = false;
inLongRange = true;
range = rng.NextInt64(10000, long.MaxValue);
data[i++] = rng.NextInt64(range - 10000, range + 10000);
}
}
}
return data;
}
// Generate random int array of given length and distribution
public static int[] GenerateInt32(int length, string pattern = "uniform",
int range = int.MaxValue)
{
var data = new int[length];
switch (pattern.ToLower())
{
case "uniform":
// Random values in [-range, range]
for (int i = 0; i < length; i++)
data[i] = rng.Next(-range, range);
break;
case "positive":
for (int i = 0; i < length; i++)
data[i] = rng.Next(0, range);
break;
case "negative":
for (int i = 0; i < length; i++)
data[i] = -rng.Next(0, range);
break;
case "alternating":
for (int i = 0; i < length; i++)
{
int val = rng.Next(0, range);
data[i] = (i % 2 == 0) ? val : -val;
}
break;
case "small":
// Focused on small magnitudes to test ZigZag fast path
for (int i = 0; i < length; i++)
data[i] = rng.Next(-64, 65);
break;
case "ascending":
{
int start = rng.Next(-range, range);
for (int i = 0; i < length; i++)
data[i] = start + i;
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
// Generate random int array of given length and distribution
public static uint[] GenerateUInt32(int length, string pattern = "uniform",
uint range = uint.MaxValue)
{
var data = new uint[length];
switch (pattern.ToLower())
{
case "uniform":
// Random values in [-range, range]
for (int i = 0; i < length; i++)
data[i] = (uint)rng.NextInt64(0, (long)range);
break;
case "small":
// Focused on small magnitudes to test ZigZag fast path
for (int i = 0; i < length; i++)
data[i] = (uint)rng.Next(0, 127);
break;
case "ascending":
{
uint start = (uint)rng.NextInt64(0, (long)range);
for (uint i = 0; i < length; i++)
data[i] = start + i;
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
// Generate random int array of given length and distribution
public static ulong[] GenerateUInt64(int length, string pattern = "uniform")
{
var data = new ulong[length];
switch (pattern.ToLower())
{
case "uniform":
// Random values in [-range, range]
for (int i = 0; i < length; i++)
data[i] = (ulong)rng.NextInt64();
break;
case "small":
// Focused on small magnitudes to test ZigZag fast path
for (int i = 0; i < length; i++)
data[i] = (uint)rng.Next(0, 127);
break;
case "ascending":
{
uint start = (uint)rng.NextInt64();
for (uint i = 0; i < length; i++)
data[i] = start + i;
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
public static uint[] GenerateUInt16(int length, string pattern = "uniform",
ushort range = ushort.MaxValue)
{
var data = new uint[length];
switch (pattern.ToLower())
{
case "uniform":
// Random values in [-range, range]
for (int i = 0; i < length; i++)
data[i] = (ushort)rng.Next(0, range);
break;
case "small":
// Focused on small magnitudes to test ZigZag fast path
for (int i = 0; i < length; i++)
data[i] = (uint)rng.Next(0, 127);
break;
case "ascending":
{
var start = (ushort)rng.Next(0, range);
for (uint i = 0; i < length; i++)
data[i] = start + i;
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
// Generate random int array of given length and distribution
public static long[] GenerateInt64(int length, string pattern = "uniform",
long range = long.MaxValue)
{
var data = new long[length];
switch (pattern.ToLower())
{
case "uniform":
// Random values in [-range, range]
for (int i = 0; i < length; i++)
data[i] = rng.NextInt64(-range, range);
break;
case "positive":
for (int i = 0; i < length; i++)
data[i] = rng.NextInt64(0, range);
break;
case "negative":
for (int i = 0; i < length; i++)
data[i] = -rng.NextInt64(0, range);
break;
case "alternating":
for (int i = 0; i < length; i++)
{
var val = rng.NextInt64(0, range);
data[i] = (i % 2 == 0) ? val : -val;
}
break;
case "small":
// Focused on small magnitudes to test ZigZag fast path
for (int i = 0; i < length; i++)
data[i] = rng.NextInt64(-64, 65);
break;
case "ascending":
{
var start = rng.NextInt64(-range, range);
for (int i = 0; i < length; i++)
data[i] = start + i;
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
public static short[] GenerateInt16(int length, string pattern = "uniform",
short range = short.MaxValue)
{
var data = new short[length];
switch (pattern.ToLower())
{
case "uniform":
for (int i = 0; i < length; i++)
data[i] = (short)rng.Next(-range, range + 1);
break;
case "positive":
for (int i = 0; i < length; i++)
data[i] = (short)rng.Next(0, range + 1);
break;
case "negative":
for (int i = 0; i < length; i++)
data[i] = (short)(-rng.Next(0, range + 1));
break;
case "alternating":
for (int i = 0; i < length; i++)
{
short val = (short)rng.Next(0, range + 1);
data[i] = (i % 2 == 0) ? val : (short)-val;
}
break;
case "small":
for (int i = 0; i < length; i++)
data[i] = (short)rng.Next(-64, 65);
break;
case "ascending":
{
short start = (short)rng.Next(-range, range);
for (int i = 0; i < length; i++)
data[i] = (short)(start + i);
}
break;
default:
throw new ArgumentException($"Unknown pattern: {pattern}");
}
return data;
}
}

View File

@@ -0,0 +1,335 @@
using Esiur.Data.GVWIE;
using FlatSharp;
using FlatSharp.Attributes;
using MessagePack;
using MongoDB.Bson;
using PeterO.Cbor;
using ProtoBuf;
using SolTechnology.Avro;
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Text;
namespace Esiur.Tests.Serialization
{
[FlatBufferTable]
public class ArrayRoot<T>
{
// Field index must be stable; start at 0
[FlatBufferItem(0)]
public virtual IList<T>? Values { get; set; }
}
internal class IntArrayRunner
{
public void Run()
{
Console.WriteLine(";Esiur;FlatBuffer;ProtoBuffer;MessagePack;BSON;CBOR;Avro,Optimal");
//Console.Write("Cluster (Int32);");
////CompareInt(int32cluster);
//Average(() => CompareInt(IntArrayGenerator.GenerateInt32Run(1000)), 1000);
Console.Write("Positive (Int32);");
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "positive")), 1000);
Console.Write("Negative (Int32);");
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "negative")), 1000);
Console.Write("Small (Int32);");
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "small")), 1000);
// CompareInt(int32small);
Console.Write("Alternating (Int32);");
//CompareInt(int32alter);
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "alternating")), 1000);
Console.Write("Ascending (Int32);");
//CompareInt(int32asc);
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "ascending")), 1000);
Console.Write("Int64;");
Average(() => CompareInt(IntArrayGenerator.GenerateInt64(1000, "uniform")), 1000);
//CompareInt(int64Uni);
Console.Write("Int32;");
//CompareInt(int32Uni);
Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "uniform")), 1000);
Console.Write("Int16;");
//CompareInt(int16Uni);
Average(() => CompareInt(IntArrayGenerator.GenerateInt16(1000, "uniform")), 1000);
Console.Write("UInt64;");
//CompareInt(uint64Uni);
Average(() => CompareInt(IntArrayGenerator.GenerateUInt64(1000, "uniform")), 1000);
Console.Write("UInt32;");
//CompareInt(uint32Uni);
Average(() => CompareInt(IntArrayGenerator.GenerateUInt32(1000, "uniform")), 1000);
Console.Write("UInt16;");
//CompareInt(uint16Uni);
Average(() => CompareInt(IntArrayGenerator.GenerateUInt16(1000, "uniform")), 1000);
}
public static (int, int, int, int, int, int, int, int) CompareInt(long[] sample)
{
var intRoot = new ArrayRoot<long>() { Values = sample };
var esiur = GroupInt64Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var bson = intRoot.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
//var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray());
//var ans1 = seq.GetDerEncoded();
var avro = AvroConvert.Serialize(sample);
var optimal = OptimalSignedEnocding(sample);
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static (int, int, int, int, int, int, int, int) CompareInt(int[] sample)
{
var intRoot = new ArrayRoot<int>() { Values = sample };
var esiur = GroupInt32Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var bson = intRoot.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
//var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray());
//var ans1 = seq.GetDerEncoded();
var avro = AvroConvert.Serialize(sample);
var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray());
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static (int, int, int, int, int, int, int, int) CompareInt(short[] sample)
{
var intRoot = new ArrayRoot<short>() { Values = sample };
var esiur = GroupInt16Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var bson = intRoot.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
//var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray());
//var ans1 = seq.GetDerEncoded();
var avro = AvroConvert.Serialize(sample);
var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray());
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static (int, int, int, int, int, int, int, int) CompareInt(uint[] sample)
{
var intRoot = new ArrayRoot<uint>() { Values = sample };
var esiur = GroupUInt32Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var intRoot2 = new ArrayRoot<int>() { Values = sample.Select(x => (int)x).ToArray() };
var bson = intRoot2.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
var avro = AvroConvert.Serialize(sample.Select(x => (int)x).ToArray());
//var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray());
//var avro = seq.GetDerEncoded();
var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray());
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static (int, int, int, int, int, int, int, int) CompareInt(ulong[] sample)
{
var intRoot = new ArrayRoot<ulong>() { Values = sample };
var esiur = GroupUInt64Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var bson = intRoot.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
//var seq = new DerSequence(sample.Select(v => new DerInteger((long)v)).ToArray());
//var ans1 = seq.GetDerEncoded();
var avro = AvroConvert.Serialize(sample);
var optimal = OptimalUnsignedEnocding(sample);
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static (int, int, int, int, int, int, int, int) CompareInt(ushort[] sample)
{
var intRoot = new ArrayRoot<ushort>() { Values = sample };
var esiur = GroupUInt16Codec.Encode(sample);
var messagePack = MessagePackSerializer.Serialize(sample);
var flatBuffer = SerializeFlatBuffers(intRoot);
using var ms = new MemoryStream();
Serializer.Serialize(ms, sample);
var protoBuffer = ms.ToArray();
var bson = intRoot.ToBson();
var cbor = CBORObject.FromObject(intRoot).EncodeToBytes();
//var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray());
//var ans1 = seq.GetDerEncoded();
var avro = AvroConvert.Serialize(sample);
var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray());
//Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}");
return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal);
}
public static int OptimalSignedEnocding(long[] data)
{
var sum = 0;
foreach (var i in data)
if (i >= sbyte.MinValue && i <= sbyte.MaxValue)
sum += 1;
else if (i >= short.MinValue && i <= short.MaxValue)
sum += 2;
else if (i >= -8_388_608 && i <= 8_388_607)
sum += 3;
else if (i >= int.MinValue && i <= int.MaxValue)
sum += 4;
else if (i >= -549_755_813_888 && i <= 549_755_813_887)
sum += 5;
else if (i >= -140_737_488_355_328 && i <= 140_737_488_355_327)
sum += 6;
else if (i >= -36_028_797_018_963_968 && i <= 36_028_797_018_963_967)
sum += 7;
else if (i >= long.MinValue && i <= long.MaxValue)
sum += 8;
return sum;
}
public static int OptimalUnsignedEnocding(ulong[] data)
{
var sum = 0;
foreach (var i in data)
if (i <= byte.MaxValue)
sum += 1;
else if (i <= ushort.MaxValue)
sum += 2;
else if (i <= uint.MaxValue)
sum += 4;
else if (i <= 0xFF_FF_FF_FF_FF)
sum += 5;
else if (i <= 0xFF_FF_FF_FF_FF_FF)
sum += 6;
else if (i <= 0xFF_FF_FF_FF_FF_FF_FF)
sum += 7;
else if (i <= ulong.MaxValue)
sum += 8;
return sum;
}
static (double, double, double, double, double, double, double, double) Average(Func<(int, int, int, int, int, int, int, int)> call, int count)
{
var sum = new List<(int, int, int, int, int, int, int, int)>();
for (var i = 0; i < count; i++)
sum.Add(call());
var rt = (sum.Average(x => x.Item1),
sum.Average(x => x.Item2),
sum.Average(x => x.Item3),
sum.Average(x => x.Item4),
sum.Average(x => x.Item5),
sum.Average(x => x.Item6),
sum.Average(x => x.Item7),
sum.Average(x => x.Item8)
);
Console.WriteLine($"{rt.Item1};{rt.Item2};{rt.Item3};{rt.Item4};{rt.Item5};{rt.Item6};{rt.Item7};{rt.Item8}");
return rt;
}
public static byte[] SerializeFlatBuffers<T>(ArrayRoot<T> array)
{
var buffer = new byte[1000000000];
var len = FlatBufferSerializer.Default.Serialize(array, buffer);
return buffer.Take(len).ToArray();
}
}
}

View File

@@ -0,0 +1,597 @@
using System;
using System.Collections.Generic;
using ProtoBuf;
using MessagePack;
using FlatSharp.Attributes;
using Esiur.Data;
using Esiur.Resource;
namespace Esiur.Tests.Serialization;
#nullable enable
// ========================= Enums =========================
// (Optional) You can add [ProtoContract]/[ProtoEnum] if you want explicit enum numbering.
// FlatSharp works fine with standard C# enums.
[FlatBufferEnum(typeof(int))]
public enum Currency { USD, EUR, IQD, JPY, GBP }
[FlatBufferEnum(typeof(int))]
public enum DocType { Quote, Order, Invoice, CreditNote }
[FlatBufferEnum(typeof(int))]
public enum PaymentMethod { Cash, Card, Wire, Crypto, Other }
[FlatBufferEnum(typeof(int))]
public enum LineType { Product, Service, Discount, Shipping }
// ========================= Variant =========================
// NOTE (FlatBuffers): a structured union in .fbs is preferable.
// Here we annotate as requested; FlatSharp will compile but youd typically replace this with a union/table family.
[ProtoContract]
[MessagePackObject(true)] // keyAsPropertyName = true, to avoid manual [Key] on every field here
[FlatBufferTable]
[Export]
public class Variant : IRecord
{
[ProtoMember(1)]
[FlatBufferItem(0)]
public Kind Tag { get; set; }
[ProtoMember(2)]
[FlatBufferItem(1)]
public bool? Bool { get; set; }
[ProtoMember(3)]
[FlatBufferItem(2)]
public long? I64 { get; set; }
[ProtoMember(4)]
[FlatBufferItem(3)]
public ulong? U64 { get; set; }
[ProtoMember(5)]
[FlatBufferItem(4)]
public double? F64 { get; set; }
//[ProtoMember(6)]
//[FlatBufferItem(5)]
//public double? Dec { get; set; }
[ProtoMember(7)]
[FlatBufferItem(6)]
public string? Str { get; set; }
[ProtoMember(8)]
[FlatBufferItem(7)]
public byte[]? Bytes { get; set; }
[ProtoMember(9)]
public DateTime? Dt { get; set; }
[FlatBufferItem(8)]
[Ignore, IgnoreMember]
public long DtAsLong { get; set; }
[ProtoMember(10)]
[FlatBufferItem(9)]
public byte[]? Guid { get; set; }
[FlatBufferEnum(typeof(int))]
public enum Kind { Null, Bool, Int64, UInt64, Double, Decimal, String, Bytes, DateTime, Guid }
public override bool Equals(object? obj)
{
var other = obj as Variant;
if (other == null) return false;
if (other.I64 != I64) return false;
if (other.U64 != U64) return false;
if (other.Bool != Bool) return false;
//if (other.Dec != Dec) return false;
if (other.Str != Str) return false;
if (Guid != null)
if (!other.Guid.SequenceEqual(Guid)) return false;
if (other.F64 != F64) return false;
if (other.Tag != Tag) return false;
if (Bytes != null)
if (!other.Bytes.SequenceEqual(Bytes)) return false;
if (other.DtAsLong != DtAsLong)
return false;
if (other.Dt != Dt)
return false;
return true;
}
}
// ========================= Address =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class Address : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public string Line1 { get; set; } = "";
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public string? Line2 { get; set; }
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public string City { get; set; } = "";
[ProtoMember(4)]
[Key(3)]
[FlatBufferItem(3)]
public string Region { get; set; } = "";
[ProtoMember(5)]
[Key(4)]
[FlatBufferItem(4)]
public string Country { get; set; } = "IQ";
[ProtoMember(6)]
[Key(5)]
[FlatBufferItem(5)]
public string? PostalCode { get; set; }
public override bool Equals(object? obj)
{
var other = obj as Address;
if (other == null) return false;
if (other.Line1 != Line1) return false;
if (other.Line2 != Line2) return false;
if (other.PostalCode != PostalCode) return false;
if (other.City != City) return false;
if (other.Country != Country) return false;
if (other.Region != Region) return false;
return true;
}
}
// ========================= Party =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class Party : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public ulong Id { get; set; }
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public string Name { get; set; } = "";
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public string? TaxId { get; set; }
[ProtoMember(4)]
[Key(3)]
[FlatBufferItem(3)]
public string? Email { get; set; }
[ProtoMember(5)]
[Key(4)]
[FlatBufferItem(4)]
public string? Phone { get; set; }
[ProtoMember(6)]
[Key(5)]
[FlatBufferItem(5)]
public Address? Address { get; set; }
// v2 field
[ProtoMember(7)]
[Key(6)]
[FlatBufferItem(6)]
public string? PreferredLanguage { get; set; }
public override bool Equals(object? obj)
{
var other = obj as Party;
if (other == null) return false;
if (other.Id != Id) return false;
if (other.TaxId != TaxId) return false;
if (!other.Address.Equals(Address)) return false;
if (other.Email != Email) return false;
if (other.Name != Name) return false;
if (other.Phone != Phone) return false;
if (other.PreferredLanguage != PreferredLanguage) return false;
return true;
}
}
// ========================= LineItem =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class LineItem : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public int LineNo { get; set; }
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public LineType Type { get; set; }
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public string SKU { get; set; } = "";
[ProtoMember(4)]
[Key(3)]
[FlatBufferItem(3)]
public string Description { get; set; } = "";
[ProtoMember(5)]
[Key(4)]
[FlatBufferItem(4)]
public double Qty { get; set; }
//[Ignore, IgnoreMember]
//public double QtyAsDouble { get; set; }
[ProtoMember(6)]
[Key(5)]
[FlatBufferItem(5)]
public string QtyUnit { get; set; } = "pcs";
[ProtoMember(7)]
[Key(6)]
[FlatBufferItem(6)]
public double UnitPrice { get; set; }
[ProtoMember(8)]
[Key(7)]
[FlatBufferItem(7)]
public double? VatRate { get; set; }
// NOTE (FlatBuffers): Dictionary is not native. Consider mapping to a vector of {Key, Value(Variant)} entries for real FlatBuffers use.
[ProtoMember(9)]
[Key(8)]
public Dictionary<string, Variant>? Ext { get; set; }
// v2 field
[ProtoMember(10)]
[Key(9)]
[FlatBufferItem(8)]
public double? Discount { get; set; }
[FlatBufferItem(9), Ignore, IgnoreMember]
public string[]? ExtKeys { get; set; }
[FlatBufferItem(10), Ignore, IgnoreMember]
public Variant[]? ExtValues { get; set; }
public override bool Equals(object? obj)
{
var other = obj as LineItem;
if (other == null) return false;
if (other.LineNo != LineNo) return false;
if (other.SKU != SKU) return false;
if (other.Description != Description) return false;
if (other.Discount != Discount) return false;
if (other.QtyUnit != QtyUnit) return false;
if (other.Type != Type) return false;
if (other.VatRate != VatRate) return false;
if (other.UnitPrice != UnitPrice) return false;
if (other.ExtKeys == null)
other.ExtKeys = other.Ext.Keys.ToArray();
if (other.ExtValues == null)
other.ExtValues = other.Ext.Values.ToArray();
if (!other.ExtKeys.SequenceEqual(ExtKeys)) return false;
if (!other.ExtValues.SequenceEqual(ExtValues)) return false;
return true;
}
}
// ========================= Payment =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class Payment : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public PaymentMethod Method { get; set; }
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public double Amount { get; set; }
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public string? Reference { get; set; }
[ProtoMember(4)]
[Key(3)]
public DateTime Timestamp { get; set; }
[FlatBufferItem(3), Ignore, IgnoreMember]
public long TimestampAsLong { get; set; }
// v2 fields
[ProtoMember(5)]
[Key(4)]
[FlatBufferItem(4)]
public double? Fee { get; set; }
//[ProtoMember(6)]
//[Key(5)]
//[FlatBufferItem(5)]
//public Currency Currency { get; set; }
public override bool Equals(object? obj)
{
var other = obj as Payment;
if (other == null) return false;
if (Method != other.Method) return false;
if (Amount != other.Amount) return false;
if (Reference != other.Reference) return false;
//if (Timestamp != other.Timestamp) return false;
//if (TimestampAsLong != other.TimestampAsLong) return false;
if (Fee != other.Fee) return false;
//if (CurrencyOverride != other.CurrencyOverride) return false;
return true;
}
}
// ========================= Attachment =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class Attachment : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public string Name { get; set; } = "";
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public string MimeType { get; set; } = "application/pdf";
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public byte[] Data { get; set; } = Array.Empty<byte>();
public override bool Equals(object? obj)
{
var other = obj as Attachment;
if (Name != other.Name) return false;
if (MimeType != other.MimeType) return false;
if (!(Data.SequenceEqual(other.Data))) return false;
return true;
}
}
// ========================= DocumentHeader =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class DocumentHeader : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public byte[] DocId { get; set; }
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public DocType Type { get; set; }
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public int Version { get; set; }
[ProtoMember(4)]
[Key(3)]
public DateTime CreatedAt { get; set; }
[FlatBufferItem(3), Ignore, IgnoreMember]
public long CreatedAtAsLong
{
get => CreatedAt.Ticks;
set => CreatedAt = new DateTime(value);
}
[ProtoMember(5)]
[Key(4)]
public DateTime? UpdatedAt { get; set; }
[FlatBufferItem(4), Ignore, IgnoreMember]
public long? UpdatedAtAsLong
{
get => UpdatedAt?.Ticks ?? 0;
set => UpdatedAt = value == null || value == 0 ? null : new DateTime(value.Value);
}
[ProtoMember(6)]
[Key(5)]
[FlatBufferItem(5)]
public Currency Currency { get; set; }
[ProtoMember(7)]
[Key(6)]
[FlatBufferItem(6)]
public string? Notes { get; set; }
[ProtoMember(8)]
[Key(7)]
public Dictionary<string, Variant>? Meta { get; set; }
// FlatBuffers: don't support dictionary.
[FlatBufferItem(7), Ignore, IgnoreMember]
public string[] MetaKeys { get; set; }
[FlatBufferItem(8), Ignore, IgnoreMember]
public Variant[] MetaValues { get; set; }
public override bool Equals(object? obj)
{
var other = obj as DocumentHeader;
if (other == null) return false;
if (!DocId.SequenceEqual(other.DocId)) return false;
if (Type != other.Type) return false;
if (Version != other.Version) return false;
//if (CreatedAtAsLong != other.CreatedAtAsLong) return false;
//if (UpdatedAtAsLong != other.UpdatedAtAsLong) return false;
if (CreatedAt != other.CreatedAt) return false;
if (UpdatedAt != other.UpdatedAt) return false;
if (Currency != other.Currency) return false;
if (Notes != other.Notes) return false;
if (other.MetaKeys == null)
other.MetaKeys = other.Meta.Keys.ToArray();
if (other.MetaValues == null)
other.MetaValues = other.Meta.Values.ToArray();
if (!MetaKeys.SequenceEqual(other.MetaKeys)) return false;
if (!MetaValues.SequenceEqual(other.MetaValues)) return false;
return true;
}
}
// ========================= BusinessDocument (root) =========================
[ProtoContract]
[MessagePackObject]
[FlatBufferTable]
[Export]
public class BusinessDocument : IRecord
{
[ProtoMember(1)]
[Key(0)]
[FlatBufferItem(0)]
public DocumentHeader? Header { get; set; }
[ProtoMember(2)]
[Key(1)]
[FlatBufferItem(1)]
public Party? Seller { get; set; }
[ProtoMember(3)]
[Key(2)]
[FlatBufferItem(2)]
public Party? Buyer { get; set; }
[ProtoMember(4)]
[Key(3)]
[FlatBufferItem(3)]
public LineItem[]? Items { get; set; }
[ProtoMember(5)]
[Key(4)]
[FlatBufferItem(4)]
public Payment[]? Payments { get; set; }
[ProtoMember(6)]
[Key(5)]
[FlatBufferItem(5)]
public Attachment[]? Attachments { get; set; }
[ProtoMember(7)]
[Key(6)]
[FlatBufferItem(6)]
public int[]? RiskScores { get; set; }
public override bool Equals(object? obj)
{
var other = obj as BusinessDocument;
if (other == null)
return false;
if (!Header.Equals(other.Header))
return false;
if (!Seller.Equals(other.Seller))
return false;
if (!Buyer.Equals(other.Buyer))
return false;
if (Items != null)
for (var i = 0; i < Items.Length; i++)
if (!Items[i].Equals(other.Items[i]))
return false;
if (Payments != null)
for (var i = 0; i < Payments.Length; i++)
if (!Payments[i].Equals(other.Payments[i]))
return false;
if (Attachments != null)
for (var i = 0; i < Attachments.Length; i++)
if (!Attachments[i].Equals(other.Attachments[i]))
return false;
if (!RiskScores.SequenceEqual(other.RiskScores))
return false;
return true;
}
[FlatBufferTable]
public class ArrayRoot<T>
{
// Field index must be stable; start at 0
[FlatBufferItem(0)]
public virtual IList<T>? Values { get; set; }
}
}

View File

@@ -0,0 +1,372 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
#nullable enable
namespace Esiur.Tests.Serialization;
public static class ModelGenerator
{
public sealed class GenOptions
{
public int Lines { get; init; } = 20; // items count
public int Attachments { get; init; } = 0; // 0..N
public int AttachmentBytes { get; init; } = 0;// per attachment
public int Payments { get; init; } = 1; // 0..N
public bool IncludeV2Fields { get; init; } = false;
public bool IncludeUnicode { get; init; } = true; // Arabic/emoji sprinkled in
public int VariantPerLine { get; init; } = 1; // ad-hoc KV per line
public Currency Currency { get; init; } = Currency.USD;
public int RiskScores { get; set; } = 20;
public int Seed { get; init; } = 12345;
}
public static BusinessDocument MakeBusinessDocument(GenOptions? options = null)
{
var opt = options ?? new GenOptions();
var rng = new Random(opt.Seed);
var seller = MakeParty(rng, opt.IncludeV2Fields, isSeller: true, opt.IncludeUnicode);
var buyer = MakeParty(rng, opt.IncludeV2Fields, isSeller: false, opt.IncludeUnicode);
var createdAt = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24));
var doc = new BusinessDocument
{
Header = new DocumentHeader
{
DocId = Guid.NewGuid().ToByteArray(),
Type = (DocType)rng.Next(0, 4),
Version = 1,
CreatedAt = createdAt,
UpdatedAt = null,
Currency = opt.Currency,
Notes = opt.IncludeUnicode ? SampleNoteUnicode(rng) : SampleNoteAscii(rng),
Meta = new Dictionary<string, Variant>
{
["source"] = VStr("benchmark"),
["region"] = VStr("ME"),
["channel"] = VStr(rng.Next(0, 2) == 0 ? "online" : "pos"),
}
},
Seller = seller,
Buyer = buyer,
Items = new LineItem[opt.Lines],
Payments = opt.Payments > 0 ? new Payment[opt.Payments] : null,
Attachments = opt.Attachments > 0 ? new Attachment[opt.Attachments] : null,
RiskScores = RandomRiskScores(rng, opt.RiskScores),
//RelatedDocs_v2 = opt.IncludeV2Fields ? new List<Guid> { Guid.NewGuid(), Guid.NewGuid() } : null
};
doc.Header.MetaValues = doc.Header.Meta.Values.ToArray();
doc.Header.MetaKeys = doc.Header.Meta.Keys.ToArray();
// Items
for (int i = 0; i < opt.Lines; i++)
doc.Items[i] = MakeLineItem(rng, i + 1, opt.IncludeV2Fields, opt.VariantPerLine, opt.IncludeUnicode);
// Payments
if (doc.Payments != null)
{
var grand = doc.Items.Sum(L => L.UnitPrice * L.Qty * (double)(1.0 - (L.Discount ?? 0.0) / 100.0));
var remain = grand;
for (int i = 0; i < opt.Payments; i++)
{
var last = (i == opt.Payments - 1);
var amt = last ? remain : RoundMoney((double)rng.NextDouble() * remain * 0.7 + 1.0);
remain = Math.Max(0.0, remain - amt);
doc.Payments[i] = MakePayment(rng, amt, opt.IncludeV2Fields);
}
}
// Attachments
if (doc.Attachments != null)
{
for (int i = 0; i < opt.Attachments; i++)
doc.Attachments[i] = MakeAttachment(rng, i, opt.AttachmentBytes);
}
return doc;
}
/// <summary>
/// Create a slightly modified copy of an existing document to test delta/partial updates.
/// Changes: tweak 510% of line items (qty/price), add or edit a payment, and bump UpdatedAt.
/// </summary>
public static BusinessDocument MakeDelta(BusinessDocument v1, int seed, double changeRatio = 0.07)
{
var rng = new Random(seed);
var v2 = DeepClone(v1);
v2.Header.UpdatedAt = DateTime.UtcNow;
var toChange = Math.Max(1, (int)Math.Round(v2.Items.Length * changeRatio));
// change random lines
for (int i = 0; i < toChange; i++)
{
var idx = rng.Next(0, v2.Items.Length);
var li = v2.Items[idx];
li.Qty = RoundQty(li.Qty + (double)(rng.NextDouble() * 2.0 - 1.0)); // ±1
li.UnitPrice = RoundMoney(li.UnitPrice * (double)(0.95 + rng.NextDouble() * 0.1)); // ±5%
if (li.Ext == null) li.Ext = new Dictionary<string, Variant>();
li.Ext["lastEdit"] = VDate(DateTime.UtcNow);
li.ExtKeys = li.Ext.Keys.ToArray();
li.ExtValues = li.Ext.Values.ToArray();
}
if (v2.Payments == null || rng.Next(0, 3) == 0)
{
if (v2.Payments == null)
{
if (v2.Payments == null) v2.Payments = new Payment[1];
v2.Payments[0] = (MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true));
}
else
{
v2.Payments = v2.Payments.Append((MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true))).ToArray();
}
}
else
{
var p = v2.Payments[rng.Next(0, v2.Payments.Length)];
p.Fee = (p.Fee ?? 0.0) + 0.25;
p.Reference = "ADJ-" + rng.Next(10000, 99999).ToString(CultureInfo.InvariantCulture);
}
return v2;
}
// -------------------------- Builders --------------------------
private static int[] RandomRiskScores(Random rng, int count)
{
var rt = new int[count];// rng.Next(100, 1000)];
for (var i = 0; i < rt.Length; i++)
rt[i] = (int)rng.Next();
return rt;
}
private static Party MakeParty(Random rng, bool includeV2, bool isSeller, bool unicode)
{
return new Party
{
Id = (ulong)rng.NextInt64(), //Guid.NewGuid().ToByteArray(),
Name = unicode ? (isSeller ? "Delta Systems — دلتا" : "Client التجربة ✅") : (isSeller ? "Delta Systems" : "Client Demo"),
TaxId = isSeller ? $"TX-{rng.Next(100000, 999999)}" : null,
Email = (isSeller ? "sales" : "contact") + "@example.com",
Phone = "+964-7" + rng.Next(100000000, 999999999).ToString(CultureInfo.InvariantCulture),
Address = new Address
{
Line1 = rng.Next(0, 2) == 0 ? "Street 14" : "Tech Park",
City = "Baghdad",
Region = "BG",
Country = "IQ",
PostalCode = rng.Next(0, 2) == 0 ? "10001" : null
},
PreferredLanguage = includeV2 ? (rng.Next(0, 2) == 0 ? "ar-IQ" : "en-US") : null
};
}
private static LineItem MakeLineItem(Random rng, int lineNo, bool includeV2, int variantKvp, bool unicode)
{
var isProduct = rng.Next(0, 100) < 80;
var desc = unicode
? (isProduct ? $"وحدة قياس — Item {lineNo} 🔧" : $"خدمة دعم — Service {lineNo} ✨")
: (isProduct ? $"Item {lineNo}" : $"Service {lineNo}");
var li = new LineItem
{
LineNo = lineNo,
Type = isProduct ? LineType.Product : LineType.Service,
SKU = isProduct ? ("SKU-" + rng.Next(1000, 9999)) : "",
Description = desc,
Qty = RoundQty((double)(rng.NextDouble() * 9.0 + 1.0)), // 1..10
QtyUnit = isProduct ? "pcs" : "h",
UnitPrice = RoundMoney((double)(rng.NextDouble() * 90.0 + 10.0)),
VatRate = rng.Next(0, 100) < 80 ? 5.0 : (double?)null,
Ext = variantKvp > 0 ? new Dictionary<string, Variant>() : null,
Discount= includeV2 && rng.Next(0, 3) == 0 ? Math.Round(rng.NextDouble() * 10.0, 2) : (double?)null
};
if (li.Ext != null)
{
li.ExtKeys = li.Ext.Keys.ToArray();
li.ExtValues = li.Ext.Values.ToArray();
}
for (int i = 0; i < variantKvp; i++)
{
var key = i switch { 0 => "color", 1 => "size", 2 => "batch", _ => "attr" + i };
li.Ext!.TryAdd(key, i switch
{
0 => VStr(rng.Next(0, 3) switch { 0 => "red", 1 => "blue", _ => "green" }),
1 => VStr(rng.Next(0, 3) switch { 0 => "S", 1 => "M", _ => "L" }),
2 => VGuid(Guid.NewGuid()),
_ => VInt(rng.Next(0, 1000))
});
}
li.ExtValues = li.Ext.Values.ToArray();
li.ExtKeys = li.Ext.Keys.ToArray();
return li;
}
private static Payment MakePayment(Random rng, double amount, bool includeV2)
{
var p = new Payment
{
Method = (PaymentMethod)rng.Next(0, 5),
Amount = RoundMoney(amount),
Reference = "REF-" + rng.Next(100_000, 999_999),
Timestamp = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24)),
Fee = includeV2 && rng.Next(0, 2) == 0 ? RoundMoney((double)rng.NextDouble() * 2.0) : null,
//CurrencyOverride = includeV2 && rng.Next(0, 2) == 0 ? Currency.IQD : Currency.USD
};
p.TimestampAsLong = p.Timestamp.Ticks;
return p;
}
private static Attachment MakeAttachment(Random rng, int index, int bytes)
{
var arr = bytes > 0 ? new byte[bytes] : Array.Empty<byte>();
if (arr.Length > 0) rng.NextBytes(arr);
return new Attachment
{
Name = $"att-{index + 1}.bin",
MimeType = "application/octet-stream",
Data = arr
};
}
private static string SampleNoteUnicode(Random rng)
=> rng.Next(0, 2) == 0
? "ملاحظة: تم إنشاء هذا المستند لأغراض الاختبار 📦"
: "Note: synthetic benchmark document 🧪";
private static string SampleNoteAscii(Random rng)
=> rng.Next(0, 2) == 0 ? "Note: synthetic benchmark document" : "Internal use only";
// -------------------------- Variant helpers --------------------------
private static Variant VStr(string s) => new() { Tag = Variant.Kind.String, Str = s };
private static Variant VInt(int v) => new() { Tag = Variant.Kind.Int64, I64 = v };
private static Variant VGuid(Guid g) => new() { Tag = Variant.Kind.Guid, Guid = g.ToByteArray() };
private static Variant VDate(DateTime d) => new() { Tag = Variant.Kind.DateTime, Dt = d, DtAsLong = d.Ticks };
// -------------------------- Utils --------------------------
private static double RoundMoney(double v) => Math.Round(v, 2, MidpointRounding.AwayFromZero);
private static double RoundQty(double v) => Math.Round(v, 3, MidpointRounding.AwayFromZero);
/// <summary>
/// Simple deep clone via manual copy to stay serializer-agnostic.
/// (Good enough for benchmarks; switch to a fast serializer if you like.)
/// </summary>
private static BusinessDocument DeepClone(BusinessDocument s)
{
var d = new BusinessDocument
{
Header = new DocumentHeader
{
DocId = s.Header.DocId,
Type = s.Header.Type,
Version = s.Header.Version,
CreatedAt = s.Header.CreatedAt,
UpdatedAt = s.Header.UpdatedAt,
//CreatedAtAsLong = s.Header.CreatedAtAsLong,
//UpdatedAtAsLong = s.Header.UpdatedAtAsLong,
Currency = s.Header.Currency,
Notes = s.Header.Notes,
Meta = s.Header.Meta?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)),
MetaKeys = s.Header.MetaKeys.ToArray(),
MetaValues = s.Header.MetaValues.ToArray(),
},
Seller = CloneParty(s.Seller),
Buyer = CloneParty(s.Buyer),
Items = s.Items.Select(CloneLine).ToArray(),
Payments = s.Payments?.Select(ClonePayment).ToArray(),
Attachments = s.Attachments?.Select(CloneAttachment).ToArray(),
RiskScores = s.RiskScores,
//RelatedDocs_v2 = s.RelatedDocs_v2?.ToList()
};
return d;
}
private static Party CloneParty(Party p) => new()
{
Id = p.Id,
Name = p.Name,
TaxId = p.TaxId,
Email = p.Email,
Phone = p.Phone,
Address = p.Address is null ? null : new Address
{
Line1 = p.Address.Line1,
Line2 = p.Address.Line2,
City = p.Address.City,
Region = p.Address.Region,
Country = p.Address.Country,
PostalCode = p.Address.PostalCode
},
PreferredLanguage = p.PreferredLanguage
};
private static LineItem CloneLine(LineItem s) => new()
{
LineNo = s.LineNo,
Type = s.Type,
SKU = s.SKU,
Description = s.Description,
Qty = s.Qty,
QtyUnit = s.QtyUnit,
UnitPrice = s.UnitPrice,
VatRate = s.VatRate,
Ext = s.Ext?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)),
Discount = s.Discount,
ExtKeys = s.Ext?.Keys.ToArray(),
ExtValues = s.Ext?.Values.ToArray(),
};
private static Payment ClonePayment(Payment s) => new()
{
Method = s.Method,
Amount = s.Amount,
Reference = s.Reference,
Timestamp = s.Timestamp,
TimestampAsLong = s.TimestampAsLong,
Fee = s.Fee,
//CurrencyOverride= s.CurrencyOverride
};
private static Attachment CloneAttachment(Attachment s) => new()
{
Name = s.Name,
MimeType = s.MimeType,
Data = s.Data.ToArray()
};
private static Variant CloneVariant(Variant v) => new()
{
Tag = v.Tag,
Bool = v.Bool,
I64 = v.I64,
U64 = v.U64,
F64 = v.F64,
//Dec = v.Dec,
Str = v.Str,
Bytes = v.Bytes?.ToArray(),
Dt = v.Dt,
DtAsLong = v.DtAsLong,
Guid = v.Guid
};
}

View File

@@ -0,0 +1,484 @@
using Avro.Generic;
using Esiur.Resource;
using FlatSharp;
using MessagePack;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using PeterO.Cbor;
using ProtoBuf;
using SolTechnology.Avro;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
#nullable enable
namespace Esiur.Tests.Serialization;
public interface ICodec
{
string Name { get; }
byte[]? Serialize(BusinessDocument obj); // returns null on failure
BusinessDocument Deserialize(byte[] data);
}
public sealed class JsonCodec : ICodec
{
static readonly JsonSerializerOptions Opt = new()
{
WriteIndented = false,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never
};
public string Name => "JSON";
public byte[]? Serialize(BusinessDocument obj)
{
var data = JsonSerializer.SerializeToUtf8Bytes(obj, obj.GetType(), Opt);
return data;
}
public BusinessDocument Deserialize(byte[] data)
{
return JsonSerializer.Deserialize<BusinessDocument>(data)!;
}
}
public sealed class EsiurCodec : ICodec
{
public string Name => "Esiur";
public BusinessDocument Deserialize(byte[] data)
{
var (_, y) = Esiur.Data.Codec.ParseSync(data, 0, Warehouse.Default);
return (BusinessDocument)y!;
}
public byte[]? Serialize(BusinessDocument obj)
{
var rt = Esiur.Data.Codec.Compose(obj, Warehouse.Default, null);
return rt;
}
}
public sealed class MessagePackCodec : ICodec
{
public string Name => "MessagePack";
public BusinessDocument Deserialize(byte[] data)
{
return MessagePackSerializer.Deserialize<BusinessDocument>(data);
}
public byte[]? Serialize(BusinessDocument obj)
{
return MessagePackSerializer.Serialize(obj.GetType(), obj);
}
}
public sealed class ProtobufCodec : ICodec
{
public string Name => "Protobuf";
public BusinessDocument Deserialize(byte[] data)
{
var dst = Serializer.Deserialize<BusinessDocument>(new MemoryStream(data));
return dst;
}
public byte[]? Serialize(BusinessDocument obj)
{
using var ms = new MemoryStream();
Serializer.Serialize(ms, obj);
var rt = ms.ToArray();
// Single correctness check (outside timing loops)
var dst = Serializer.Deserialize<BusinessDocument>(new MemoryStream(rt));
if (!obj.Equals(dst))
throw new NotSupportedException("Protobuf roundtrip mismatch.");
return rt;
}
}
public sealed class FlatBuffersCodec : ICodec
{
public string Name => "FlatBuffers";
public BusinessDocument Deserialize(byte[] data)
{
var m = FlatBufferSerializer.Default.Parse<BusinessDocument>(data);
return m;
}
public byte[]? Serialize(BusinessDocument obj)
{
var buffer = new byte[1_000_000];
var count = FlatBufferSerializer.Default.Serialize(obj, buffer);
var msg = buffer.AsSpan(0, count).ToArray();
// Single correctness check (outside timing loops)
var m = FlatBufferSerializer.Default.Parse<BusinessDocument>(msg);
if (!m!.Equals(obj))
throw new Exception("FlatBuffers roundtrip mismatch.");
return msg;
}
}
public sealed class CborCodec : ICodec
{
public string Name => "CBOR";
public BusinessDocument Deserialize(byte[] data)
{
return CBORObject.DecodeObjectFromBytes<BusinessDocument>(data)!;
}
public byte[]? Serialize(BusinessDocument obj)
{
var c = CBORObject.FromObject(obj);
return c.EncodeToBytes();
}
}
public sealed class BsonCodec : ICodec
{
private static bool _init;
private static void EnsureMaps()
{
if (_init) return;
_init = true;
// Register class maps if needed; defaults usually work for POCOs.
}
public string Name => "BSON";
public byte[]? Serialize(BusinessDocument obj)
{
try
{
EnsureMaps();
using var ms = new MemoryStream();
using var writer = new MongoDB.Bson.IO.BsonBinaryWriter(ms);
var context = MongoDB.Bson.Serialization.BsonSerializationContext.CreateRoot(writer);
var args = new MongoDB.Bson.Serialization.BsonSerializationArgs(obj.GetType(), true, false);
var serializer = BsonSerializer.LookupSerializer(obj.GetType());
serializer.Serialize(context, args, obj);
return ms.ToArray();
}
catch { return null; }
}
public BusinessDocument Deserialize(byte[] data)
{
using var ms = new MemoryStream(data);
using var reader = new MongoDB.Bson.IO.BsonBinaryReader(ms);
var args = new MongoDB.Bson.Serialization.BsonSerializationArgs(typeof(BusinessDocument), true, false);
var context = MongoDB.Bson.Serialization.BsonDeserializationContext.CreateRoot(reader);
var serializer = BsonSerializer.LookupSerializer(typeof(BusinessDocument));
return (BusinessDocument)serializer.Deserialize(context)!;
}
}
public sealed class AvroCodec : ICodec
{
public string Name => "Avro";
public BusinessDocument Deserialize(byte[] data)
{
return AvroConvert.Deserialize<BusinessDocument>(data);
}
public byte[]? Serialize(BusinessDocument obj)
{
return AvroConvert.Serialize(obj);
}
}
// ------------------------- Stat helpers -------------------------
public static class Stats
{
public static double Mean(IReadOnlyList<long> xs) =>
xs.Count == 0 ? double.NaN : xs.Average();
public static double Median(IReadOnlyList<long> xs)
{
if (xs.Count == 0) return double.NaN;
var arr = xs.OrderBy(v => v).ToArray();
int n = arr.Length;
return n % 2 == 1 ? arr[n / 2] : (arr[n / 2 - 1] + arr[n / 2]) / 2.0;
}
public static string ClassifyVsJson(double ratio)
{
if (double.IsNaN(ratio)) return "N/A";
if (ratio <= 0.75) return "Smaller (≤0.75× JSON)";
if (ratio <= 1.25) return "Similar (~0.751.25×)";
return "Larger (≥1.25× JSON)";
}
}
// ------------------------- Workload config -------------------------
public enum Workload
{
Small, // ~5 lines, no attachments
Medium, // ~20 lines, 1 small attachment
Large, // ~100 lines, 3 x 64KB attachments
}
public sealed class WorkItem
{
public required string Name { get; init; }
public required BusinessDocument Payload { get; init; }
}
// ------------------------- CPU timing helpers -------------------------
public static class CpuTimer
{
// small warm-up to reduce JIT/first-use bias
public static void WarmUp(Action action, int rounds = 5)
{
for (int i = 0; i < rounds; i++) action();
}
// Measures process CPU time consumed by running `action` N times.
// Returns average microseconds per operation.
public static double MeasureAverageMicros(Action action, int rounds)
{
var proc = Process.GetCurrentProcess();
// GC before timing to reduce random interference (still CPU, but more stable)
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var start = proc.TotalProcessorTime;
for (int i = 0; i < rounds; i++) action();
var end = proc.TotalProcessorTime;
var delta = end - start;
var micros = delta.TotalMilliseconds * 1000.0;
return micros / rounds;
}
}
// ------------------------- Runner -------------------------
public sealed class ModelRunner
{
private readonly ICodec[] _codecs;
public ModelRunner()
{
_codecs = new ICodec[]
{
new JsonCodec(),
new EsiurCodec(),
new MessagePackCodec(),
new ProtobufCodec(),
new FlatBuffersCodec(),
new CborCodec(),
new BsonCodec(),
new AvroCodec()
};
}
record WorkloadResult
{
public string Name { get; init; } = "";
public List<long> Sizes { get; init; } = new();
public double EncodeCpuUsSum { get; set; } // sum of per-item avg CPU us/op
public double DecodeCpuUsSum { get; set; }
public int Samples { get; set; } // count of successful samples
}
// volatile sink to avoid aggressive JIT elimination in tight loops
private static volatile byte[]? _sinkBytes;
private static volatile BusinessDocument? _sinkDoc;
public void Run()
{
const int Rounds = 100;
var workloads = BuildWorkloads();
Console.WriteLine("=== Serialization Size & CPU (process) Benchmark ===");
Console.WriteLine($"Date (UTC): {DateTime.UtcNow:O}");
Console.WriteLine($"Rounds per op (CPU): {Rounds}");
Console.WriteLine();
foreach (var (wName, items) in workloads)
{
Console.WriteLine($"--- Workload: {wName} ---");
// Collect results: Codec -> result container
var results = new Dictionary<string, WorkloadResult>();
foreach (var c in _codecs) results[c.Name] = new WorkloadResult { Name = c.Name };
foreach (var item in items)
{
foreach (var c in _codecs)
{
try
{
// Single functional serialize to get bytes & verify equality ONCE (not in timed loop)
var bytes = c.Serialize(item.Payload);
if (bytes == null)
{
results[c.Name].Sizes.Add(long.MinValue);
continue;
}
var back = c.Deserialize(bytes);
if (!item.Payload.Equals(back))
throw new InvalidOperationException($"{c.Name} roundtrip inequality.");
results[c.Name].Sizes.Add(bytes.LongLength);
// ---- CPU timing ----
// Warm-up (tiny)
CpuTimer.WarmUp(() => { _sinkBytes = c.Serialize(item.Payload); }, 3);
CpuTimer.WarmUp(() => { _sinkDoc = c.Deserialize(bytes); }, 3);
// Measure serialize CPU (average µs/op over Rounds)
var encUs = CpuTimer.MeasureAverageMicros(() =>
{
_sinkBytes = c.Serialize(item.Payload);
}, Rounds);
// Measure deserialize CPU
var decUs = CpuTimer.MeasureAverageMicros(() =>
{
_sinkDoc = c.Deserialize(bytes);
}, Rounds);
results[c.Name].EncodeCpuUsSum += encUs;
results[c.Name].DecodeCpuUsSum += decUs;
results[c.Name].Samples += 1;
}
catch
{
// mark size failure for this sample if not already added
results[c.Name].Sizes.Add(long.MinValue);
}
}
}
// Compute stats, using only successful size samples
var jsonSizes = results["JSON"].Sizes.Where(v => v != long.MinValue).ToList();
var jsonMean = Stats.Mean(jsonSizes);
var jsonMed = Stats.Median(jsonSizes);
Console.WriteLine($"JSON mean: {jsonMean:F1} B, median: {jsonMed:F1} B");
Console.WriteLine();
Console.WriteLine("{0,-14} {1,12} {2,12} {3,10} {4,26} {5,18} {6,18}",
"Codec", "Mean(B)", "Median(B)", "Ratio", "Class vs JSON", "Enc CPU (µs)", "Dec CPU (µs)");
Console.WriteLine(new string('-', 118));
foreach (var c in _codecs)
{
var r = results[c.Name];
var okSizes = r.Sizes.Where(v => v != long.MinValue).ToList();
var mean = Stats.Mean(okSizes);
var med = Stats.Median(okSizes);
double ratio = double.NaN;
if (!double.IsNaN(mean) && !double.IsNaN(jsonMean) && jsonMean > 0) ratio = mean / jsonMean;
string cls = Stats.ClassifyVsJson(ratio);
string meanS = double.IsNaN(mean) ? "N/A" : mean.ToString("F1");
string medS = double.IsNaN(med) ? "N/A" : med.ToString("F1");
string ratioS = double.IsNaN(ratio) ? "N/A" : ratio.ToString("F3");
// average CPU µs/op across samples where serialization succeeded
string encCpuS = (r.Samples == 0) ? "N/A" : (r.EncodeCpuUsSum / r.Samples).ToString("F1");
string decCpuS = (r.Samples == 0) ? "N/A" : (r.DecodeCpuUsSum / r.Samples).ToString("F1");
Console.WriteLine("{0,-14} {1,12} {2,12} {3,10} {4,26} {5,18} {6,18}",
c.Name, meanS, medS, ratioS, cls, encCpuS, decCpuS);
}
Console.WriteLine();
Console.ReadLine();
}
}
private static List<(string, List<WorkItem>)> BuildWorkloads()
{
var result = new List<(string, List<WorkItem>)>();
// Small
{
var items = new List<WorkItem>();
for (int i = 0; i < 16; i++)
{
var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions
{
Lines = 5,
Payments = 3,
Attachments = 0,
IncludeV2Fields = (i % 2 == 0),
IncludeUnicode = true,
RiskScores = 500,
Seed = 1000 + i
});
items.Add(new WorkItem { Name = $"S-{i}", Payload = doc });
}
result.Add(("Small", items));
}
// Medium
{
var items = new List<WorkItem>();
for (int i = 0; i < 16; i++)
{
var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions
{
Lines = 20,
Payments = 5,
Attachments = 1,
AttachmentBytes = 8 * 1024,
IncludeV2Fields = (i % 3 == 0),
IncludeUnicode = true,
RiskScores = 1000,
Seed = 2000 + i
});
items.Add(new WorkItem { Name = $"M-{i}", Payload = doc });
}
result.Add(("Medium", items));
}
// Large
{
var items = new List<WorkItem>();
for (int i = 0; i < 12; i++)
{
var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions
{
Lines = 100,
Payments = 20,
Attachments = 3,
AttachmentBytes = 64 * 1024,
IncludeV2Fields = (i % 2 == 1),
IncludeUnicode = true,
RiskScores = 3000,
Seed = 3000 + i
});
items.Add(new WorkItem { Name = $"L-{i}", Payload = doc });
}
result.Add(("Large", items));
}
return result;
}
}

View File

@@ -0,0 +1,15 @@
using Esiur.Tests.Serialization;
using MessagePack;
MessagePack.MessagePackSerializer.DefaultOptions = MessagePackSerializerOptions.Standard
.WithCompression(MessagePackCompression.None); // optional; remove if you want raw size
var ints = new IntArrayRunner();
ints.Run();
var models = new ModelRunner();
models.Run();