diff --git a/Esiur.sln b/Esiur.sln index 606d6d0..db7886b 100644 --- a/Esiur.sln +++ b/Esiur.sln @@ -26,6 +26,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serialization", "Serializat EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AI", "AI", "{00E37E58-7E43-4D95-83E4-75F669E89584}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Esiur.Tests.Annotations", "Tests\AI\Annotations\Esiur.Tests.Annotations.csproj", "{E87F60C9-F167-3F03-A4BD-43DBB1C76BDD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Esiur.Tests.Functional", "Tests\Features\Functional\Esiur.Tests.Functional.csproj", "{9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Esiur.Tests.ComplexModel", "Tests\Serialization\ComplexObject\Esiur.Tests.ComplexModel.csproj", "{0255BB42-2742-59C6-F18D-42C6A7C0F017}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Esiur.Tests.GVWIE", "Tests\Serialization\GWVIE\Esiur.Tests.GVWIE.csproj", "{93B71253-8B62-38F4-7B0F-EFEE2619FF2F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -64,6 +72,22 @@ Global {7B0C521F-8B13-4F2A-BD78-7C692620C831}.Debug|Any CPU.Build.0 = Debug|Any CPU {7B0C521F-8B13-4F2A-BD78-7C692620C831}.Release|Any CPU.ActiveCfg = Release|Any CPU {7B0C521F-8B13-4F2A-BD78-7C692620C831}.Release|Any CPU.Build.0 = Release|Any CPU + {E87F60C9-F167-3F03-A4BD-43DBB1C76BDD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E87F60C9-F167-3F03-A4BD-43DBB1C76BDD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E87F60C9-F167-3F03-A4BD-43DBB1C76BDD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E87F60C9-F167-3F03-A4BD-43DBB1C76BDD}.Release|Any CPU.Build.0 = Release|Any CPU + {9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E}.Release|Any CPU.Build.0 = Release|Any CPU + {0255BB42-2742-59C6-F18D-42C6A7C0F017}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0255BB42-2742-59C6-F18D-42C6A7C0F017}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0255BB42-2742-59C6-F18D-42C6A7C0F017}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0255BB42-2742-59C6-F18D-42C6A7C0F017}.Release|Any CPU.Build.0 = Release|Any CPU + {93B71253-8B62-38F4-7B0F-EFEE2619FF2F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93B71253-8B62-38F4-7B0F-EFEE2619FF2F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93B71253-8B62-38F4-7B0F-EFEE2619FF2F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93B71253-8B62-38F4-7B0F-EFEE2619FF2F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -72,6 +96,10 @@ Global {6BFF5946-A829-4AD7-BFAB-80B71CD7FF49} = {2769C4C3-2595-413B-B7FE-5903826770C1} {7B4B39AD-B85C-4182-A385-F478A5AD28EC} = {2769C4C3-2595-413B-B7FE-5903826770C1} {00E37E58-7E43-4D95-83E4-75F669E89584} = {2769C4C3-2595-413B-B7FE-5903826770C1} + {E87F60C9-F167-3F03-A4BD-43DBB1C76BDD} = {00E37E58-7E43-4D95-83E4-75F669E89584} + {9BB3B5A1-CD1F-EEB6-89D5-F3D3766E740E} = {6BFF5946-A829-4AD7-BFAB-80B71CD7FF49} + {0255BB42-2742-59C6-F18D-42C6A7C0F017} = {7B4B39AD-B85C-4182-A385-F478A5AD28EC} + {93B71253-8B62-38F4-7B0F-EFEE2619FF2F} = {7B4B39AD-B85C-4182-A385-F478A5AD28EC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C584421D-5EC0-4821-B7D8-2633D8D405F2} diff --git a/Tests/Serialization/ComplexObject/Esiur.Tests.ComplexObject.csproj b/Tests/Serialization/ComplexObject/Esiur.Tests.ComplexModel.csproj similarity index 92% rename from Tests/Serialization/ComplexObject/Esiur.Tests.ComplexObject.csproj rename to Tests/Serialization/ComplexObject/Esiur.Tests.ComplexModel.csproj index 0b51025..526b326 100644 --- a/Tests/Serialization/ComplexObject/Esiur.Tests.ComplexObject.csproj +++ b/Tests/Serialization/ComplexObject/Esiur.Tests.ComplexModel.csproj @@ -19,7 +19,7 @@ - + diff --git a/Tests/Serialization/ComplexObject/IntArrayGenerator.cs b/Tests/Serialization/ComplexObject/IntArrayGenerator.cs deleted file mode 100644 index e31f608..0000000 --- a/Tests/Serialization/ComplexObject/IntArrayGenerator.cs +++ /dev/null @@ -1,353 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Esiur.Tests.Serialization; - -public static class IntArrayGenerator -{ - private static readonly Random rng = new Random(24241564); - - - public static long[] GenerateInt32Run(int length) - { - var data = new long[length]; - - int i = 0; - var inSmallRange = true; - var inShortRange = false; - var inLargeRange = false; - var inLongRange = false; - - long range = 30; - - while (i < length) - { - // stay same range - if (rng.NextDouble() < 0.9) - { - if (inSmallRange) - data[i++] = rng.Next(-64, 65); - else if (inShortRange) - data[i++] = rng.NextInt64(range - 100, range + 100); - else if (inLargeRange) - data[i++] = rng.NextInt64(range - 1000, range + 1000); - else if (inLongRange) - data[i++] = rng.NextInt64(range - 10000, range + 10000); - } - else - { - // switch range - var rand = rng.NextDouble(); - if (rand < 0.25) - { - inSmallRange = true; - inShortRange = false; - inLargeRange = false; - inLongRange = false; - data[i++] = rng.Next(-64, 65); - } - else if (rand < 0.50) - { - inSmallRange = false; - inShortRange = true; - inLargeRange = false; - inLongRange = false; - range = rng.NextInt64(1000, short.MaxValue); - data[i++] = rng.NextInt64(range - 100, range + 100); - } - else if (rand < 0.75) - { - inSmallRange = false; - inShortRange = false; - inLargeRange = true; - inLongRange = false; - range = rng.NextInt64(1000, int.MaxValue); - data[i++] = rng.NextInt64(range - 1000, range + 1000); - } - else - { - inSmallRange = false; - inShortRange = false; - inLargeRange = false; - inLongRange = true; - range = rng.NextInt64(10000, long.MaxValue); - data[i++] = rng.NextInt64(range - 10000, range + 10000); - - } - } - - } - - return data; - } - - // Generate random int array of given length and distribution - public static int[] GenerateInt32(int length, string pattern = "uniform", - int range = int.MaxValue) - { - var data = new int[length]; - - switch (pattern.ToLower()) - { - case "uniform": - // Random values in [-range, range] - for (int i = 0; i < length; i++) - data[i] = rng.Next(-range, range); - break; - - case "positive": - for (int i = 0; i < length; i++) - data[i] = rng.Next(0, range); - break; - - case "negative": - for (int i = 0; i < length; i++) - data[i] = -rng.Next(0, range); - break; - - case "alternating": - for (int i = 0; i < length; i++) - { - int val = rng.Next(0, range); - data[i] = (i % 2 == 0) ? val : -val; - } - break; - - case "small": - // Focused on small magnitudes to test ZigZag fast path - for (int i = 0; i < length; i++) - data[i] = rng.Next(-64, 65); - break; - - - case "ascending": - { - int start = rng.Next(-range, range); - for (int i = 0; i < length; i++) - data[i] = start + i; - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } - - - // Generate random int array of given length and distribution - public static uint[] GenerateUInt32(int length, string pattern = "uniform", - uint range = uint.MaxValue) - { - - var data = new uint[length]; - - switch (pattern.ToLower()) - { - case "uniform": - // Random values in [-range, range] - for (int i = 0; i < length; i++) - data[i] = (uint)rng.NextInt64(0, (long)range); - break; - - case "small": - // Focused on small magnitudes to test ZigZag fast path - for (int i = 0; i < length; i++) - data[i] = (uint)rng.Next(0, 127); - break; - - - case "ascending": - { - uint start = (uint)rng.NextInt64(0, (long)range); - for (uint i = 0; i < length; i++) - data[i] = start + i; - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } - - // Generate random int array of given length and distribution - public static ulong[] GenerateUInt64(int length, string pattern = "uniform") - { - var data = new ulong[length]; - - switch (pattern.ToLower()) - { - case "uniform": - // Random values in [-range, range] - for (int i = 0; i < length; i++) - data[i] = (ulong)rng.NextInt64(); - break; - - case "small": - // Focused on small magnitudes to test ZigZag fast path - for (int i = 0; i < length; i++) - data[i] = (uint)rng.Next(0, 127); - break; - - - case "ascending": - { - uint start = (uint)rng.NextInt64(); - for (uint i = 0; i < length; i++) - data[i] = start + i; - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } - - public static uint[] GenerateUInt16(int length, string pattern = "uniform", - ushort range = ushort.MaxValue) - { - var data = new uint[length]; - - switch (pattern.ToLower()) - { - case "uniform": - // Random values in [-range, range] - for (int i = 0; i < length; i++) - data[i] = (ushort)rng.Next(0, range); - break; - - case "small": - // Focused on small magnitudes to test ZigZag fast path - for (int i = 0; i < length; i++) - data[i] = (uint)rng.Next(0, 127); - break; - - - case "ascending": - { - var start = (ushort)rng.Next(0, range); - for (uint i = 0; i < length; i++) - data[i] = start + i; - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } - - // Generate random int array of given length and distribution - public static long[] GenerateInt64(int length, string pattern = "uniform", - long range = long.MaxValue) - { - var data = new long[length]; - - switch (pattern.ToLower()) - { - case "uniform": - // Random values in [-range, range] - for (int i = 0; i < length; i++) - data[i] = rng.NextInt64(-range, range); - break; - - case "positive": - for (int i = 0; i < length; i++) - data[i] = rng.NextInt64(0, range); - break; - - case "negative": - for (int i = 0; i < length; i++) - data[i] = -rng.NextInt64(0, range); - break; - - case "alternating": - for (int i = 0; i < length; i++) - { - var val = rng.NextInt64(0, range); - data[i] = (i % 2 == 0) ? val : -val; - } - break; - - case "small": - // Focused on small magnitudes to test ZigZag fast path - for (int i = 0; i < length; i++) - data[i] = rng.NextInt64(-64, 65); - break; - - - case "ascending": - { - var start = rng.NextInt64(-range, range); - for (int i = 0; i < length; i++) - data[i] = start + i; - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } - - public static short[] GenerateInt16(int length, string pattern = "uniform", - short range = short.MaxValue) - { - var data = new short[length]; - - switch (pattern.ToLower()) - { - case "uniform": - for (int i = 0; i < length; i++) - data[i] = (short)rng.Next(-range, range + 1); - break; - - case "positive": - for (int i = 0; i < length; i++) - data[i] = (short)rng.Next(0, range + 1); - break; - - case "negative": - for (int i = 0; i < length; i++) - data[i] = (short)(-rng.Next(0, range + 1)); - break; - - case "alternating": - for (int i = 0; i < length; i++) - { - short val = (short)rng.Next(0, range + 1); - data[i] = (i % 2 == 0) ? val : (short)-val; - } - break; - - case "small": - for (int i = 0; i < length; i++) - data[i] = (short)rng.Next(-64, 65); - break; - - - case "ascending": - { - short start = (short)rng.Next(-range, range); - for (int i = 0; i < length; i++) - data[i] = (short)(start + i); - } - break; - - default: - throw new ArgumentException($"Unknown pattern: {pattern}"); - } - - return data; - } -} \ No newline at end of file diff --git a/Tests/Serialization/ComplexObject/IntArrayRunner.cs b/Tests/Serialization/ComplexObject/IntArrayRunner.cs deleted file mode 100644 index cc97139..0000000 --- a/Tests/Serialization/ComplexObject/IntArrayRunner.cs +++ /dev/null @@ -1,335 +0,0 @@ -using Esiur.Data.GVWIE; -using FlatSharp; -using FlatSharp.Attributes; -using MessagePack; -using MongoDB.Bson; -using PeterO.Cbor; -using ProtoBuf; -using SolTechnology.Avro; -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Text; - -namespace Esiur.Tests.Serialization -{ - [FlatBufferTable] - public class ArrayRoot - { - // Field index must be stable; start at 0 - [FlatBufferItem(0)] - public virtual IList? Values { get; set; } - } - - internal class IntArrayRunner - { - public void Run() - { - Console.WriteLine(";Esiur;FlatBuffer;ProtoBuffer;MessagePack;BSON;CBOR;Avro,Optimal"); - - - - //Console.Write("Cluster (Int32);"); - ////CompareInt(int32cluster); - //Average(() => CompareInt(IntArrayGenerator.GenerateInt32Run(1000)), 1000); - - - Console.Write("Positive (Int32);"); - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "positive")), 1000); - - Console.Write("Negative (Int32);"); - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "negative")), 1000); - - - Console.Write("Small (Int32);"); - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "small")), 1000); - - // CompareInt(int32small); - - Console.Write("Alternating (Int32);"); - //CompareInt(int32alter); - - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "alternating")), 1000); - - - Console.Write("Ascending (Int32);"); - //CompareInt(int32asc); - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "ascending")), 1000); - - - Console.Write("Int64;"); - Average(() => CompareInt(IntArrayGenerator.GenerateInt64(1000, "uniform")), 1000); - //CompareInt(int64Uni); - - Console.Write("Int32;"); - //CompareInt(int32Uni); - Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "uniform")), 1000); - - Console.Write("Int16;"); - //CompareInt(int16Uni); - - Average(() => CompareInt(IntArrayGenerator.GenerateInt16(1000, "uniform")), 1000); - - - Console.Write("UInt64;"); - //CompareInt(uint64Uni); - Average(() => CompareInt(IntArrayGenerator.GenerateUInt64(1000, "uniform")), 1000); - - - Console.Write("UInt32;"); - //CompareInt(uint32Uni); - Average(() => CompareInt(IntArrayGenerator.GenerateUInt32(1000, "uniform")), 1000); - - Console.Write("UInt16;"); - //CompareInt(uint16Uni); - Average(() => CompareInt(IntArrayGenerator.GenerateUInt16(1000, "uniform")), 1000); - - } - - public static (int, int, int, int, int, int, int, int) CompareInt(long[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupInt64Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var bson = intRoot.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); - //var ans1 = seq.GetDerEncoded(); - - - var avro = AvroConvert.Serialize(sample); - - var optimal = OptimalSignedEnocding(sample); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - - } - - public static (int, int, int, int, int, int, int, int) CompareInt(int[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupInt32Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var bson = intRoot.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); - //var ans1 = seq.GetDerEncoded(); - - - var avro = AvroConvert.Serialize(sample); - - var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray()); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - - } - - - public static (int, int, int, int, int, int, int, int) CompareInt(short[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupInt16Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var bson = intRoot.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); - //var ans1 = seq.GetDerEncoded(); - - var avro = AvroConvert.Serialize(sample); - - var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray()); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - - } - - public static (int, int, int, int, int, int, int, int) CompareInt(uint[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupUInt32Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var intRoot2 = new ArrayRoot() { Values = sample.Select(x => (int)x).ToArray() }; - - var bson = intRoot2.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - - - var avro = AvroConvert.Serialize(sample.Select(x => (int)x).ToArray()); - - //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); - //var avro = seq.GetDerEncoded(); - - var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray()); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - - } - - public static (int, int, int, int, int, int, int, int) CompareInt(ulong[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupUInt64Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var bson = intRoot.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - //var seq = new DerSequence(sample.Select(v => new DerInteger((long)v)).ToArray()); - //var ans1 = seq.GetDerEncoded(); - - var avro = AvroConvert.Serialize(sample); - - - var optimal = OptimalUnsignedEnocding(sample); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - } - - public static (int, int, int, int, int, int, int, int) CompareInt(ushort[] sample) - { - var intRoot = new ArrayRoot() { Values = sample }; - - var esiur = GroupUInt16Codec.Encode(sample); - var messagePack = MessagePackSerializer.Serialize(sample); - var flatBuffer = SerializeFlatBuffers(intRoot); - - using var ms = new MemoryStream(); - Serializer.Serialize(ms, sample); - var protoBuffer = ms.ToArray(); - - var bson = intRoot.ToBson(); - - var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); - //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); - //var ans1 = seq.GetDerEncoded(); - - var avro = AvroConvert.Serialize(sample); - - var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray()); - //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); - return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); - - } - - public static int OptimalSignedEnocding(long[] data) - { - var sum = 0; - - foreach (var i in data) - if (i >= sbyte.MinValue && i <= sbyte.MaxValue) - sum += 1; - else if (i >= short.MinValue && i <= short.MaxValue) - sum += 2; - else if (i >= -8_388_608 && i <= 8_388_607) - sum += 3; - else if (i >= int.MinValue && i <= int.MaxValue) - sum += 4; - else if (i >= -549_755_813_888 && i <= 549_755_813_887) - sum += 5; - else if (i >= -140_737_488_355_328 && i <= 140_737_488_355_327) - sum += 6; - else if (i >= -36_028_797_018_963_968 && i <= 36_028_797_018_963_967) - sum += 7; - else if (i >= long.MinValue && i <= long.MaxValue) - sum += 8; - - return sum; - } - - public static int OptimalUnsignedEnocding(ulong[] data) - { - var sum = 0; - - foreach (var i in data) - if (i <= byte.MaxValue) - sum += 1; - else if (i <= ushort.MaxValue) - sum += 2; - else if (i <= uint.MaxValue) - sum += 4; - else if (i <= 0xFF_FF_FF_FF_FF) - sum += 5; - else if (i <= 0xFF_FF_FF_FF_FF_FF) - sum += 6; - else if (i <= 0xFF_FF_FF_FF_FF_FF_FF) - sum += 7; - else if (i <= ulong.MaxValue) - sum += 8; - - return sum; - } - - - static (double, double, double, double, double, double, double, double) Average(Func<(int, int, int, int, int, int, int, int)> call, int count) - { - var sum = new List<(int, int, int, int, int, int, int, int)>(); - - for (var i = 0; i < count; i++) - sum.Add(call()); - - - var rt = (sum.Average(x => x.Item1), - sum.Average(x => x.Item2), - sum.Average(x => x.Item3), - sum.Average(x => x.Item4), - sum.Average(x => x.Item5), - sum.Average(x => x.Item6), - sum.Average(x => x.Item7), - sum.Average(x => x.Item8) - ); - - Console.WriteLine($"{rt.Item1};{rt.Item2};{rt.Item3};{rt.Item4};{rt.Item5};{rt.Item6};{rt.Item7};{rt.Item8}"); - - return rt; - } - - - public static byte[] SerializeFlatBuffers(ArrayRoot array) - { - var buffer = new byte[1000000000]; - var len = FlatBufferSerializer.Default.Serialize(array, buffer); - return buffer.Take(len).ToArray(); - } - - } -} diff --git a/Tests/Serialization/ComplexObject/Program.cs b/Tests/Serialization/ComplexObject/Program.cs index 9daca1e..ac5f912 100644 --- a/Tests/Serialization/ComplexObject/Program.cs +++ b/Tests/Serialization/ComplexObject/Program.cs @@ -6,8 +6,6 @@ MessagePack.MessagePackSerializer.DefaultOptions = MessagePackSerializerOptions. .WithCompression(MessagePackCompression.None); // optional; remove if you want raw size -var ints = new IntArrayRunner(); -ints.Run(); var models = new ModelRunner(); models.Run(); diff --git a/Tests/Serialization/GWVIE/Model.cs b/Tests/Serialization/GWVIE/Model.cs deleted file mode 100644 index a7f9372..0000000 --- a/Tests/Serialization/GWVIE/Model.cs +++ /dev/null @@ -1,597 +0,0 @@ -using System; -using System.Collections.Generic; -using ProtoBuf; -using MessagePack; -using FlatSharp.Attributes; -using Esiur.Data; -using Esiur.Resource; - -namespace Esiur.Tests.Serialization; - -#nullable enable - -// ========================= Enums ========================= -// (Optional) You can add [ProtoContract]/[ProtoEnum] if you want explicit enum numbering. -// FlatSharp works fine with standard C# enums. - -[FlatBufferEnum(typeof(int))] -public enum Currency { USD, EUR, IQD, JPY, GBP } -[FlatBufferEnum(typeof(int))] -public enum DocType { Quote, Order, Invoice, CreditNote } -[FlatBufferEnum(typeof(int))] -public enum PaymentMethod { Cash, Card, Wire, Crypto, Other } -[FlatBufferEnum(typeof(int))] -public enum LineType { Product, Service, Discount, Shipping } - -// ========================= Variant ========================= -// NOTE (FlatBuffers): a structured union in .fbs is preferable. -// Here we annotate as requested; FlatSharp will compile but you’d typically replace this with a union/table family. - -[ProtoContract] -[MessagePackObject(true)] // keyAsPropertyName = true, to avoid manual [Key] on every field here -[FlatBufferTable] -[Export] -public class Variant : IRecord -{ - [ProtoMember(1)] - [FlatBufferItem(0)] - public Kind Tag { get; set; } - - [ProtoMember(2)] - [FlatBufferItem(1)] - public bool? Bool { get; set; } - - [ProtoMember(3)] - [FlatBufferItem(2)] - public long? I64 { get; set; } - - [ProtoMember(4)] - [FlatBufferItem(3)] - public ulong? U64 { get; set; } - - [ProtoMember(5)] - [FlatBufferItem(4)] - public double? F64 { get; set; } - - //[ProtoMember(6)] - //[FlatBufferItem(5)] - //public double? Dec { get; set; } - - [ProtoMember(7)] - [FlatBufferItem(6)] - public string? Str { get; set; } - - [ProtoMember(8)] - [FlatBufferItem(7)] - public byte[]? Bytes { get; set; } - - [ProtoMember(9)] - public DateTime? Dt { get; set; } - - [FlatBufferItem(8)] - [Ignore, IgnoreMember] - public long DtAsLong { get; set; } - - [ProtoMember(10)] - [FlatBufferItem(9)] - public byte[]? Guid { get; set; } - - [FlatBufferEnum(typeof(int))] - public enum Kind { Null, Bool, Int64, UInt64, Double, Decimal, String, Bytes, DateTime, Guid } - - public override bool Equals(object? obj) - { - var other = obj as Variant; - if (other == null) return false; - - if (other.I64 != I64) return false; - if (other.U64 != U64) return false; - if (other.Bool != Bool) return false; - //if (other.Dec != Dec) return false; - if (other.Str != Str) return false; - if (Guid != null) - if (!other.Guid.SequenceEqual(Guid)) return false; - if (other.F64 != F64) return false; - if (other.Tag != Tag) return false; - if (Bytes != null) - if (!other.Bytes.SequenceEqual(Bytes)) return false; - - if (other.DtAsLong != DtAsLong) - return false; - if (other.Dt != Dt) - return false; - - return true; - } -} - -// ========================= Address ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class Address : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public string Line1 { get; set; } = ""; - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public string? Line2 { get; set; } - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public string City { get; set; } = ""; - - [ProtoMember(4)] - [Key(3)] - [FlatBufferItem(3)] - public string Region { get; set; } = ""; - - [ProtoMember(5)] - [Key(4)] - [FlatBufferItem(4)] - public string Country { get; set; } = "IQ"; - - [ProtoMember(6)] - [Key(5)] - [FlatBufferItem(5)] - public string? PostalCode { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as Address; - if (other == null) return false; - if (other.Line1 != Line1) return false; - if (other.Line2 != Line2) return false; - if (other.PostalCode != PostalCode) return false; - if (other.City != City) return false; - if (other.Country != Country) return false; - if (other.Region != Region) return false; - - return true; - } -} - -// ========================= Party ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class Party : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public ulong Id { get; set; } - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public string Name { get; set; } = ""; - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public string? TaxId { get; set; } - - [ProtoMember(4)] - [Key(3)] - [FlatBufferItem(3)] - public string? Email { get; set; } - - [ProtoMember(5)] - [Key(4)] - [FlatBufferItem(4)] - public string? Phone { get; set; } - - [ProtoMember(6)] - [Key(5)] - [FlatBufferItem(5)] - public Address? Address { get; set; } - - // v2 field - [ProtoMember(7)] - [Key(6)] - [FlatBufferItem(6)] - public string? PreferredLanguage { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as Party; - if (other == null) return false; - - if (other.Id != Id) return false; - - if (other.TaxId != TaxId) return false; - if (!other.Address.Equals(Address)) return false; - if (other.Email != Email) return false; - if (other.Name != Name) return false; - if (other.Phone != Phone) return false; - if (other.PreferredLanguage != PreferredLanguage) return false; - - return true; - } -} - -// ========================= LineItem ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class LineItem : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public int LineNo { get; set; } - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public LineType Type { get; set; } - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public string SKU { get; set; } = ""; - - [ProtoMember(4)] - [Key(3)] - [FlatBufferItem(3)] - public string Description { get; set; } = ""; - - [ProtoMember(5)] - [Key(4)] - [FlatBufferItem(4)] - - public double Qty { get; set; } - - //[Ignore, IgnoreMember] - //public double QtyAsDouble { get; set; } - - [ProtoMember(6)] - [Key(5)] - [FlatBufferItem(5)] - public string QtyUnit { get; set; } = "pcs"; - - [ProtoMember(7)] - [Key(6)] - [FlatBufferItem(6)] - public double UnitPrice { get; set; } - - [ProtoMember(8)] - [Key(7)] - [FlatBufferItem(7)] - public double? VatRate { get; set; } - - // NOTE (FlatBuffers): Dictionary is not native. Consider mapping to a vector of {Key, Value(Variant)} entries for real FlatBuffers use. - [ProtoMember(9)] - [Key(8)] - public Dictionary? Ext { get; set; } - - // v2 field - [ProtoMember(10)] - [Key(9)] - [FlatBufferItem(8)] - public double? Discount { get; set; } - - [FlatBufferItem(9), Ignore, IgnoreMember] - public string[]? ExtKeys { get; set; } - - [FlatBufferItem(10), Ignore, IgnoreMember] - public Variant[]? ExtValues { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as LineItem; - if (other == null) return false; - if (other.LineNo != LineNo) return false; - if (other.SKU != SKU) return false; - if (other.Description != Description) return false; - if (other.Discount != Discount) return false; - if (other.QtyUnit != QtyUnit) return false; - if (other.Type != Type) return false; - if (other.VatRate != VatRate) return false; - if (other.UnitPrice != UnitPrice) return false; - - if (other.ExtKeys == null) - other.ExtKeys = other.Ext.Keys.ToArray(); - - if (other.ExtValues == null) - other.ExtValues = other.Ext.Values.ToArray(); - - - if (!other.ExtKeys.SequenceEqual(ExtKeys)) return false; - if (!other.ExtValues.SequenceEqual(ExtValues)) return false; - - return true; - } -} - -// ========================= Payment ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class Payment : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public PaymentMethod Method { get; set; } - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public double Amount { get; set; } - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public string? Reference { get; set; } - - [ProtoMember(4)] - [Key(3)] - public DateTime Timestamp { get; set; } - - [FlatBufferItem(3), Ignore, IgnoreMember] - public long TimestampAsLong { get; set; } - - // v2 fields - [ProtoMember(5)] - [Key(4)] - [FlatBufferItem(4)] - public double? Fee { get; set; } - - //[ProtoMember(6)] - //[Key(5)] - //[FlatBufferItem(5)] - //public Currency Currency { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as Payment; - - if (other == null) return false; - - if (Method != other.Method) return false; - if (Amount != other.Amount) return false; - if (Reference != other.Reference) return false; - //if (Timestamp != other.Timestamp) return false; - //if (TimestampAsLong != other.TimestampAsLong) return false; - if (Fee != other.Fee) return false; - //if (CurrencyOverride != other.CurrencyOverride) return false; - - return true; - } -} - -// ========================= Attachment ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class Attachment : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public string Name { get; set; } = ""; - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public string MimeType { get; set; } = "application/pdf"; - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public byte[] Data { get; set; } = Array.Empty(); - - public override bool Equals(object? obj) - { - var other = obj as Attachment; - if (Name != other.Name) return false; - if (MimeType != other.MimeType) return false; - if (!(Data.SequenceEqual(other.Data))) return false; - - return true; - } -} - -// ========================= DocumentHeader ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class DocumentHeader : IRecord -{ - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public byte[] DocId { get; set; } - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public DocType Type { get; set; } - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public int Version { get; set; } - - [ProtoMember(4)] - [Key(3)] - public DateTime CreatedAt { get; set; } - - [FlatBufferItem(3), Ignore, IgnoreMember] - public long CreatedAtAsLong - { - get => CreatedAt.Ticks; - set => CreatedAt = new DateTime(value); - } - - [ProtoMember(5)] - [Key(4)] - public DateTime? UpdatedAt { get; set; } - - [FlatBufferItem(4), Ignore, IgnoreMember] - public long? UpdatedAtAsLong - { - get => UpdatedAt?.Ticks ?? 0; - set => UpdatedAt = value == null || value == 0 ? null : new DateTime(value.Value); - } - - - [ProtoMember(6)] - [Key(5)] - [FlatBufferItem(5)] - public Currency Currency { get; set; } - - [ProtoMember(7)] - [Key(6)] - [FlatBufferItem(6)] - public string? Notes { get; set; } - - [ProtoMember(8)] - [Key(7)] - public Dictionary? Meta { get; set; } - - // FlatBuffers: don't support dictionary. - [FlatBufferItem(7), Ignore, IgnoreMember] - public string[] MetaKeys { get; set; } - [FlatBufferItem(8), Ignore, IgnoreMember] - public Variant[] MetaValues { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as DocumentHeader; - - if (other == null) return false; - if (!DocId.SequenceEqual(other.DocId)) return false; - if (Type != other.Type) return false; - if (Version != other.Version) return false; - //if (CreatedAtAsLong != other.CreatedAtAsLong) return false; - //if (UpdatedAtAsLong != other.UpdatedAtAsLong) return false; - if (CreatedAt != other.CreatedAt) return false; - if (UpdatedAt != other.UpdatedAt) return false; - - if (Currency != other.Currency) return false; - if (Notes != other.Notes) return false; - - if (other.MetaKeys == null) - other.MetaKeys = other.Meta.Keys.ToArray(); - - if (other.MetaValues == null) - other.MetaValues = other.Meta.Values.ToArray(); - - if (!MetaKeys.SequenceEqual(other.MetaKeys)) return false; - if (!MetaValues.SequenceEqual(other.MetaValues)) return false; - - - return true; - } -} - -// ========================= BusinessDocument (root) ========================= - -[ProtoContract] -[MessagePackObject] -[FlatBufferTable] -[Export] -public class BusinessDocument : IRecord -{ - - [ProtoMember(1)] - [Key(0)] - [FlatBufferItem(0)] - public DocumentHeader? Header { get; set; } - - [ProtoMember(2)] - [Key(1)] - [FlatBufferItem(1)] - public Party? Seller { get; set; } - - [ProtoMember(3)] - [Key(2)] - [FlatBufferItem(2)] - public Party? Buyer { get; set; } - - [ProtoMember(4)] - [Key(3)] - [FlatBufferItem(3)] - public LineItem[]? Items { get; set; } - - [ProtoMember(5)] - [Key(4)] - [FlatBufferItem(4)] - public Payment[]? Payments { get; set; } - - [ProtoMember(6)] - [Key(5)] - [FlatBufferItem(5)] - public Attachment[]? Attachments { get; set; } - - [ProtoMember(7)] - [Key(6)] - [FlatBufferItem(6)] - public int[]? RiskScores { get; set; } - - public override bool Equals(object? obj) - { - var other = obj as BusinessDocument; - if (other == null) - return false; - - - if (!Header.Equals(other.Header)) - return false; - if (!Seller.Equals(other.Seller)) - return false; - if (!Buyer.Equals(other.Buyer)) - return false; - - if (Items != null) - for (var i = 0; i < Items.Length; i++) - if (!Items[i].Equals(other.Items[i])) - return false; - - if (Payments != null) - for (var i = 0; i < Payments.Length; i++) - if (!Payments[i].Equals(other.Payments[i])) - return false; - - if (Attachments != null) - for (var i = 0; i < Attachments.Length; i++) - if (!Attachments[i].Equals(other.Attachments[i])) - return false; - - if (!RiskScores.SequenceEqual(other.RiskScores)) - return false; - - return true; - } - - - [FlatBufferTable] - public class ArrayRoot - { - // Field index must be stable; start at 0 - [FlatBufferItem(0)] - public virtual IList? Values { get; set; } - } - - -} diff --git a/Tests/Serialization/GWVIE/ModelGenerator.cs b/Tests/Serialization/GWVIE/ModelGenerator.cs deleted file mode 100644 index 777c6cc..0000000 --- a/Tests/Serialization/GWVIE/ModelGenerator.cs +++ /dev/null @@ -1,372 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; - -#nullable enable -namespace Esiur.Tests.Serialization; - -public static class ModelGenerator -{ - public sealed class GenOptions - { - public int Lines { get; init; } = 20; // items count - public int Attachments { get; init; } = 0; // 0..N - public int AttachmentBytes { get; init; } = 0;// per attachment - public int Payments { get; init; } = 1; // 0..N - public bool IncludeV2Fields { get; init; } = false; - public bool IncludeUnicode { get; init; } = true; // Arabic/emoji sprinkled in - public int VariantPerLine { get; init; } = 1; // ad-hoc KV per line - public Currency Currency { get; init; } = Currency.USD; - - public int RiskScores { get; set; } = 20; - public int Seed { get; init; } = 12345; - } - - public static BusinessDocument MakeBusinessDocument(GenOptions? options = null) - { - var opt = options ?? new GenOptions(); - var rng = new Random(opt.Seed); - - var seller = MakeParty(rng, opt.IncludeV2Fields, isSeller: true, opt.IncludeUnicode); - var buyer = MakeParty(rng, opt.IncludeV2Fields, isSeller: false, opt.IncludeUnicode); - - var createdAt = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24)); - var doc = new BusinessDocument - { - Header = new DocumentHeader - { - DocId = Guid.NewGuid().ToByteArray(), - Type = (DocType)rng.Next(0, 4), - Version = 1, - CreatedAt = createdAt, - UpdatedAt = null, - Currency = opt.Currency, - Notes = opt.IncludeUnicode ? SampleNoteUnicode(rng) : SampleNoteAscii(rng), - Meta = new Dictionary - { - ["source"] = VStr("benchmark"), - ["region"] = VStr("ME"), - ["channel"] = VStr(rng.Next(0, 2) == 0 ? "online" : "pos"), - } - }, - Seller = seller, - Buyer = buyer, - Items = new LineItem[opt.Lines], - Payments = opt.Payments > 0 ? new Payment[opt.Payments] : null, - Attachments = opt.Attachments > 0 ? new Attachment[opt.Attachments] : null, - - RiskScores = RandomRiskScores(rng, opt.RiskScores), - //RelatedDocs_v2 = opt.IncludeV2Fields ? new List { Guid.NewGuid(), Guid.NewGuid() } : null - }; - - doc.Header.MetaValues = doc.Header.Meta.Values.ToArray(); - doc.Header.MetaKeys = doc.Header.Meta.Keys.ToArray(); - - // Items - for (int i = 0; i < opt.Lines; i++) - doc.Items[i] = MakeLineItem(rng, i + 1, opt.IncludeV2Fields, opt.VariantPerLine, opt.IncludeUnicode); - - // Payments - if (doc.Payments != null) - { - var grand = doc.Items.Sum(L => L.UnitPrice * L.Qty * (double)(1.0 - (L.Discount ?? 0.0) / 100.0)); - var remain = grand; - for (int i = 0; i < opt.Payments; i++) - { - var last = (i == opt.Payments - 1); - var amt = last ? remain : RoundMoney((double)rng.NextDouble() * remain * 0.7 + 1.0); - remain = Math.Max(0.0, remain - amt); - doc.Payments[i] = MakePayment(rng, amt, opt.IncludeV2Fields); - } - } - - // Attachments - if (doc.Attachments != null) - { - for (int i = 0; i < opt.Attachments; i++) - doc.Attachments[i] = MakeAttachment(rng, i, opt.AttachmentBytes); - } - - return doc; - } - - /// - /// Create a slightly modified copy of an existing document to test delta/partial updates. - /// Changes: tweak 5–10% of line items (qty/price), add or edit a payment, and bump UpdatedAt. - /// - public static BusinessDocument MakeDelta(BusinessDocument v1, int seed, double changeRatio = 0.07) - { - var rng = new Random(seed); - var v2 = DeepClone(v1); - - v2.Header.UpdatedAt = DateTime.UtcNow; - var toChange = Math.Max(1, (int)Math.Round(v2.Items.Length * changeRatio)); - - // change random lines - for (int i = 0; i < toChange; i++) - { - var idx = rng.Next(0, v2.Items.Length); - var li = v2.Items[idx]; - li.Qty = RoundQty(li.Qty + (double)(rng.NextDouble() * 2.0 - 1.0)); // ±1 - li.UnitPrice = RoundMoney(li.UnitPrice * (double)(0.95 + rng.NextDouble() * 0.1)); // ±5% - if (li.Ext == null) li.Ext = new Dictionary(); - li.Ext["lastEdit"] = VDate(DateTime.UtcNow); - li.ExtKeys = li.Ext.Keys.ToArray(); - li.ExtValues = li.Ext.Values.ToArray(); - } - - - if (v2.Payments == null || rng.Next(0, 3) == 0) - { - if (v2.Payments == null) - { - if (v2.Payments == null) v2.Payments = new Payment[1]; - v2.Payments[0] = (MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true)); - - } - else - { - v2.Payments = v2.Payments.Append((MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true))).ToArray(); - } - } - else - { - var p = v2.Payments[rng.Next(0, v2.Payments.Length)]; - p.Fee = (p.Fee ?? 0.0) + 0.25; - p.Reference = "ADJ-" + rng.Next(10000, 99999).ToString(CultureInfo.InvariantCulture); - } - - return v2; - } - - // -------------------------- Builders -------------------------- - - private static int[] RandomRiskScores(Random rng, int count) - { - - var rt = new int[count];// rng.Next(100, 1000)]; - for (var i = 0; i < rt.Length; i++) - rt[i] = (int)rng.Next(); - return rt; - } - - private static Party MakeParty(Random rng, bool includeV2, bool isSeller, bool unicode) - { - return new Party - { - Id = (ulong)rng.NextInt64(), //Guid.NewGuid().ToByteArray(), - Name = unicode ? (isSeller ? "Delta Systems — دلتا" : "Client التجربة ✅") : (isSeller ? "Delta Systems" : "Client Demo"), - TaxId = isSeller ? $"TX-{rng.Next(100000, 999999)}" : null, - Email = (isSeller ? "sales" : "contact") + "@example.com", - Phone = "+964-7" + rng.Next(100000000, 999999999).ToString(CultureInfo.InvariantCulture), - Address = new Address - { - Line1 = rng.Next(0, 2) == 0 ? "Street 14" : "Tech Park", - City = "Baghdad", - Region = "BG", - Country = "IQ", - PostalCode = rng.Next(0, 2) == 0 ? "10001" : null - }, - PreferredLanguage = includeV2 ? (rng.Next(0, 2) == 0 ? "ar-IQ" : "en-US") : null - }; - } - - private static LineItem MakeLineItem(Random rng, int lineNo, bool includeV2, int variantKvp, bool unicode) - { - var isProduct = rng.Next(0, 100) < 80; - var desc = unicode - ? (isProduct ? $"وحدة قياس — Item {lineNo} 🔧" : $"خدمة دعم — Service {lineNo} ✨") - : (isProduct ? $"Item {lineNo}" : $"Service {lineNo}"); - - var li = new LineItem - { - LineNo = lineNo, - Type = isProduct ? LineType.Product : LineType.Service, - SKU = isProduct ? ("SKU-" + rng.Next(1000, 9999)) : "", - Description = desc, - Qty = RoundQty((double)(rng.NextDouble() * 9.0 + 1.0)), // 1..10 - QtyUnit = isProduct ? "pcs" : "h", - UnitPrice = RoundMoney((double)(rng.NextDouble() * 90.0 + 10.0)), - VatRate = rng.Next(0, 100) < 80 ? 5.0 : (double?)null, - Ext = variantKvp > 0 ? new Dictionary() : null, - Discount= includeV2 && rng.Next(0, 3) == 0 ? Math.Round(rng.NextDouble() * 10.0, 2) : (double?)null - }; - - if (li.Ext != null) - { - li.ExtKeys = li.Ext.Keys.ToArray(); - li.ExtValues = li.Ext.Values.ToArray(); - } - - - for (int i = 0; i < variantKvp; i++) - { - var key = i switch { 0 => "color", 1 => "size", 2 => "batch", _ => "attr" + i }; - li.Ext!.TryAdd(key, i switch - { - 0 => VStr(rng.Next(0, 3) switch { 0 => "red", 1 => "blue", _ => "green" }), - 1 => VStr(rng.Next(0, 3) switch { 0 => "S", 1 => "M", _ => "L" }), - 2 => VGuid(Guid.NewGuid()), - _ => VInt(rng.Next(0, 1000)) - }); - } - - li.ExtValues = li.Ext.Values.ToArray(); - li.ExtKeys = li.Ext.Keys.ToArray(); - - return li; - } - - private static Payment MakePayment(Random rng, double amount, bool includeV2) - { - var p = new Payment - { - Method = (PaymentMethod)rng.Next(0, 5), - Amount = RoundMoney(amount), - Reference = "REF-" + rng.Next(100_000, 999_999), - Timestamp = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24)), - Fee = includeV2 && rng.Next(0, 2) == 0 ? RoundMoney((double)rng.NextDouble() * 2.0) : null, - //CurrencyOverride = includeV2 && rng.Next(0, 2) == 0 ? Currency.IQD : Currency.USD - }; - - p.TimestampAsLong = p.Timestamp.Ticks; - - return p; - } - - private static Attachment MakeAttachment(Random rng, int index, int bytes) - { - var arr = bytes > 0 ? new byte[bytes] : Array.Empty(); - if (arr.Length > 0) rng.NextBytes(arr); - return new Attachment - { - Name = $"att-{index + 1}.bin", - MimeType = "application/octet-stream", - Data = arr - }; - } - - private static string SampleNoteUnicode(Random rng) - => rng.Next(0, 2) == 0 - ? "ملاحظة: تم إنشاء هذا المستند لأغراض الاختبار 📦" - : "Note: synthetic benchmark document 🧪"; - - private static string SampleNoteAscii(Random rng) - => rng.Next(0, 2) == 0 ? "Note: synthetic benchmark document" : "Internal use only"; - - // -------------------------- Variant helpers -------------------------- - private static Variant VStr(string s) => new() { Tag = Variant.Kind.String, Str = s }; - private static Variant VInt(int v) => new() { Tag = Variant.Kind.Int64, I64 = v }; - private static Variant VGuid(Guid g) => new() { Tag = Variant.Kind.Guid, Guid = g.ToByteArray() }; - private static Variant VDate(DateTime d) => new() { Tag = Variant.Kind.DateTime, Dt = d, DtAsLong = d.Ticks }; - - // -------------------------- Utils -------------------------- - private static double RoundMoney(double v) => Math.Round(v, 2, MidpointRounding.AwayFromZero); - private static double RoundQty(double v) => Math.Round(v, 3, MidpointRounding.AwayFromZero); - - /// - /// Simple deep clone via manual copy to stay serializer-agnostic. - /// (Good enough for benchmarks; switch to a fast serializer if you like.) - /// - private static BusinessDocument DeepClone(BusinessDocument s) - { - var d = new BusinessDocument - { - Header = new DocumentHeader - { - DocId = s.Header.DocId, - Type = s.Header.Type, - Version = s.Header.Version, - CreatedAt = s.Header.CreatedAt, - UpdatedAt = s.Header.UpdatedAt, - //CreatedAtAsLong = s.Header.CreatedAtAsLong, - //UpdatedAtAsLong = s.Header.UpdatedAtAsLong, - Currency = s.Header.Currency, - Notes = s.Header.Notes, - Meta = s.Header.Meta?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)), - MetaKeys = s.Header.MetaKeys.ToArray(), - MetaValues = s.Header.MetaValues.ToArray(), - }, - Seller = CloneParty(s.Seller), - Buyer = CloneParty(s.Buyer), - Items = s.Items.Select(CloneLine).ToArray(), - Payments = s.Payments?.Select(ClonePayment).ToArray(), - Attachments = s.Attachments?.Select(CloneAttachment).ToArray(), - RiskScores = s.RiskScores, - //RelatedDocs_v2 = s.RelatedDocs_v2?.ToList() - }; - - - return d; - } - - private static Party CloneParty(Party p) => new() - { - Id = p.Id, - Name = p.Name, - TaxId = p.TaxId, - Email = p.Email, - Phone = p.Phone, - Address = p.Address is null ? null : new Address - { - Line1 = p.Address.Line1, - Line2 = p.Address.Line2, - City = p.Address.City, - Region = p.Address.Region, - Country = p.Address.Country, - PostalCode = p.Address.PostalCode - }, - PreferredLanguage = p.PreferredLanguage - }; - - private static LineItem CloneLine(LineItem s) => new() - { - LineNo = s.LineNo, - Type = s.Type, - SKU = s.SKU, - Description = s.Description, - Qty = s.Qty, - QtyUnit = s.QtyUnit, - UnitPrice = s.UnitPrice, - VatRate = s.VatRate, - Ext = s.Ext?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)), - Discount = s.Discount, - ExtKeys = s.Ext?.Keys.ToArray(), - ExtValues = s.Ext?.Values.ToArray(), - }; - - private static Payment ClonePayment(Payment s) => new() - { - Method = s.Method, - Amount = s.Amount, - Reference = s.Reference, - Timestamp = s.Timestamp, - TimestampAsLong = s.TimestampAsLong, - Fee = s.Fee, - //CurrencyOverride= s.CurrencyOverride - }; - - private static Attachment CloneAttachment(Attachment s) => new() - { - Name = s.Name, - MimeType = s.MimeType, - Data = s.Data.ToArray() - }; - - private static Variant CloneVariant(Variant v) => new() - { - Tag = v.Tag, - Bool = v.Bool, - I64 = v.I64, - U64 = v.U64, - F64 = v.F64, - //Dec = v.Dec, - Str = v.Str, - Bytes = v.Bytes?.ToArray(), - Dt = v.Dt, - DtAsLong = v.DtAsLong, - Guid = v.Guid - }; -} diff --git a/Tests/Serialization/GWVIE/ModelRunner.cs b/Tests/Serialization/GWVIE/ModelRunner.cs deleted file mode 100644 index 4883d66..0000000 --- a/Tests/Serialization/GWVIE/ModelRunner.cs +++ /dev/null @@ -1,484 +0,0 @@ -using Avro.Generic; -using Esiur.Resource; -using FlatSharp; -using MessagePack; -using MongoDB.Bson; -using MongoDB.Bson.Serialization; -using PeterO.Cbor; -using ProtoBuf; -using SolTechnology.Avro; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; - -#nullable enable - -namespace Esiur.Tests.Serialization; - -public interface ICodec -{ - string Name { get; } - byte[]? Serialize(BusinessDocument obj); // returns null on failure - BusinessDocument Deserialize(byte[] data); -} - -public sealed class JsonCodec : ICodec -{ - static readonly JsonSerializerOptions Opt = new() - { - WriteIndented = false, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never - }; - public string Name => "JSON"; - public byte[]? Serialize(BusinessDocument obj) - { - var data = JsonSerializer.SerializeToUtf8Bytes(obj, obj.GetType(), Opt); - return data; - } - - public BusinessDocument Deserialize(byte[] data) - { - return JsonSerializer.Deserialize(data)!; - } -} - -public sealed class EsiurCodec : ICodec -{ - public string Name => "Esiur"; - - public BusinessDocument Deserialize(byte[] data) - { - var (_, y) = Esiur.Data.Codec.ParseSync(data, 0, Warehouse.Default); - return (BusinessDocument)y!; - } - - public byte[]? Serialize(BusinessDocument obj) - { - var rt = Esiur.Data.Codec.Compose(obj, Warehouse.Default, null); - return rt; - } -} - -public sealed class MessagePackCodec : ICodec -{ - public string Name => "MessagePack"; - - public BusinessDocument Deserialize(byte[] data) - { - return MessagePackSerializer.Deserialize(data); - } - - public byte[]? Serialize(BusinessDocument obj) - { - return MessagePackSerializer.Serialize(obj.GetType(), obj); - } -} - -public sealed class ProtobufCodec : ICodec -{ - public string Name => "Protobuf"; - - public BusinessDocument Deserialize(byte[] data) - { - var dst = Serializer.Deserialize(new MemoryStream(data)); - return dst; - } - - public byte[]? Serialize(BusinessDocument obj) - { - using var ms = new MemoryStream(); - Serializer.Serialize(ms, obj); - var rt = ms.ToArray(); - - // Single correctness check (outside timing loops) - var dst = Serializer.Deserialize(new MemoryStream(rt)); - if (!obj.Equals(dst)) - throw new NotSupportedException("Protobuf roundtrip mismatch."); - - return rt; - } -} - -public sealed class FlatBuffersCodec : ICodec -{ - public string Name => "FlatBuffers"; - - public BusinessDocument Deserialize(byte[] data) - { - var m = FlatBufferSerializer.Default.Parse(data); - return m; - } - - public byte[]? Serialize(BusinessDocument obj) - { - var buffer = new byte[1_000_000]; - var count = FlatBufferSerializer.Default.Serialize(obj, buffer); - var msg = buffer.AsSpan(0, count).ToArray(); - - // Single correctness check (outside timing loops) - var m = FlatBufferSerializer.Default.Parse(msg); - if (!m!.Equals(obj)) - throw new Exception("FlatBuffers roundtrip mismatch."); - - return msg; - } -} - -public sealed class CborCodec : ICodec -{ - public string Name => "CBOR"; - - public BusinessDocument Deserialize(byte[] data) - { - return CBORObject.DecodeObjectFromBytes(data)!; - } - - public byte[]? Serialize(BusinessDocument obj) - { - var c = CBORObject.FromObject(obj); - return c.EncodeToBytes(); - } -} - -public sealed class BsonCodec : ICodec -{ - private static bool _init; - private static void EnsureMaps() - { - if (_init) return; - _init = true; - // Register class maps if needed; defaults usually work for POCOs. - } - - public string Name => "BSON"; - public byte[]? Serialize(BusinessDocument obj) - { - try - { - EnsureMaps(); - using var ms = new MemoryStream(); - using var writer = new MongoDB.Bson.IO.BsonBinaryWriter(ms); - var context = MongoDB.Bson.Serialization.BsonSerializationContext.CreateRoot(writer); - var args = new MongoDB.Bson.Serialization.BsonSerializationArgs(obj.GetType(), true, false); - var serializer = BsonSerializer.LookupSerializer(obj.GetType()); - serializer.Serialize(context, args, obj); - return ms.ToArray(); - } - catch { return null; } - } - - public BusinessDocument Deserialize(byte[] data) - { - using var ms = new MemoryStream(data); - using var reader = new MongoDB.Bson.IO.BsonBinaryReader(ms); - var args = new MongoDB.Bson.Serialization.BsonSerializationArgs(typeof(BusinessDocument), true, false); - var context = MongoDB.Bson.Serialization.BsonDeserializationContext.CreateRoot(reader); - var serializer = BsonSerializer.LookupSerializer(typeof(BusinessDocument)); - return (BusinessDocument)serializer.Deserialize(context)!; - } -} - -public sealed class AvroCodec : ICodec -{ - public string Name => "Avro"; - - public BusinessDocument Deserialize(byte[] data) - { - return AvroConvert.Deserialize(data); - } - - public byte[]? Serialize(BusinessDocument obj) - { - return AvroConvert.Serialize(obj); - } -} - -// ------------------------- Stat helpers ------------------------- - -public static class Stats -{ - public static double Mean(IReadOnlyList xs) => - xs.Count == 0 ? double.NaN : xs.Average(); - - public static double Median(IReadOnlyList xs) - { - if (xs.Count == 0) return double.NaN; - var arr = xs.OrderBy(v => v).ToArray(); - int n = arr.Length; - return n % 2 == 1 ? arr[n / 2] : (arr[n / 2 - 1] + arr[n / 2]) / 2.0; - } - - public static string ClassifyVsJson(double ratio) - { - if (double.IsNaN(ratio)) return "N/A"; - if (ratio <= 0.75) return "Smaller (≤0.75× JSON)"; - if (ratio <= 1.25) return "Similar (~0.75–1.25×)"; - return "Larger (≥1.25× JSON)"; - } -} - -// ------------------------- Workload config ------------------------- - -public enum Workload -{ - Small, // ~5 lines, no attachments - Medium, // ~20 lines, 1 small attachment - Large, // ~100 lines, 3 x 64KB attachments -} - -public sealed class WorkItem -{ - public required string Name { get; init; } - public required BusinessDocument Payload { get; init; } -} - -// ------------------------- CPU timing helpers ------------------------- - -public static class CpuTimer -{ - // small warm-up to reduce JIT/first-use bias - public static void WarmUp(Action action, int rounds = 5) - { - for (int i = 0; i < rounds; i++) action(); - } - - // Measures process CPU time consumed by running `action` N times. - // Returns average microseconds per operation. - public static double MeasureAverageMicros(Action action, int rounds) - { - var proc = Process.GetCurrentProcess(); - - // GC before timing to reduce random interference (still CPU, but more stable) - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - - var start = proc.TotalProcessorTime; - for (int i = 0; i < rounds; i++) action(); - var end = proc.TotalProcessorTime; - - var delta = end - start; - var micros = delta.TotalMilliseconds * 1000.0; - return micros / rounds; - } -} - -// ------------------------- Runner ------------------------- - -public sealed class ModelRunner -{ - private readonly ICodec[] _codecs; - - public ModelRunner() - { - _codecs = new ICodec[] - { - new JsonCodec(), - new EsiurCodec(), - new MessagePackCodec(), - new ProtobufCodec(), - new FlatBuffersCodec(), - new CborCodec(), - new BsonCodec(), - new AvroCodec() - }; - } - - record WorkloadResult - { - public string Name { get; init; } = ""; - public List Sizes { get; init; } = new(); - public double EncodeCpuUsSum { get; set; } // sum of per-item avg CPU us/op - public double DecodeCpuUsSum { get; set; } - public int Samples { get; set; } // count of successful samples - } - - // volatile sink to avoid aggressive JIT elimination in tight loops - private static volatile byte[]? _sinkBytes; - private static volatile BusinessDocument? _sinkDoc; - - public void Run() - { - const int Rounds = 100; - - var workloads = BuildWorkloads(); - - Console.WriteLine("=== Serialization Size & CPU (process) Benchmark ==="); - Console.WriteLine($"Date (UTC): {DateTime.UtcNow:O}"); - Console.WriteLine($"Rounds per op (CPU): {Rounds}"); - Console.WriteLine(); - - foreach (var (wName, items) in workloads) - { - Console.WriteLine($"--- Workload: {wName} ---"); - - // Collect results: Codec -> result container - var results = new Dictionary(); - foreach (var c in _codecs) results[c.Name] = new WorkloadResult { Name = c.Name }; - - foreach (var item in items) - { - foreach (var c in _codecs) - { - try - { - // Single functional serialize to get bytes & verify equality ONCE (not in timed loop) - var bytes = c.Serialize(item.Payload); - if (bytes == null) - { - results[c.Name].Sizes.Add(long.MinValue); - continue; - } - - var back = c.Deserialize(bytes); - if (!item.Payload.Equals(back)) - throw new InvalidOperationException($"{c.Name} roundtrip inequality."); - - results[c.Name].Sizes.Add(bytes.LongLength); - - // ---- CPU timing ---- - - // Warm-up (tiny) - CpuTimer.WarmUp(() => { _sinkBytes = c.Serialize(item.Payload); }, 3); - CpuTimer.WarmUp(() => { _sinkDoc = c.Deserialize(bytes); }, 3); - - // Measure serialize CPU (average µs/op over Rounds) - var encUs = CpuTimer.MeasureAverageMicros(() => - { - _sinkBytes = c.Serialize(item.Payload); - }, Rounds); - - // Measure deserialize CPU - var decUs = CpuTimer.MeasureAverageMicros(() => - { - _sinkDoc = c.Deserialize(bytes); - }, Rounds); - - results[c.Name].EncodeCpuUsSum += encUs; - results[c.Name].DecodeCpuUsSum += decUs; - results[c.Name].Samples += 1; - } - catch - { - // mark size failure for this sample if not already added - results[c.Name].Sizes.Add(long.MinValue); - } - } - } - - // Compute stats, using only successful size samples - var jsonSizes = results["JSON"].Sizes.Where(v => v != long.MinValue).ToList(); - var jsonMean = Stats.Mean(jsonSizes); - var jsonMed = Stats.Median(jsonSizes); - - Console.WriteLine($"JSON mean: {jsonMean:F1} B, median: {jsonMed:F1} B"); - Console.WriteLine(); - - Console.WriteLine("{0,-14} {1,12} {2,12} {3,10} {4,26} {5,18} {6,18}", - "Codec", "Mean(B)", "Median(B)", "Ratio", "Class vs JSON", "Enc CPU (µs)", "Dec CPU (µs)"); - Console.WriteLine(new string('-', 118)); - - foreach (var c in _codecs) - { - var r = results[c.Name]; - var okSizes = r.Sizes.Where(v => v != long.MinValue).ToList(); - var mean = Stats.Mean(okSizes); - var med = Stats.Median(okSizes); - - double ratio = double.NaN; - if (!double.IsNaN(mean) && !double.IsNaN(jsonMean) && jsonMean > 0) ratio = mean / jsonMean; - - string cls = Stats.ClassifyVsJson(ratio); - string meanS = double.IsNaN(mean) ? "N/A" : mean.ToString("F1"); - string medS = double.IsNaN(med) ? "N/A" : med.ToString("F1"); - string ratioS = double.IsNaN(ratio) ? "N/A" : ratio.ToString("F3"); - - // average CPU µs/op across samples where serialization succeeded - string encCpuS = (r.Samples == 0) ? "N/A" : (r.EncodeCpuUsSum / r.Samples).ToString("F1"); - string decCpuS = (r.Samples == 0) ? "N/A" : (r.DecodeCpuUsSum / r.Samples).ToString("F1"); - - Console.WriteLine("{0,-14} {1,12} {2,12} {3,10} {4,26} {5,18} {6,18}", - c.Name, meanS, medS, ratioS, cls, encCpuS, decCpuS); - } - - Console.WriteLine(); - - Console.ReadLine(); - } - } - - private static List<(string, List)> BuildWorkloads() - { - var result = new List<(string, List)>(); - - // Small - { - var items = new List(); - for (int i = 0; i < 16; i++) - { - var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions - { - Lines = 5, - Payments = 3, - Attachments = 0, - IncludeV2Fields = (i % 2 == 0), - IncludeUnicode = true, - RiskScores = 500, - Seed = 1000 + i - }); - - items.Add(new WorkItem { Name = $"S-{i}", Payload = doc }); - } - result.Add(("Small", items)); - } - - // Medium - { - var items = new List(); - for (int i = 0; i < 16; i++) - { - var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions - { - Lines = 20, - Payments = 5, - Attachments = 1, - AttachmentBytes = 8 * 1024, - IncludeV2Fields = (i % 3 == 0), - IncludeUnicode = true, - RiskScores = 1000, - Seed = 2000 + i - }); - items.Add(new WorkItem { Name = $"M-{i}", Payload = doc }); - } - result.Add(("Medium", items)); - } - - // Large - { - var items = new List(); - for (int i = 0; i < 12; i++) - { - var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions - { - Lines = 100, - Payments = 20, - Attachments = 3, - AttachmentBytes = 64 * 1024, - IncludeV2Fields = (i % 2 == 1), - IncludeUnicode = true, - RiskScores = 3000, - Seed = 3000 + i - }); - items.Add(new WorkItem { Name = $"L-{i}", Payload = doc }); - } - result.Add(("Large", items)); - } - - return result; - } -}