using Esiur.Data.GVWIE; using FlatSharp; using FlatSharp.Attributes; using MessagePack; using MongoDB.Bson; using PeterO.Cbor; using ProtoBuf; using SolTechnology.Avro; using System; using System.Buffers; using System.Collections.Generic; using System.Text; namespace Esiur.Tests.Serialization { [FlatBufferTable] public class ArrayRoot { // Field index must be stable; start at 0 [FlatBufferItem(0)] public virtual IList? Values { get; set; } } internal class IntArrayRunner { public void Run() { Console.WriteLine(";Esiur;FlatBuffer;ProtoBuffer;MessagePack;BSON;CBOR;Avro,Optimal"); //Console.Write("Cluster (Int32);"); ////CompareInt(int32cluster); //Average(() => CompareInt(IntArrayGenerator.GenerateInt32Run(1000)), 1000); Console.Write("Positive (Int32);"); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "positive")), 1000); Console.Write("Negative (Int32);"); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "negative")), 1000); Console.Write("Small (Int32);"); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "small")), 1000); // CompareInt(int32small); Console.Write("Alternating (Int32);"); //CompareInt(int32alter); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "alternating")), 1000); Console.Write("Ascending (Int32);"); //CompareInt(int32asc); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "ascending")), 1000); Console.Write("Int64;"); Average(() => CompareInt(IntArrayGenerator.GenerateInt64(1000, "uniform")), 1000); //CompareInt(int64Uni); Console.Write("Int32;"); //CompareInt(int32Uni); Average(() => CompareInt(IntArrayGenerator.GenerateInt32(1000, "uniform")), 1000); Console.Write("Int16;"); //CompareInt(int16Uni); Average(() => CompareInt(IntArrayGenerator.GenerateInt16(1000, "uniform")), 1000); Console.Write("UInt64;"); //CompareInt(uint64Uni); Average(() => CompareInt(IntArrayGenerator.GenerateUInt64(1000, "uniform")), 1000); Console.Write("UInt32;"); //CompareInt(uint32Uni); Average(() => CompareInt(IntArrayGenerator.GenerateUInt32(1000, "uniform")), 1000); Console.Write("UInt16;"); //CompareInt(uint16Uni); Average(() => CompareInt(IntArrayGenerator.GenerateUInt16(1000, "uniform")), 1000); } public static (int, int, int, int, int, int, int, int) CompareInt(long[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupInt64Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var bson = intRoot.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); //var ans1 = seq.GetDerEncoded(); var avro = AvroConvert.Serialize(sample); var optimal = OptimalSignedEnocding(sample); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static (int, int, int, int, int, int, int, int) CompareInt(int[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupInt32Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var bson = intRoot.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); //var ans1 = seq.GetDerEncoded(); var avro = AvroConvert.Serialize(sample); var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray()); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static (int, int, int, int, int, int, int, int) CompareInt(short[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupInt16Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var bson = intRoot.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); //var ans1 = seq.GetDerEncoded(); var avro = AvroConvert.Serialize(sample); var optimal = OptimalSignedEnocding(sample.Select(x => (long)x).ToArray()); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static (int, int, int, int, int, int, int, int) CompareInt(uint[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupUInt32Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var intRoot2 = new ArrayRoot() { Values = sample.Select(x => (int)x).ToArray() }; var bson = intRoot2.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); var avro = AvroConvert.Serialize(sample.Select(x => (int)x).ToArray()); //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); //var avro = seq.GetDerEncoded(); var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray()); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static (int, int, int, int, int, int, int, int) CompareInt(ulong[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupUInt64Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var bson = intRoot.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); //var seq = new DerSequence(sample.Select(v => new DerInteger((long)v)).ToArray()); //var ans1 = seq.GetDerEncoded(); var avro = AvroConvert.Serialize(sample); var optimal = OptimalUnsignedEnocding(sample); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static (int, int, int, int, int, int, int, int) CompareInt(ushort[] sample) { var intRoot = new ArrayRoot() { Values = sample }; var esiur = GroupUInt16Codec.Encode(sample); var messagePack = MessagePackSerializer.Serialize(sample); var flatBuffer = SerializeFlatBuffers(intRoot); using var ms = new MemoryStream(); Serializer.Serialize(ms, sample); var protoBuffer = ms.ToArray(); var bson = intRoot.ToBson(); var cbor = CBORObject.FromObject(intRoot).EncodeToBytes(); //var seq = new DerSequence(sample.Select(v => new DerInteger(v)).ToArray()); //var ans1 = seq.GetDerEncoded(); var avro = AvroConvert.Serialize(sample); var optimal = OptimalUnsignedEnocding(sample.Select(x => (ulong)x).ToArray()); //Console.WriteLine($"{esiur.Length};{flatBuffer.Length};{protoBuffer.Length};{messagePack.Length};{bson.Length};{cbor.Length};{avro.Length};{optimal}"); return (esiur.Length, flatBuffer.Length, protoBuffer.Length, messagePack.Length, bson.Length, cbor.Length, avro.Length, optimal); } public static int OptimalSignedEnocding(long[] data) { var sum = 0; foreach (var i in data) if (i >= sbyte.MinValue && i <= sbyte.MaxValue) sum += 1; else if (i >= short.MinValue && i <= short.MaxValue) sum += 2; else if (i >= -8_388_608 && i <= 8_388_607) sum += 3; else if (i >= int.MinValue && i <= int.MaxValue) sum += 4; else if (i >= -549_755_813_888 && i <= 549_755_813_887) sum += 5; else if (i >= -140_737_488_355_328 && i <= 140_737_488_355_327) sum += 6; else if (i >= -36_028_797_018_963_968 && i <= 36_028_797_018_963_967) sum += 7; else if (i >= long.MinValue && i <= long.MaxValue) sum += 8; return sum; } public static int OptimalUnsignedEnocding(ulong[] data) { var sum = 0; foreach (var i in data) if (i <= byte.MaxValue) sum += 1; else if (i <= ushort.MaxValue) sum += 2; else if (i <= uint.MaxValue) sum += 4; else if (i <= 0xFF_FF_FF_FF_FF) sum += 5; else if (i <= 0xFF_FF_FF_FF_FF_FF) sum += 6; else if (i <= 0xFF_FF_FF_FF_FF_FF_FF) sum += 7; else if (i <= ulong.MaxValue) sum += 8; return sum; } static (double, double, double, double, double, double, double, double) Average(Func<(int, int, int, int, int, int, int, int)> call, int count) { var sum = new List<(int, int, int, int, int, int, int, int)>(); for (var i = 0; i < count; i++) sum.Add(call()); var rt = (sum.Average(x => x.Item1), sum.Average(x => x.Item2), sum.Average(x => x.Item3), sum.Average(x => x.Item4), sum.Average(x => x.Item5), sum.Average(x => x.Item6), sum.Average(x => x.Item7), sum.Average(x => x.Item8) ); Console.WriteLine($"{rt.Item1};{rt.Item2};{rt.Item3};{rt.Item4};{rt.Item5};{rt.Item6};{rt.Item7};{rt.Item8}"); return rt; } public static byte[] SerializeFlatBuffers(ArrayRoot array) { var buffer = new byte[1000000000]; var len = FlatBufferSerializer.Default.Serialize(array, buffer); return buffer.Take(len).ToArray(); } } }