From 2f39aabf7e563083a08a72cf2cc9a02a087243c4 Mon Sep 17 00:00:00 2001 From: ahmed Date: Tue, 9 Jun 2026 20:56:01 +0300 Subject: [PATCH] ProxyTypes --- Libraries/Esiur/Protocol/EpConnection.cs | 60 ++- .../ResourceCount/Client/Program.cs | 4 +- .../ResourceCount/Server/Program.cs | 3 +- Tests/RPC/Client/EsiurTest.cs | 54 +-- Tests/RPC/Client/ExperimentResultWriter.cs | 25 +- Tests/RPC/Client/GrpcTest.cs | 58 +-- Tests/RPC/Client/JsonTest.cs | 52 +- Tests/RPC/Client/ModelGenerator.cs | 446 ++++++++++++++++++ Tests/RPC/Client/Program.cs | 48 +- Tests/RPC/Client/SignalRTest.cs | 52 +- Tests/RPC/Client/ThriftTest.cs | 52 +- 11 files changed, 652 insertions(+), 202 deletions(-) create mode 100644 Tests/RPC/Client/ModelGenerator.cs diff --git a/Libraries/Esiur/Protocol/EpConnection.cs b/Libraries/Esiur/Protocol/EpConnection.cs index 5168e55..1defd23 100644 --- a/Libraries/Esiur/Protocol/EpConnection.cs +++ b/Libraries/Esiur/Protocol/EpConnection.cs @@ -1031,7 +1031,7 @@ public partial class EpConnection : NetworkConnection, IStore - async Task AuthenticatonCompleted() + void AuthenticatonCompleted() { if (this.Instance == null) @@ -1070,35 +1070,45 @@ public partial class EpConnection : NetworkConnection, IStore var proxyTypes = Instance.Warehouse.GetProxyTypesByDomain(_remoteDomain); - var typeDefNames = new List(); - - foreach (var kk in proxyTypes) + if (proxyTypes != null) { - foreach (var kv in kk.Value) - { - typeDefNames.Add(kv.Key); - } - } - if (typeDefNames.Count > 0) - { - GetTypeDefIds(typeDefNames.ToArray()).Then(ids => - { - var bag = new AsyncBag(); - foreach (var id in ids) - bag.Add(FetchTypeDef(id, null)); + var typeDefNames = new List(); - bag.Seal(); - bag.Then((o) => + foreach (var kk in proxyTypes) + { + foreach (var kv in kk.Value) { - _openReply?.Trigger(true); - _openReply = null; - }); - }).Error(ex => + typeDefNames.Add(kv.Key); + } + } + + if (typeDefNames.Count > 0) { - _openReply.TriggerError(ex); - // do nothing, proxies won't work but connection is established - }); + GetTypeDefIds(typeDefNames.ToArray()).Then(ids => + { + var bag = new AsyncBag(); + foreach (var id in ids) + bag.Add(FetchTypeDef(id, null)); + + bag.Seal(); + + bag.Then((o) => + { + _openReply?.Trigger(true); + _openReply = null; + }); + }).Error(ex => + { + _openReply.TriggerError(ex); + // do nothing, proxies won't work but connection is established + }); + } + else + { + _openReply?.Trigger(true); + _openReply = null; + } } else { diff --git a/Tests/Distribution/ResourceCount/Client/Program.cs b/Tests/Distribution/ResourceCount/Client/Program.cs index 4d92958..299a49b 100644 --- a/Tests/Distribution/ResourceCount/Client/Program.cs +++ b/Tests/Distribution/ResourceCount/Client/Program.cs @@ -44,10 +44,10 @@ for (int batch = 0; batch < resourceCount; batch += batchSize) { var sw = Stopwatch.StartNew(); - Console.WriteLine(capturedI); + //Console.WriteLine(capturedI); proxies[capturedI] = await connnection.Get($"sys/sensor_{capturedI}"); - Console.WriteLine(proxies[capturedI].Instance.Link); + //Console.WriteLine(proxies[capturedI].Instance.Link); sw.Stop(); diff --git a/Tests/Distribution/ResourceCount/Server/Program.cs b/Tests/Distribution/ResourceCount/Server/Program.cs index 63071ac..624d831 100644 --- a/Tests/Distribution/ResourceCount/Server/Program.cs +++ b/Tests/Distribution/ResourceCount/Server/Program.cs @@ -18,7 +18,8 @@ Console.WriteLine($"[Server-T2] Creating {resourceCount} resources on port {port var wh = new Warehouse(); await wh.Put("sys", new MemoryStore()); -var server = await wh.Put("sys/server", new EpServer() { Port = (ushort)port }); +var server = await wh.Put("sys/server", new EpServer() { Port = (ushort)port, + AllowUnauthorizedAccess = true}); long memBefore = GC.GetTotalMemory(forceFullCollection: true); diff --git a/Tests/RPC/Client/EsiurTest.cs b/Tests/RPC/Client/EsiurTest.cs index e2240a7..c16d112 100644 --- a/Tests/RPC/Client/EsiurTest.cs +++ b/Tests/RPC/Client/EsiurTest.cs @@ -9,8 +9,8 @@ namespace Esiur.Tests.RPC.Client { public static async Task DoTest(string address, Dictionary docsWorkloads, - Dictionary dataWorkloads, - Dictionary intWorkloads, + //Dictionary dataWorkloads, + //Dictionary intWorkloads, int warmupDelayMs = 3000, int postHandshakeDelayMs = 2000, int sampleDelayMs = 3000) @@ -19,7 +19,7 @@ namespace Esiur.Tests.RPC.Client var rt = new TestResults(); using var mon = new PerProcessNetMonitor(Process.GetCurrentProcess().Id); - //mon.Start(); + mon.Start(); Console.WriteLine($"\n== Esiur @ {address} =="); @@ -60,45 +60,45 @@ namespace Esiur.Tests.RPC.Client - foreach (var w in dataWorkloads) - { - Console.Write("Bytes Workload: " + w.Key); - var res = await service.EchoBytes(w.Value); + //foreach (var w in dataWorkloads) + //{ + // Console.Write("Bytes Workload: " + w.Key); + // var res = await service.EchoBytes(w.Value); - if (!w.Value.SequenceEqual(res)) - throw new Exception("No match"); + // if (!w.Value.SequenceEqual(res)) + // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Bytes.Add(w.Key, (ctx, crx)); + // rt.Bytes.Add(w.Key, (ctx, crx)); - } + //} - foreach (var w in intWorkloads) - { - Console.Write("Ints Workload: " + w.Key); - var res = await service.EchoIntArray(w.Value); + //foreach (var w in intWorkloads) + //{ + // Console.Write("Ints Workload: " + w.Key); + // var res = await service.EchoIntArray(w.Value); - if (!w.Value.SequenceEqual(res)) - throw new Exception("No match"); + // if (!w.Value.SequenceEqual(res)) + // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Ints.Add(w.Key, (ctx, crx)); + // rt.Ints.Add(w.Key, (ctx, crx)); - } + //} await Task.Delay(sampleDelayMs); diff --git a/Tests/RPC/Client/ExperimentResultWriter.cs b/Tests/RPC/Client/ExperimentResultWriter.cs index a837043..245c8b6 100644 --- a/Tests/RPC/Client/ExperimentResultWriter.cs +++ b/Tests/RPC/Client/ExperimentResultWriter.cs @@ -122,13 +122,13 @@ public static class ExperimentResultWriter IReadOnlyList<(string Protocol, string Category, string Workload, int Count, NumberStats Tx, NumberStats Rx)> rows) { var csv = new StringBuilder(); - csv.AppendLine("protocol,category,workload,samples,tx_avg_bytes,tx_min_bytes,tx_max_bytes,tx_median_bytes,rx_avg_bytes,rx_min_bytes,rx_max_bytes,rx_median_bytes"); + csv.AppendLine("protocol,category,workload,samples,tx_avg_bytes,tx_stddev_bytes,tx_min_bytes,tx_max_bytes,tx_median_bytes,rx_avg_bytes,rx_stddev_bytes,rx_min_bytes,rx_max_bytes,rx_median_bytes"); foreach (var x in rows) { csv.AppendLine(string.Join(",", Csv(x.Protocol), Csv(x.Category), Csv(x.Workload), x.Count, - D(x.Tx.Average), D(x.Tx.Minimum), D(x.Tx.Maximum), D(x.Tx.Median), - D(x.Rx.Average), D(x.Rx.Minimum), D(x.Rx.Maximum), D(x.Rx.Median))); + D(x.Tx.Average), D(x.Tx.StandardDeviation), D(x.Tx.Minimum), D(x.Tx.Maximum), D(x.Tx.Median), + D(x.Rx.Average), D(x.Rx.StandardDeviation), D(x.Rx.Minimum), D(x.Rx.Maximum), D(x.Rx.Median))); } File.WriteAllText(path, csv.ToString()); } @@ -151,14 +151,14 @@ public static class ExperimentResultWriter IReadOnlyList<(string Protocol, string Category, string Workload, int Count, NumberStats Payload, NumberStats Serialize, NumberStats Deserialize)> rows) { var csv = new StringBuilder(); - csv.AppendLine("protocol,category,workload,samples,payload_avg_bytes,payload_min_bytes,payload_max_bytes,payload_median_bytes,serialize_avg_ms,serialize_min_ms,serialize_max_ms,serialize_median_ms,deserialize_avg_ms,deserialize_min_ms,deserialize_max_ms,deserialize_median_ms"); + csv.AppendLine("protocol,category,workload,samples,payload_avg_bytes,payload_stddev_bytes,payload_min_bytes,payload_max_bytes,payload_median_bytes,serialize_avg_ms,serialize_stddev_ms,serialize_min_ms,serialize_max_ms,serialize_median_ms,deserialize_avg_ms,deserialize_stddev_ms,deserialize_min_ms,deserialize_max_ms,deserialize_median_ms"); foreach (var x in rows) { csv.AppendLine(string.Join(",", Csv(x.Protocol), Csv(x.Category), Csv(x.Workload), x.Count, - D(x.Payload.Average), D(x.Payload.Minimum), D(x.Payload.Maximum), D(x.Payload.Median), - D(x.Serialize.Average), D(x.Serialize.Minimum), D(x.Serialize.Maximum), D(x.Serialize.Median), - D(x.Deserialize.Average), D(x.Deserialize.Minimum), D(x.Deserialize.Maximum), D(x.Deserialize.Median))); + D(x.Payload.Average), D(x.Payload.StandardDeviation), D(x.Payload.Minimum), D(x.Payload.Maximum), D(x.Payload.Median), + D(x.Serialize.Average), D(x.Serialize.StandardDeviation), D(x.Serialize.Minimum), D(x.Serialize.Maximum), D(x.Serialize.Median), + D(x.Deserialize.Average), D(x.Deserialize.StandardDeviation), D(x.Deserialize.Minimum), D(x.Deserialize.Maximum), D(x.Deserialize.Median))); } File.WriteAllText(path, csv.ToString()); } @@ -232,19 +232,24 @@ public static class ExperimentResultWriter private static string D(double value) => value.ToString("0.###", CultureInfo.InvariantCulture); - public readonly record struct NumberStats(double Average, double Minimum, double Maximum, double Median) + public readonly record struct NumberStats(double Average, double Minimum, double Maximum, double Median, double StandardDeviation) { public static NumberStats From(IEnumerable values) { var sorted = values.OrderBy(x => x).ToArray(); if (sorted.Length == 0) - return new NumberStats(double.NaN, double.NaN, double.NaN, double.NaN); + return new NumberStats(double.NaN, double.NaN, double.NaN, double.NaN, double.NaN); + var avg = sorted.Average(); var median = sorted.Length % 2 == 1 ? sorted[sorted.Length / 2] : (sorted[sorted.Length / 2 - 1] + sorted[sorted.Length / 2]) / 2.0; - return new NumberStats(sorted.Average(), sorted[0], sorted[^1], median); + // Calculate standard deviation + var variance = sorted.Aggregate(0.0, (sum, x) => sum + Math.Pow(x - avg, 2)) / sorted.Length; + var stdDev = Math.Sqrt(variance); + + return new NumberStats(avg, sorted[0], sorted[^1], median, stdDev); } } } diff --git a/Tests/RPC/Client/GrpcTest.cs b/Tests/RPC/Client/GrpcTest.cs index eab4a90..55e4f82 100644 --- a/Tests/RPC/Client/GrpcTest.cs +++ b/Tests/RPC/Client/GrpcTest.cs @@ -15,8 +15,8 @@ public class GrpcTest public static async Task DoTest(string address, Dictionary docsWorkloads, - Dictionary dataWorkloads, - Dictionary intWorkloads, + //Dictionary dataWorkloads, + //Dictionary intWorkloads, int warmupDelayMs = 3000, int postHandshakeDelayMs = 2000, int sampleDelayMs = 3000) @@ -63,50 +63,50 @@ public class GrpcTest - foreach (var w in dataWorkloads) - { - Console.Write("Bytes Workload: " + w.Key); + //foreach (var w in dataWorkloads) + //{ + // Console.Write("Bytes Workload: " + w.Key); - var br = new BytesRequest() { Data = ByteString.CopyFrom(w.Value) }; - var res = await service.EchoBytesAsync(br); + // var br = new BytesRequest() { Data = ByteString.CopyFrom(w.Value) }; + // var res = await service.EchoBytesAsync(br); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Bytes.Add(w.Key, (ctx, crx)); + // rt.Bytes.Add(w.Key, (ctx, crx)); - } + //} - foreach (var w in intWorkloads) - { - Console.Write("Ints Workload: " + w.Key); + //foreach (var w in intWorkloads) + //{ + // Console.Write("Ints Workload: " + w.Key); - var ir = new IntArrayRequest(); - ir.Array.AddRange(w.Value); + // var ir = new IntArrayRequest(); + // ir.Array.AddRange(w.Value); - var res = await service.EchoIntArrayAsync(ir); + // var res = await service.EchoIntArrayAsync(ir); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Ints.Add(w.Key, (ctx, crx)); + // rt.Ints.Add(w.Key, (ctx, crx)); - } + //} await Task.Delay(sampleDelayMs); diff --git a/Tests/RPC/Client/JsonTest.cs b/Tests/RPC/Client/JsonTest.cs index 411faf9..fe15f14 100644 --- a/Tests/RPC/Client/JsonTest.cs +++ b/Tests/RPC/Client/JsonTest.cs @@ -14,8 +14,8 @@ public class JsonTest public static async Task DoTest(string address, Dictionary docsWorkloads, - Dictionary dataWorkloads, - Dictionary intWorkloads, + //Dictionary dataWorkloads, + //Dictionary intWorkloads, int warmupDelayMs = 3000, int postHandshakeDelayMs = 2000, int sampleDelayMs = 3000) @@ -54,45 +54,45 @@ public class JsonTest - foreach (var w in dataWorkloads) - { - Console.Write("Bytes Workload: " + w.Key); + //foreach (var w in dataWorkloads) + //{ + // Console.Write("Bytes Workload: " + w.Key); - var res = await JsonRpcCallAsync(http, "EchoBytes", w.Value); + // var res = await JsonRpcCallAsync(http, "EchoBytes", w.Value); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Bytes.Add(w.Key, (ctx, crx)); + // rt.Bytes.Add(w.Key, (ctx, crx)); - } + //} - foreach (var w in intWorkloads) - { - Console.Write("Ints Workload: " + w.Key); + //foreach (var w in intWorkloads) + //{ + // Console.Write("Ints Workload: " + w.Key); - var res = await JsonRpcCallAsync(http, "EchoIntArray", w.Value); + // var res = await JsonRpcCallAsync(http, "EchoIntArray", w.Value); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Ints.Add(w.Key, (ctx, crx)); + // rt.Ints.Add(w.Key, (ctx, crx)); - } + //} await Task.Delay(sampleDelayMs); diff --git a/Tests/RPC/Client/ModelGenerator.cs b/Tests/RPC/Client/ModelGenerator.cs new file mode 100644 index 0000000..f51d9b7 --- /dev/null +++ b/Tests/RPC/Client/ModelGenerator.cs @@ -0,0 +1,446 @@ +using Esiur.Data; +using Esiur.Tests.RPC.EsiurServer; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace Esiur.Tests.RPC.Client; + + +public static class ModelGenerator +{ + public sealed class GenOptions + { + public int Lines { get; init; } = 20; // items count + public int Attachments { get; init; } = 0; // 0..N + public int AttachmentBytes { get; init; } = 0;// per attachment + public int Payments { get; init; } = 1; // 0..N + public bool IncludeV2Fields { get; init; } = false; + public bool IncludeUnicode { get; init; } = true; // Arabic/emoji sprinkled in + public int VariantPerLine { get; init; } = 1; // ad-hoc KV per line + public Currency Currency { get; init; } = Currency.USD; + + public int RiskScores { get; set; } = 20; + public int Seed { get; init; } = 12345; + } + + public sealed class WorkItem + { + public required string Name { get; init; } + public required BusinessDocument Payload { get; init; } + } + + public static List<(string, List)> BuildWorkloads() + { + var result = new List<(string, List)>(); + + // Small + { + var items = new List(); + for (int i = 0; i < 10; i++) + { + var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions + { + Lines = 5, + Payments = 3, + Attachments = 0, + IncludeV2Fields = (i % 2 == 0), + IncludeUnicode = true, + RiskScores = 500, + Seed = 1000 + i + }); + + items.Add(new WorkItem { Name = $"S-{i}", Payload = doc }); + } + result.Add(("Small", items)); + } + + // Medium + { + var items = new List(); + for (int i = 0; i < 10; i++) + { + var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions + { + Lines = 20, + Payments = 5, + Attachments = 1, + AttachmentBytes = 8 * 1024, + IncludeV2Fields = (i % 3 == 0), + IncludeUnicode = true, + RiskScores = 1000, + Seed = 2000 + i + }); + items.Add(new WorkItem { Name = $"M-{i}", Payload = doc }); + } + result.Add(("Medium", items)); + } + + // Large + { + var items = new List(); + for (int i = 0; i < 10; i++) + { + var doc = ModelGenerator.MakeBusinessDocument(new ModelGenerator.GenOptions + { + Lines = 100, + Payments = 20, + Attachments = 3, + AttachmentBytes = 64 * 1024, + IncludeV2Fields = (i % 2 == 1), + IncludeUnicode = true, + RiskScores = 3000, + Seed = 3000 + i + }); + items.Add(new WorkItem { Name = $"L-{i}", Payload = doc }); + } + result.Add(("Large", items)); + } + + return result; + } + public static BusinessDocument MakeBusinessDocument(GenOptions? options = null) + { + var opt = options ?? new GenOptions(); + var rng = new Random(opt.Seed); + + var seller = MakeParty(rng, opt.IncludeV2Fields, isSeller: true, opt.IncludeUnicode); + var buyer = MakeParty(rng, opt.IncludeV2Fields, isSeller: false, opt.IncludeUnicode); + + var createdAt = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24)); + var doc = new BusinessDocument + { + Header = new DocumentHeader + { + DocId = Guid.NewGuid().ToByteArray(), + Type = (DocType)rng.Next(0, 4), + Version = 1, + CreatedAt = createdAt, + UpdatedAt = null, + Currency = opt.Currency, + Notes = opt.IncludeUnicode ? SampleNoteUnicode(rng) : SampleNoteAscii(rng), + Meta = new Map + { + ["source"] = VStr("benchmark"), + ["region"] = VStr("ME"), + ["channel"] = VStr(rng.Next(0, 2) == 0 ? "online" : "pos"), + } + }, + Seller = seller, + Buyer = buyer, + Items = new LineItem[opt.Lines], + Payments = opt.Payments > 0 ? new Payment[opt.Payments] : null, + Attachments = opt.Attachments > 0 ? new Attachment[opt.Attachments] : null, + + RiskScores = RandomRiskScores(rng, opt.RiskScores), + //RelatedDocs_v2 = opt.IncludeV2Fields ? new List { Guid.NewGuid(), Guid.NewGuid() } : null + }; + + + // Items + for (int i = 0; i < opt.Lines; i++) + doc.Items[i] = MakeLineItem(rng, i + 1, opt.IncludeV2Fields, opt.VariantPerLine, opt.IncludeUnicode); + + // Payments + if (doc.Payments != null) + { + var grand = doc.Items.Sum(L => L.UnitPrice * L.Qty * (double)(1.0 - (L.Discount ?? 0.0) / 100.0)); + var remain = grand; + for (int i = 0; i < opt.Payments; i++) + { + var last = (i == opt.Payments - 1); + var amt = last ? remain : RoundMoney((double)rng.NextDouble() * remain * 0.7 + 1.0); + remain = Math.Max(0.0, remain - amt); + doc.Payments[i] = MakePayment(rng, amt, opt.IncludeV2Fields); + } + } + + // Attachments + if (doc.Attachments != null) + { + for (int i = 0; i < opt.Attachments; i++) + doc.Attachments[i] = MakeAttachment(rng, i, opt.AttachmentBytes); + } + + return doc; + } + + /// + /// Create a slightly modified copy of an existing document to test delta/partial updates. + /// Changes: tweak 5–10% of line items (qty/price), add or edit a payment, and bump UpdatedAt. + /// + public static BusinessDocument MakeDelta(BusinessDocument v1, int seed, double changeRatio = 0.07) + { + var rng = new Random(seed); + var v2 = DeepClone(v1); + + v2.Header.UpdatedAt = DateTime.UtcNow; + var toChange = Math.Max(1, (int)Math.Round(v2.Items.Length * changeRatio)); + + // change random lines + for (int i = 0; i < toChange; i++) + { + var idx = rng.Next(0, v2.Items.Length); + var li = v2.Items[idx]; + li.Qty = RoundQty(li.Qty + (double)(rng.NextDouble() * 2.0 - 1.0)); // ±1 + li.UnitPrice = RoundMoney(li.UnitPrice * (double)(0.95 + rng.NextDouble() * 0.1)); // ±5% + if (li.Ext == null) li.Ext = new Map(); + li.Ext["lastEdit"] = VDate(DateTime.UtcNow); + //li.ExtKeys = li.Ext.Keys.ToArray(); + //li.ExtValues = li.Ext.Values.ToArray(); + } + + + if (v2.Payments == null || rng.Next(0, 3) == 0) + { + if (v2.Payments == null) + { + if (v2.Payments == null) v2.Payments = new Payment[1]; + v2.Payments[0] = (MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true)); + + } + else + { + v2.Payments = v2.Payments.Append((MakePayment(rng, RoundMoney((double)rng.NextDouble() * 50.0 + 10.0), includeV2: true))).ToArray(); + } + } + else + { + var p = v2.Payments[rng.Next(0, v2.Payments.Length)]; + p.Fee = (p.Fee ?? 0.0) + 0.25; + p.Reference = "ADJ-" + rng.Next(10000, 99999).ToString(CultureInfo.InvariantCulture); + } + + return v2; + } + + // -------------------------- Builders -------------------------- + + private static int[] RandomRiskScores(Random rng, int count) + { + + var rt = new int[count];// rng.Next(100, 1000)]; + for (var i = 0; i < rt.Length; i++) + rt[i] = (int)rng.Next(); + return rt; + } + + private static Party MakeParty(Random rng, bool includeV2, bool isSeller, bool unicode) + { + return new Party + { + Id = (ulong)rng.NextInt64(), //Guid.NewGuid().ToByteArray(), + Name = unicode ? (isSeller ? "Delta Systems — دلتا" : "Client التجربة ✅") : (isSeller ? "Delta Systems" : "Client Demo"), + TaxId = isSeller ? $"TX-{rng.Next(100000, 999999)}" : null, + Email = (isSeller ? "sales" : "contact") + "@example.com", + Phone = "+964-7" + rng.Next(100000000, 999999999).ToString(CultureInfo.InvariantCulture), + Address = new Address + { + Line1 = rng.Next(0, 2) == 0 ? "Street 14" : "Tech Park", + City = "Baghdad", + Region = "BG", + Country = "IQ", + PostalCode = rng.Next(0, 2) == 0 ? "10001" : null + }, + PreferredLanguage = includeV2 ? (rng.Next(0, 2) == 0 ? "ar-IQ" : "en-US") : null + }; + } + + private static LineItem MakeLineItem(Random rng, int lineNo, bool includeV2, int variantKvp, bool unicode) + { + var isProduct = rng.Next(0, 100) < 80; + var desc = unicode + ? (isProduct ? $"وحدة قياس — Item {lineNo} 🔧" : $"خدمة دعم — Service {lineNo} ✨") + : (isProduct ? $"Item {lineNo}" : $"Service {lineNo}"); + + var li = new LineItem + { + LineNo = lineNo, + Type = isProduct ? LineType.Product : LineType.Service, + SKU = isProduct ? ("SKU-" + rng.Next(1000, 9999)) : "", + Description = desc, + Qty = RoundQty((double)(rng.NextDouble() * 9.0 + 1.0)), // 1..10 + QtyUnit = isProduct ? "pcs" : "h", + UnitPrice = RoundMoney((double)(rng.NextDouble() * 90.0 + 10.0)), + VatRate = rng.Next(0, 100) < 80 ? 5.0 : (double?)null, + Ext = variantKvp > 0 ? new Map() : null, + Discount = includeV2 && rng.Next(0, 3) == 0 ? Math.Round(rng.NextDouble() * 10.0, 2) : (double?)null + }; + + if (li.Ext != null) + { + //li.ExtKeys = li.Ext.Keys.ToArray(); + //li.ExtValues = li.Ext.Values.ToArray(); + } + + + for (int i = 0; i < variantKvp; i++) + { + var key = i switch { 0 => "color", 1 => "size", 2 => "batch", _ => "attr" + i }; + li.Ext!.TryAdd(key, i switch + { + 0 => VStr(rng.Next(0, 3) switch { 0 => "red", 1 => "blue", _ => "green" }), + 1 => VStr(rng.Next(0, 3) switch { 0 => "S", 1 => "M", _ => "L" }), + 2 => VGuid(Guid.NewGuid()), + _ => VInt(rng.Next(0, 1000)) + }); + } + + //li.ExtValues = li.Ext.Values.ToArray(); + //li.ExtKeys = li.Ext.Keys.ToArray(); + + return li; + } + + private static Payment MakePayment(Random rng, double amount, bool includeV2) + { + var p = new Payment + { + Method = (PaymentMethod)rng.Next(0, 5), + Amount = RoundMoney(amount), + Reference = "REF-" + rng.Next(100_000, 999_999), + Timestamp = DateTime.UtcNow.AddMinutes(-rng.Next(0, 60 * 24)), + Fee = includeV2 && rng.Next(0, 2) == 0 ? RoundMoney((double)rng.NextDouble() * 2.0) : null, + //CurrencyOverride = includeV2 && rng.Next(0, 2) == 0 ? Currency.IQD : Currency.USD + }; + + //p.TimestampAsLong = p.Timestamp.Ticks; + + return p; + } + + private static Attachment MakeAttachment(Random rng, int index, int bytes) + { + var arr = bytes > 0 ? new byte[bytes] : Array.Empty(); + if (arr.Length > 0) rng.NextBytes(arr); + return new Attachment + { + Name = $"att-{index + 1}.bin", + MimeType = "application/octet-stream", + Data = arr + }; + } + + private static string SampleNoteUnicode(Random rng) + => rng.Next(0, 2) == 0 + ? "ملاحظة: تم إنشاء هذا المستند لأغراض الاختبار 📦" + : "Note: synthetic benchmark document 🧪"; + + private static string SampleNoteAscii(Random rng) + => rng.Next(0, 2) == 0 ? "Note: synthetic benchmark document" : "Internal use only"; + + // -------------------------- Variant helpers -------------------------- + private static Variant VStr(string s) => new() { Tag = Kind.String, Str = s }; + private static Variant VInt(int v) => new() { Tag = Kind.Int64, I64 = v }; + private static Variant VGuid(Guid g) => new() { Tag = Kind.Guid, Guid = g.ToByteArray() }; + private static Variant VDate(DateTime d) => new() { Tag = Kind.DateTime, Dt = d }; + + // -------------------------- Utils -------------------------- + private static double RoundMoney(double v) => Math.Round(v, 2, MidpointRounding.AwayFromZero); + private static double RoundQty(double v) => Math.Round(v, 3, MidpointRounding.AwayFromZero); + + /// + /// Simple deep clone via manual copy to stay serializer-agnostic. + /// (Good enough for benchmarks; switch to a fast serializer if you like.) + /// + private static BusinessDocument DeepClone(BusinessDocument s) + { + var d = new BusinessDocument + { + Header = new DocumentHeader + { + DocId = s.Header.DocId, + Type = s.Header.Type, + Version = s.Header.Version, + CreatedAt = s.Header.CreatedAt, + UpdatedAt = s.Header.UpdatedAt, + //CreatedAtAsLong = s.Header.CreatedAtAsLong, + //UpdatedAtAsLong = s.Header.UpdatedAtAsLong, + Currency = s.Header.Currency, + Notes = s.Header.Notes, + Meta = s.Header.Meta,//?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)), + //MetaKeys = s.Header.MetaKeys.ToArray(), + //MetaValues = s.Header.MetaValues.ToArray(), + }, + Seller = CloneParty(s.Seller), + Buyer = CloneParty(s.Buyer), + Items = s.Items.Select(CloneLine).ToArray(), + Payments = s.Payments?.Select(ClonePayment).ToArray(), + Attachments = s.Attachments?.Select(CloneAttachment).ToArray(), + RiskScores = s.RiskScores, + //RelatedDocs_v2 = s.RelatedDocs_v2?.ToList() + }; + + + return d; + } + + private static Party CloneParty(Party p) => new() + { + Id = p.Id, + Name = p.Name, + TaxId = p.TaxId, + Email = p.Email, + Phone = p.Phone, + Address = p.Address is null ? null : new Address + { + Line1 = p.Address.Line1, + Line2 = p.Address.Line2, + City = p.Address.City, + Region = p.Address.Region, + Country = p.Address.Country, + PostalCode = p.Address.PostalCode + }, + PreferredLanguage = p.PreferredLanguage + }; + + private static LineItem CloneLine(LineItem s) => new() + { + LineNo = s.LineNo, + Type = s.Type, + SKU = s.SKU, + Description = s.Description, + Qty = s.Qty, + QtyUnit = s.QtyUnit, + UnitPrice = s.UnitPrice, + VatRate = s.VatRate, + Ext = s.Ext,//?.ToDictionary(kv => kv.Key, kv => CloneVariant(kv.Value)), + Discount = s.Discount, + //ExtKeys = s.Ext?.Keys.ToArray(), + //ExtValues = s.Ext?.Values.ToArray(), + }; + + private static Payment ClonePayment(Payment s) => new() + { + Method = s.Method, + Amount = s.Amount, + Reference = s.Reference, + Timestamp = s.Timestamp, + //TimestampAsLong = s.TimestampAsLong, + Fee = s.Fee, + //CurrencyOverride= s.CurrencyOverride + }; + + private static Attachment CloneAttachment(Attachment s) => new() + { + Name = s.Name, + MimeType = s.MimeType, + Data = s.Data.ToArray() + }; + + private static Variant CloneVariant(Variant v) => new() + { + Tag = v.Tag, + Bool = v.Bool, + I64 = v.I64, + U64 = v.U64, + F64 = v.F64, + //Dec = v.Dec, + Str = v.Str, + Bytes = v.Bytes?.ToArray(), + Dt = v.Dt, + //DtAsLong = v.DtAsLong, + Guid = v.Guid + }; +} diff --git a/Tests/RPC/Client/Program.cs b/Tests/RPC/Client/Program.cs index f62b57c..81b691f 100644 --- a/Tests/RPC/Client/Program.cs +++ b/Tests/RPC/Client/Program.cs @@ -36,34 +36,22 @@ for (var i = 0; i < rounds; i++) Console.WriteLine($"\n# Round {round}/{rounds}, seed {seed}"); - var docsWorkloads = DocGenerator.BuildWorkloads(seed); - var dataWorkLoads = Shared.BuildBytesWorkLoads(seed); - var intWorkloads = Shared.BuildIntWorkloads(seed); - - if (runSerialization) - { - Console.WriteLine("Collecting local serialization samples..."); - serializationSamples.AddRange(SerializationExperiment.RunRound( - round, - seed, - docsWorkloads, - dataWorkLoads, - intWorkloads, - serializationIterations)); - } + //var docsWorkloads = DocGenerator.BuildWorkloads(seed); + var docsWorkloads = ModelGenerator.BuildWorkloads(); if (!runRpc) continue; - var thriftDocs = docsWorkloads.ToDictionary(x => x.Key, v => v.Value.Select(x => x.ToThrift()).ToArray()); - var signalRDocs = docsWorkloads.ToDictionary(x => x.Key, v => v.Value.Select(x => x.ToShared()).ToArray()); - var grpcDocs = docsWorkloads.ToDictionary(x => x.Key, v => v.Value.Select(x => x.ToGrpc()).ToArray()); + var esiurWorkload = docsWorkloads.ToDictionary(x => x.Item1, v => v.Item2.Select(x => x.Payload).ToArray()); + var thriftDocs = docsWorkloads.ToDictionary(x => x.Item1, v => v.Item2.Select(x => x.Payload.ToThrift()).ToArray()); + var signalRDocs = docsWorkloads.ToDictionary(x => x.Item1, v => v.Item2.Select(x => x.Payload.ToShared()).ToArray()); + var grpcDocs = docsWorkloads.ToDictionary(x => x.Item1, v => v.Item2.Select(x => x.Payload.ToGrpc()).ToArray()); if (await RunProtocol("esiur", () => EsiurTest.DoTest( "ep://localhost:5005/sys/service", - docsWorkloads, - dataWorkLoads, - intWorkloads, + esiurWorkload, + //dataWorkLoads, + //intWorkloads, warmupDelayMs, postHandshakeDelayMs, sampleDelayMs), @@ -76,8 +64,8 @@ for (var i = 0; i < rounds; i++) "127.0.0.1", 5400, thriftDocs, - dataWorkLoads, - intWorkloads, + //dataWorkLoads, + //intWorkloads, warmupDelayMs, postHandshakeDelayMs, sampleDelayMs), @@ -89,8 +77,8 @@ for (var i = 0; i < rounds; i++) if (await RunProtocol("signalr", () => SignalRTest.DoTest( "http://127.0.0.1:5200/hub/echo", signalRDocs, - dataWorkLoads, - intWorkloads, + //dataWorkLoads, + //intWorkloads, warmupDelayMs, postHandshakeDelayMs, sampleDelayMs), @@ -101,9 +89,9 @@ for (var i = 0; i < rounds; i++) if (await RunProtocol("json", () => JsonTest.DoTest( "http://127.0.0.1:5100", - docsWorkloads, - dataWorkLoads, - intWorkloads, + esiurWorkload, + //dataWorkLoads, + //intWorkloads, warmupDelayMs, postHandshakeDelayMs, sampleDelayMs), @@ -115,8 +103,8 @@ for (var i = 0; i < rounds; i++) if (await RunProtocol("grpc", () => GrpcTest.DoTest( "http://127.0.0.1:5300", grpcDocs, - dataWorkLoads, - intWorkloads, + //dataWorkLoads, + //intWorkloads, warmupDelayMs, postHandshakeDelayMs, sampleDelayMs), diff --git a/Tests/RPC/Client/SignalRTest.cs b/Tests/RPC/Client/SignalRTest.cs index 8206071..584c2ca 100644 --- a/Tests/RPC/Client/SignalRTest.cs +++ b/Tests/RPC/Client/SignalRTest.cs @@ -19,8 +19,8 @@ namespace Esiur.Tests.RPC.Client { public static async Task DoTest(string address, Dictionary docsWorkloads, - Dictionary dataWorkloads, - Dictionary intWorkloads, + //Dictionary dataWorkloads, + //Dictionary intWorkloads, int warmupDelayMs = 3000, int postHandshakeDelayMs = 2000, int sampleDelayMs = 3000) @@ -72,44 +72,44 @@ namespace Esiur.Tests.RPC.Client - foreach (var w in dataWorkloads) - { - Console.Write("Bytes Workload: " + w.Key); - await service.InvokeAsync("EchoBytes", w.Value); + //foreach (var w in dataWorkloads) + //{ + // Console.Write("Bytes Workload: " + w.Key); + // await service.InvokeAsync("EchoBytes", w.Value); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Bytes.Add(w.Key, (ctx, crx)); + // rt.Bytes.Add(w.Key, (ctx, crx)); - } + //} - foreach (var w in intWorkloads) - { - Console.Write("Ints Workload: " + w.Key); - await service.InvokeAsync("EchoIntArray", w.Value); + //foreach (var w in intWorkloads) + //{ + // Console.Write("Ints Workload: " + w.Key); + // await service.InvokeAsync("EchoIntArray", w.Value); - //if (!w.Value.SequenceEqual(rt)) - // throw new Exception("No match"); + // //if (!w.Value.SequenceEqual(rt)) + // // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Ints.Add(w.Key, (ctx, crx)); + // rt.Ints.Add(w.Key, (ctx, crx)); - } + //} await Task.Delay(sampleDelayMs); diff --git a/Tests/RPC/Client/ThriftTest.cs b/Tests/RPC/Client/ThriftTest.cs index 2983b34..cdc47a9 100644 --- a/Tests/RPC/Client/ThriftTest.cs +++ b/Tests/RPC/Client/ThriftTest.cs @@ -11,8 +11,8 @@ public class ThriftTest public static async Task DoTest(string host, int port, Dictionary docsWorkloads, - Dictionary dataWorkloads, - Dictionary intWorkloads, + //Dictionary dataWorkloads, + //Dictionary intWorkloads, int warmupDelayMs = 3000, int postHandshakeDelayMs = 2000, int sampleDelayMs = 3000) @@ -60,46 +60,46 @@ public class ThriftTest - foreach (var w in dataWorkloads) - { - Console.Write("Bytes Workload: " + w.Key); + //foreach (var w in dataWorkloads) + //{ + // Console.Write("Bytes Workload: " + w.Key); - var res = await service.EchoBytes(w.Value); + // var res = await service.EchoBytes(w.Value); - if (!w.Value.SequenceEqual(res)) - throw new Exception("No match"); + // if (!w.Value.SequenceEqual(res)) + // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Bytes.Add(w.Key, (ctx, crx)); + // rt.Bytes.Add(w.Key, (ctx, crx)); - } + //} - foreach (var w in intWorkloads) - { - Console.Write("Ints Workload: " + w.Key); + //foreach (var w in intWorkloads) + //{ + // Console.Write("Ints Workload: " + w.Key); - var res = await service.EchoIntArray(w.Value.ToList()); + // var res = await service.EchoIntArray(w.Value.ToList()); - if (!w.Value.SequenceEqual(res)) - throw new Exception("No match"); + // if (!w.Value.SequenceEqual(res)) + // throw new Exception("No match"); - await Task.Delay(sampleDelayMs); - (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); - Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); - //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); + // await Task.Delay(sampleDelayMs); + // (tx, rx, ctx, crx) = mon.GetDiff(tx, rx); + // Console.WriteLine($", {tx}/{rx}, {ctx}/{crx}"); + // //Console.WriteLine($"Socket {sock.BytesSent}/{sock.BytesReceived}"); - rt.Ints.Add(w.Key, (ctx, crx)); + // rt.Ints.Add(w.Key, (ctx, crx)); - } + //} await Task.Delay(sampleDelayMs);