mirror of
https://github.com/esiur/esiur-dotnet.git
synced 2026-06-13 14:38:43 +00:00
deadlock
This commit is contained in:
@@ -40,9 +40,9 @@ var roots = rootsArg.Equals("all", StringComparison.OrdinalIgnoreCase)
|
||||
|
||||
Console.WriteLine($"[Client] {host}:{port} nodes={nodeCount} mode={mode} roots={roots.Length} " +
|
||||
$"iterations={iterations} stallMs={stallMs} hardMs={hardMs}");
|
||||
Console.WriteLine($"[Client] {"iter",-5}{"outcome",-14}{"ms",10}{"breaks",10}{"unnec",8}{"unpublished",13}");
|
||||
Console.WriteLine($"[Client] {"iter",-5}{"outcome",-13}{"ms",10}{"attached",10}{"breaks",9}{"unnec",8}{"unpub",8}");
|
||||
|
||||
var rows = new List<(int iter, string outcome, double ms, long breaks, long unnec, int unpublished)>();
|
||||
var rows = new List<(int iter, string outcome, double ms, long attached, long breaks, long unnec, int unpublished)>();
|
||||
|
||||
for (var it = 0; it < iterations; it++)
|
||||
{
|
||||
@@ -58,8 +58,8 @@ for (var it = 0; it < iterations; it++)
|
||||
var (outcome, ms, results) = await Classify(con, roots, stallMs, hardMs);
|
||||
var unpublished = results == null ? -1 : CountUnpublished(results);
|
||||
|
||||
rows.Add((it + 1, outcome, ms, con.CycleBreakCount, con.UnnecessaryPlaceholderCount, unpublished));
|
||||
Console.WriteLine($"[Client] {it + 1,-5}{outcome,-14}{ms,10:F1}{con.CycleBreakCount,10}{con.UnnecessaryPlaceholderCount,8}{unpublished,13}");
|
||||
rows.Add((it + 1, outcome, ms, con.AttachedResourceCount, con.CycleBreakCount, con.UnnecessaryPlaceholderCount, unpublished));
|
||||
Console.WriteLine($"[Client] {it + 1,-5}{outcome,-13}{ms,10:F1}{con.AttachedResourceCount,10}{con.CycleBreakCount,9}{con.UnnecessaryPlaceholderCount,8}{unpublished,8}");
|
||||
|
||||
try { con.Destroy(); } catch { }
|
||||
}
|
||||
@@ -73,17 +73,20 @@ Console.WriteLine();
|
||||
Console.WriteLine($"[Client] === summary ({mode}) ===");
|
||||
Console.WriteLine($" completed={completed.Count} deadlocked={rows.Count(r => r.outcome == "Deadlocked")} " +
|
||||
$"slow={rows.Count(r => r.outcome == "SlowTimeout")} faulted={rows.Count(r => r.outcome == "Faulted")}");
|
||||
Console.WriteLine($" resources attached per run (max)={rows.Max(r => r.attached)}");
|
||||
Console.WriteLine($" completion ms: median={Pct(0.5):F1} p99={Pct(0.99):F1} max={(times.Count > 0 ? times[^1] : 0):F1}");
|
||||
Console.WriteLine($" cycle-breaks total={rows.Sum(r => r.breaks)} unnecessary-placeholders total={rows.Sum(r => r.unnec)}");
|
||||
Console.WriteLine($" partial deliveries (unpublished>0) in {rows.Count(r => r.unpublished > 0)}/{rows.Count} runs");
|
||||
|
||||
var csv = "iteration,outcome,ms,cycle_breaks,unnecessary_placeholders,unpublished\n" +
|
||||
string.Join("\n", rows.Select(r => $"{r.iter},{r.outcome},{r.ms:F1},{r.breaks},{r.unnec},{r.unpublished}"));
|
||||
var csv = "iteration,outcome,ms,attached,cycle_breaks,unnecessary_placeholders,unpublished\n" +
|
||||
string.Join("\n", rows.Select(r => $"{r.iter},{r.outcome},{r.ms:F1},{r.attached},{r.breaks},{r.unnec},{r.unpublished}"));
|
||||
var outFile = $"deadlock_{mode}_{host}_{port}.csv";
|
||||
await File.WriteAllTextAsync(outFile, csv);
|
||||
Console.WriteLine($"[Client] results written to {outFile}");
|
||||
|
||||
Console.ReadLine();
|
||||
// Keep the window open only when run interactively; scripted/redirected runs exit immediately.
|
||||
if (!Console.IsInputRedirected)
|
||||
Console.ReadLine();
|
||||
|
||||
// ---- stall-based classification ---------------------------------------------------------------
|
||||
|
||||
@@ -131,7 +134,9 @@ static async Task<(string outcome, double ms, EpResource[]? results)> Classify(
|
||||
}
|
||||
|
||||
// Counts resources reachable from the delivered roots that are not Published — i.e. handed to the
|
||||
// application while their dependency graph was not fully attached. Links is property index 1.
|
||||
// application while their dependency graph was not fully attached. Traverses every reference-typed
|
||||
// property (Node.Links/Resources1/Resources2 and the Resource1/Resource2 cross-references) so the
|
||||
// whole delivered graph is checked, not just the node links.
|
||||
static int CountUnpublished(EpResource[] roots)
|
||||
{
|
||||
var seen = new HashSet<uint>();
|
||||
@@ -145,14 +150,26 @@ static int CountUnpublished(EpResource[] roots)
|
||||
|
||||
if (node.Status != ResourceStatus.Published) unpublished++;
|
||||
|
||||
if (node.Status == ResourceStatus.Attached && node.TryGetPropertyValue((byte)1, out var linksObj) && linksObj is IEnumerable links)
|
||||
foreach (var child in links)
|
||||
if (child is EpResource childResource)
|
||||
queue.Enqueue(childResource);
|
||||
// Only attached/published resources can be safely read for further references.
|
||||
if (node.Status != ResourceStatus.Attached && node.Status != ResourceStatus.Published) continue;
|
||||
|
||||
var properties = node.Instance.Definition.Properties.Length;
|
||||
for (byte p = 0; p < properties; p++)
|
||||
if (node.TryGetPropertyValue(p, out var value))
|
||||
Flatten(value, queue);
|
||||
}
|
||||
return unpublished;
|
||||
}
|
||||
|
||||
static void Flatten(object? value, Queue<EpResource> queue)
|
||||
{
|
||||
if (value is EpResource resource)
|
||||
queue.Enqueue(resource);
|
||||
else if (value is IEnumerable sequence && value is not string)
|
||||
foreach (var item in sequence)
|
||||
Flatten(item, queue);
|
||||
}
|
||||
|
||||
static string GetArg(string[] args, string key, string def)
|
||||
{
|
||||
var i = Array.IndexOf(args, key);
|
||||
|
||||
@@ -25,11 +25,45 @@ var res2Count = int.Parse(GetArg(args, "--res2", "100"));
|
||||
var seed = int.Parse(GetArg(args, "--seed", "20260603"));
|
||||
var edgeProb = double.Parse(GetArg(args, "--edge-prob", "0.22"));
|
||||
|
||||
var edges = BuildTopology(topology, ref nodeCount, seed, edgeProb);
|
||||
var (hasCycle, backEdges) = CycleCensus(nodeCount, edges);
|
||||
var nodeEdges = BuildTopology(topology, ref nodeCount, seed, edgeProb);
|
||||
|
||||
Console.WriteLine($"[Server] topology={topology} nodes={nodeCount} edges={edges.Count} " +
|
||||
$"cyclic={hasCycle} backEdges={backEdges} port={port}");
|
||||
// One RNG, seeded once, for all random assignment. (Previously a new Random(seed) was created
|
||||
// inside each loop, so every node/resource pointed at the same target and the cycle structure
|
||||
// collapsed; one RNG yields a genuinely random, densely cyclic resource graph.)
|
||||
var rng = new Random(seed);
|
||||
|
||||
// Plan the resource cross-references as indices first, so the FULL graph (nodes + Resource1 +
|
||||
// Resource2 + every reference) can be censused for circular dependencies before it is wired.
|
||||
var nodeRes1 = new int[nodeCount][];
|
||||
var nodeRes2 = new int[nodeCount][];
|
||||
for (var i = 0; i < nodeCount; i++)
|
||||
{
|
||||
nodeRes1[i] = Sample(rng, res1Count, res1Count / 2);
|
||||
nodeRes2[i] = Sample(rng, res2Count, res2Count / 2);
|
||||
}
|
||||
|
||||
var res1Ref1 = new int[res1Count];
|
||||
var res1Ref2 = new int[res1Count];
|
||||
for (var i = 0; i < res1Count; i++)
|
||||
{
|
||||
res1Ref1[i] = res1Count > 0 ? rng.Next(res1Count) : -1;
|
||||
res1Ref2[i] = res2Count > 0 ? rng.Next(res2Count) : -1;
|
||||
}
|
||||
|
||||
var res2Ref1 = new int[res2Count];
|
||||
var res2Ref2 = new int[res2Count];
|
||||
for (var i = 0; i < res2Count; i++)
|
||||
{
|
||||
res2Ref1[i] = res1Count > 0 ? rng.Next(res1Count) : -1;
|
||||
res2Ref2[i] = res2Count > 0 ? rng.Next(res2Count) : -1;
|
||||
}
|
||||
|
||||
var totalResources = nodeCount + res1Count + res2Count;
|
||||
var (hasCycle, backEdges, totalEdges) = FullCensus(
|
||||
nodeCount, res1Count, res2Count, nodeEdges, nodeRes1, nodeRes2, res1Ref1, res1Ref2, res2Ref1, res2Ref2);
|
||||
|
||||
Console.WriteLine($"[Server] topology={topology} nodes={nodeCount} res1={res1Count} res2={res2Count} " +
|
||||
$"totalResources={totalResources} edges={totalEdges} cyclic={hasCycle} backEdges={backEdges} port={port}");
|
||||
|
||||
var wh = new Warehouse();
|
||||
await wh.Put("sys", new MemoryStore());
|
||||
@@ -41,52 +75,28 @@ var nodes = new Node[nodeCount];
|
||||
var resources1 = new Resource1[res1Count];
|
||||
var resources2 = new Resource2[res2Count];
|
||||
|
||||
for (var i = 0; i < nodeCount; i++) {
|
||||
nodes[i] = new Node { Id = i };
|
||||
await wh.Put($"sys/n{i}", nodes[i]);
|
||||
}
|
||||
for (var i = 0; i < nodeCount; i++) { nodes[i] = new Node { Id = i }; await wh.Put($"sys/n{i}", nodes[i]); }
|
||||
for (var i = 0; i < res1Count; i++) { resources1[i] = new Resource1(); await wh.Put($"sys/r1_{i}", resources1[i]); }
|
||||
for (var i = 0; i < res2Count; i++) { resources2[i] = new Resource2(); await wh.Put($"sys/r2_{i}", resources2[i]); }
|
||||
|
||||
// Wire the planned references: each Node also pulls in a random subset of Resource1/Resource2, and
|
||||
// the resources cross-reference one another, creating dense cycles for the fetch to resolve.
|
||||
for (var i = 0; i < nodeCount; i++)
|
||||
{
|
||||
nodes[i].Resources1 = nodeRes1[i].Select(k => resources1[k]).ToArray();
|
||||
nodes[i].Resources2 = nodeRes2[i].Select(k => resources2[k]).ToArray();
|
||||
}
|
||||
for (var i = 0; i < res1Count; i++)
|
||||
{
|
||||
resources1[i] = new Resource1();
|
||||
await wh.Put($"sys/r1_{i}", resources1[i]);
|
||||
if (res1Ref1[i] >= 0) resources1[i].res1 = resources1[res1Ref1[i]];
|
||||
if (res1Ref2[i] >= 0) resources1[i].res2 = resources2[res1Ref2[i]];
|
||||
}
|
||||
|
||||
for (var i = 0; i < res2Count; i++)
|
||||
{
|
||||
resources2[i] = new Resource2();
|
||||
await wh.Put($"sys/r2_{i}", resources2[i]);
|
||||
if (res2Ref1[i] >= 0) resources2[i].res1 = resources1[res2Ref1[i]];
|
||||
if (res2Ref2[i] >= 0) resources2[i].res2 = resources2[res2Ref2[i]];
|
||||
}
|
||||
|
||||
// randomly assign some resources to each node so the fetches do some work beyond just traversing the links; this also
|
||||
for(var i = 0; i < nodeCount; i++)
|
||||
{
|
||||
var rng = new Random(seed);
|
||||
|
||||
|
||||
nodes[i].Resources1 = rng.GetItems(resources1, res1Count / 2);
|
||||
nodes[i].Resources2 = rng.GetItems(resources2, res2Count / 2);
|
||||
}
|
||||
|
||||
for(var i =0; i < res1Count; i++)
|
||||
{
|
||||
var rng = new Random(seed);
|
||||
var res1Index = rng.Next(res1Count);
|
||||
var res2Index = rng.Next(res2Count);
|
||||
resources1[i].res1 = resources1[res1Index];
|
||||
resources1[i].res2 = resources2[res2Index];
|
||||
}
|
||||
|
||||
for (var i = 0; i < res2Count; i++)
|
||||
{
|
||||
var rng = new Random(seed);
|
||||
var res1Index = rng.Next(res1Count);
|
||||
var res2Index = rng.Next(res2Count);
|
||||
resources2[i].res1 = resources1[res1Index];
|
||||
resources2[i].res2 = resources2[res2Index];
|
||||
}
|
||||
|
||||
foreach (var grp in edges.GroupBy(e => e.from))
|
||||
foreach (var grp in nodeEdges.GroupBy(e => e.from))
|
||||
nodes[grp.Key].Links = grp.Select(e => nodes[e.to]).ToArray();
|
||||
|
||||
await wh.Open();
|
||||
@@ -152,16 +162,54 @@ static List<(int from, int to)> BuildTopology(string topo, ref int n, int seed,
|
||||
return edges;
|
||||
}
|
||||
|
||||
// DFS three-colouring; counts back edges (cycle-closing edges, including self loops).
|
||||
static (bool hasCycle, int backEdges) CycleCensus(int n, IReadOnlyList<(int from, int to)> edges)
|
||||
// k indices drawn (with replacement) from [0, count); empty if count or k is 0.
|
||||
static int[] Sample(Random rng, int count, int k)
|
||||
{
|
||||
var adj = new List<int>[n];
|
||||
for (var i = 0; i < n; i++) adj[i] = new List<int>();
|
||||
var back = 0;
|
||||
foreach (var (a, b) in edges) { if (a == b) back++; else adj[a].Add(b); }
|
||||
if (count <= 0 || k <= 0) return Array.Empty<int>();
|
||||
var result = new int[k];
|
||||
for (var i = 0; i < k; i++) result[i] = rng.Next(count);
|
||||
return result;
|
||||
}
|
||||
|
||||
var color = new byte[n]; // 0 unvisited, 1 on-stack, 2 done
|
||||
for (var s = 0; s < n; s++)
|
||||
// Censuses the FULL request graph — Node Links + Node->Resource1/2 + Resource1/2 cross-references —
|
||||
// for circular dependencies via DFS three-colouring. Vertices: [0..nodes) nodes, then res1, then
|
||||
// res2. Returns whether the graph is cyclic, the number of cycle-closing (back) edges, and the
|
||||
// total edge count.
|
||||
static (bool hasCycle, int backEdges, int totalEdges) FullCensus(
|
||||
int nodes, int r1, int r2,
|
||||
IReadOnlyList<(int from, int to)> nodeEdges,
|
||||
int[][] nodeRes1, int[][] nodeRes2,
|
||||
int[] res1Ref1, int[] res1Ref2, int[] res2Ref1, int[] res2Ref2)
|
||||
{
|
||||
var v = nodes + r1 + r2;
|
||||
int R1(int i) => nodes + i;
|
||||
int R2(int i) => nodes + r1 + i;
|
||||
|
||||
var adj = new List<int>[v];
|
||||
for (var i = 0; i < v; i++) adj[i] = new List<int>();
|
||||
var total = 0;
|
||||
void Add(int a, int b) { adj[a].Add(b); total++; }
|
||||
|
||||
foreach (var (a, b) in nodeEdges) Add(a, b);
|
||||
for (var i = 0; i < nodes; i++)
|
||||
{
|
||||
foreach (var k in nodeRes1[i]) Add(i, R1(k));
|
||||
foreach (var k in nodeRes2[i]) Add(i, R2(k));
|
||||
}
|
||||
for (var i = 0; i < r1; i++)
|
||||
{
|
||||
if (res1Ref1[i] >= 0) Add(R1(i), R1(res1Ref1[i]));
|
||||
if (res1Ref2[i] >= 0) Add(R1(i), R2(res1Ref2[i]));
|
||||
}
|
||||
for (var i = 0; i < r2; i++)
|
||||
{
|
||||
if (res2Ref1[i] >= 0) Add(R2(i), R1(res2Ref1[i]));
|
||||
if (res2Ref2[i] >= 0) Add(R2(i), R2(res2Ref2[i]));
|
||||
}
|
||||
|
||||
var back = 0;
|
||||
var color = new byte[v]; // 0 unvisited, 1 on-stack, 2 done
|
||||
for (var s = 0; s < v; s++)
|
||||
{
|
||||
if (color[s] != 0) continue;
|
||||
var stack = new Stack<(int node, int idx)>();
|
||||
@@ -172,14 +220,15 @@ static (bool hasCycle, int backEdges) CycleCensus(int n, IReadOnlyList<(int from
|
||||
if (idx < adj[u].Count)
|
||||
{
|
||||
stack.Push((u, idx + 1));
|
||||
var v = adj[u][idx];
|
||||
if (color[v] == 1) back++;
|
||||
else if (color[v] == 0) { color[v] = 1; stack.Push((v, 0)); }
|
||||
var w = adj[u][idx];
|
||||
if (w == u) back++; // self-loop
|
||||
else if (color[w] == 1) back++; // back edge -> cycle
|
||||
else if (color[w] == 0) { color[w] = 1; stack.Push((w, 0)); }
|
||||
}
|
||||
else color[u] = 2;
|
||||
}
|
||||
}
|
||||
return (back > 0, back);
|
||||
return (back > 0, back, total);
|
||||
}
|
||||
|
||||
static string GetArg(string[] args, string key, string def)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"nodes","res1","res2","totalResources","backEdges","attached","cycleBreaks","medianMs","completed","deadlocked"
|
||||
"25","25","25","75","20","75","42","23.3","3","0"
|
||||
"50","50","50","150","39","150","90","79.8","3","0"
|
||||
"100","100","100","300","88","300","159","273.7","3","0"
|
||||
"200","200","200","600","149","600","243","670.0","3","0"
|
||||
"400","400","400","1200","329","1200","486","5649.9","3","0"
|
||||
"800","800","800","2400","614","2400","975","56724.2","3","0"
|
||||
|
@@ -0,0 +1,9 @@
|
||||
| nodes | res1 | res2 | total resources | back-edges | attached/run | cycle-breaks(3 runs) | median ms | completed | deadlocked |
|
||||
|------:|-----:|-----:|----------------:|-----------:|-------------:|---------------------:|----------:|----------:|-----------:|
|
||||
| 25 | 25 | 25 | 75 | 20 | 75 | 42 | 23.3 | 3 | 0 |
|
||||
| 50 | 50 | 50 | 150 | 39 | 150 | 90 | 79.8 | 3 | 0 |
|
||||
| 100 | 100 | 100 | 300 | 88 | 300 | 159 | 273.7 | 3 | 0 |
|
||||
| 200 | 200 | 200 | 600 | 149 | 600 | 243 | 670.0 | 3 | 0 |
|
||||
| 400 | 400 | 400 | 1200 | 329 | 1200 | 486 | 5649.9 | 3 | 0 |
|
||||
| 800 | 800 | 800 | 2400 | 614 | 2400 | 975 | 56724.2 | 3 | 0 |
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Scalability sweep for the distributed deadlock test (loopback).
|
||||
# For each size, starts a fresh server hosting a ring of `nodes` plus `res1`+`res2` densely
|
||||
# cross-referencing resources, then runs the client (WaitWithCycleDetection) and records the
|
||||
# graph size, back-edges, cycle-breaks, and completion time. Output: sweep-results.csv / .md
|
||||
$ErrorActionPreference = "SilentlyContinue"
|
||||
Set-Location $PSScriptRoot\..\..\..
|
||||
|
||||
$srv = "Tests\Distribution\Deadlock\Server\bin\Release\net10.0\Esiur.Tests.Deadlock.Server.exe"
|
||||
$cli = "Tests\Distribution\Deadlock\Client\bin\Release\net10.0\Esiur.Tests.Deadlock.Client.exe"
|
||||
|
||||
$sizes = @(25, 50, 100, 200, 400, 800)
|
||||
$port = 11200
|
||||
$rows = @()
|
||||
|
||||
foreach ($s in $sizes) {
|
||||
$port++
|
||||
Get-Process -Name "Esiur.Tests.Deadlock.Server" -ErrorAction SilentlyContinue | Stop-Process -Force
|
||||
$so = "Tests\Distribution\Deadlock\sweep_srv_$s.txt"
|
||||
Remove-Item $so -ErrorAction SilentlyContinue
|
||||
$p = Start-Process -FilePath $srv -ArgumentList "--port $port --topology ring --nodes $s --res1 $s --res2 $s" -PassThru -NoNewWindow -RedirectStandardOutput $so
|
||||
|
||||
# Wait until the server reports it is listening (poll up to 90s).
|
||||
$census = ""
|
||||
for ($t = 0; $t -lt 180; $t++) {
|
||||
Start-Sleep -Milliseconds 500
|
||||
$c = Get-Content $so -ErrorAction SilentlyContinue
|
||||
if ($c -match "Listening") { $census = ($c | Select-Object -First 1); break }
|
||||
}
|
||||
|
||||
$backEdges = if ($census -match "backEdges=(\d+)") { $matches[1] } else { "?" }
|
||||
|
||||
# Run the client; parse its summary.
|
||||
$out = & $cli --host 127.0.0.1 --port $port --nodes $s --mode WaitWithCycleDetection --iterations 3 --stall-ms 30000 --hard-ms 180000 2>&1
|
||||
$total = $s * 3
|
||||
$median = ($out | Select-String "completion ms: median=([\d.]+)").Matches.Groups[1].Value
|
||||
$attached = ($out | Select-String "resources attached per run \(max\)=(\d+)").Matches.Groups[1].Value
|
||||
$breaks = ($out | Select-String "cycle-breaks total=(\d+)").Matches.Groups[1].Value
|
||||
$completed = ($out | Select-String "completed=(\d+)").Matches.Groups[1].Value
|
||||
$deadlocked = ($out | Select-String "deadlocked=(\d+)").Matches.Groups[1].Value
|
||||
|
||||
$rows += [pscustomobject]@{ nodes=$s; res1=$s; res2=$s; totalResources=$total; backEdges=$backEdges; attached=$attached; cycleBreaks=$breaks; medianMs=$median; completed=$completed; deadlocked=$deadlocked }
|
||||
Write-Output "size=$s total=$total back=$backEdges attached=$attached breaks=$breaks median=$median ms completed=$completed"
|
||||
|
||||
Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue
|
||||
Get-Process -Name "Esiur.Tests.Deadlock.Server" -ErrorAction SilentlyContinue | Stop-Process -Force
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
|
||||
$rows | Export-Csv -Path "Tests\Distribution\Deadlock\sweep-results.csv" -NoTypeInformation
|
||||
$md = "| nodes | res1 | res2 | total resources | back-edges | attached/run | cycle-breaks(3 runs) | median ms | completed | deadlocked |`n"
|
||||
$md += "|------:|-----:|-----:|----------------:|-----------:|-------------:|---------------------:|----------:|----------:|-----------:|`n"
|
||||
foreach ($r in $rows) { $md += "| $($r.nodes) | $($r.res1) | $($r.res2) | $($r.totalResources) | $($r.backEdges) | $($r.attached) | $($r.cycleBreaks) | $($r.medianMs) | $($r.completed) | $($r.deadlocked) |`n" }
|
||||
Set-Content -Path "Tests\Distribution\Deadlock\sweep-results.md" -Value $md -Encoding utf8
|
||||
Write-Output "=== DONE ==="
|
||||
Write-Output $md
|
||||
Reference in New Issue
Block a user