Deadlock tests

Libraries/Esiur/Data/Codec.cs

@@ -489,7 +489,9 @@ public static class Codec
         [typeof(Map<object?, object>)] = DataSerializer.MapComposer,
         [typeof(Map<object, object?>)] = DataSerializer.MapComposer,
         [typeof(Map<object?, object?>)] = DataSerializer.MapComposer,
-        [typeof(PropertyValue[])] = DataSerializer.PropertyValueArrayComposer
+        [typeof(PropertyValue[])] = DataSerializer.PropertyValueArrayComposer,
+        // Sparse property delta for the reattach reply (index -> value/age/date).
+        [typeof(Map<byte, PropertyValue>)] = DataSerializer.PropertyValueMapComposer
         // Typed
         // [typeof(bool[])] = (value, con) => DataSerializer.TypedListComposer((IEnumerable)value, typeof(bool), con),
         // [typeof(bool?[])] = (value, con) => (TransmissionDataUnitIdentifier.TypedList, new byte[] { (byte)value }),

Libraries/Esiur/Data/DataDeserializer.cs

@@ -1898,6 +1898,31 @@ public static class DataDeserializer
 
     }
 
+    /// <summary>
+    /// Parses a sparse property delta produced by <c>PropertyValueMapComposer</c> (the reattach
+    /// reply): a flat sequence of (index, age, date, value) TDUs, returned as a map keyed by the
+    /// property index. Mirrors <see cref="PropertyValueArrayParserAsync"/> but in groups of four.
+    /// </summary>
+    public static AsyncReply<Map<byte, PropertyValue>> PropertyValueMapParserAsync(byte[] data, uint offset, uint length, EpConnection connection, uint[] requestSequence)
+    {
+        var rt = new AsyncReply<Map<byte, PropertyValue>>();
+
+        ListParserAsync(new ParsedTdu() { Data = data, PayloadOffset = offset, PayloadLength = length }
+                        , connection, requestSequence).Then(x =>
+        {
+            var ar = (object[])x;
+            var map = new Map<byte, PropertyValue>();
+
+            for (var i = 0; i + 3 < ar.Length; i += 4)
+                map[Convert.ToByte(ar[i])] =
+                    new PropertyValue(ar[i + 3], Convert.ToUInt64(ar[i + 1]), (DateTime?)ar[i + 2]);
+
+            rt.Trigger(map);
+        });
+
+        return rt;
+    }
+
 
     public static async AsyncReply<ParseResult<PropertyValue>> PropertyValueParserAsync(byte[] data, uint offset, EpConnection connection, uint[] requestSequence)//, bool ageIncluded = true)
     {

Libraries/Esiur/Data/DataSerializer.cs

@@ -630,6 +630,31 @@ public static class DataSerializer
             (uint)rt.Count, null, null);
     }
 
+    /// <summary>
+    /// Composes a sparse property delta (index -> value/age/date) used by the reattach reply, as
+    /// a flat sequence of (index, age, date, value) TDUs per modified property. PropertyValue is
+    /// not a self-describing type, so this dedicated composer is used instead of the generic map
+    /// path. Mirrors <see cref="PropertyValueArrayComposer"/> with a leading property index.
+    /// </summary>
+    public static Tdu PropertyValueMapComposer(object value, Warehouse warehouse, EpConnection connection)
+    {
+        if (value == null)
+            return new Tdu(TduIdentifier.Null, new byte[0], 0, null, null);
+
+        var rt = new List<byte>();
+        var map = (Map<byte, PropertyValue>)value;
+
+        foreach (var kv in map)
+        {
+            rt.AddRange(Codec.Compose(kv.Key, warehouse, connection));          // property index (u8)
+            rt.AddRange(Codec.Compose(kv.Value.Age, warehouse, connection));    // age
+            rt.AddRange(Codec.Compose(kv.Value.Date, warehouse, connection));   // modification date
+            rt.AddRange(Codec.Compose(kv.Value.Value, warehouse, connection));  // value
+        }
+
+        return new Tdu(TduIdentifier.RawData, rt.ToArray(), (uint)rt.Count, null, null);
+    }
+
     public static Tdu TypedMapComposer(object value, Type keyType, Type valueType, Warehouse warehouse, EpConnection connection)
     {
         if (value == null)

Libraries/Esiur/Esiur.csproj

@@ -36,6 +36,11 @@
 		<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
 	</PropertyGroup>
 
+	<ItemGroup>
+		<!-- Allow the unit test project to exercise internal helpers (e.g. wait-for cycle detection). -->
+		<InternalsVisibleTo Include="Esiur.Tests.Unit" />
+	</ItemGroup>
+
 	<ItemGroup>
 		<Compile Remove="obj\**" />
 		<EmbeddedResource Remove="obj\**" />

Libraries/Esiur/Net/NetworkServer.cs

@@ -158,19 +158,18 @@ public abstract class NetworkServer<TConnection> : IDestructible where TConnecti
 
         try
         {
-            if (listener != null)
+            var currentListener = listener;
+            if (currentListener != null)
             {
-                port = listener.LocalEndPoint.Port;
-                listener.Close();
+                // Reading the endpoint can throw if the socket is already disposed (e.g. a second
+                // Stop or the finalizer after Destroy), so it is best-effort and only used for logging.
+                try { port = currentListener.LocalEndPoint.Port; } catch { }
+                try { currentListener.Close(); } catch { }
+                listener = null; // make Stop idempotent
             }
-            var cons = Connections.ToArray();
-
-            //lock (connections.SyncRoot)
-            //{
-            foreach (TConnection con in cons)
-                con.Close();
-            //}
 
+            foreach (TConnection con in Connections.ToArray())
+                try { con.Close(); } catch { }
         }
         finally
         {
@@ -206,6 +205,7 @@ public abstract class NetworkServer<TConnection> : IDestructible where TConnecti
     {
         Stop();
         OnDestroy?.Invoke(this);
+        GC.SuppressFinalize(this); // explicit teardown done; no need for the finalizer to run Stop again
     }
 
     private void ClientDisconnectedEventReceiver(NetworkConnection connection)
@@ -228,7 +228,8 @@ public abstract class NetworkServer<TConnection> : IDestructible where TConnecti
 
     ~NetworkServer()
     {
-        Stop();
+        // Finalizers must never throw; Stop() is already guarded but wrap defensively.
+        try { Stop(); } catch { }
         listener = null;
     }
 }

Libraries/Esiur/Protocol/DeadlockResolutionMode.cs

@@ -0,0 +1,31 @@
+namespace Esiur.Protocol;
+
+/// <summary>
+/// Strategy used by <c>EpConnection.FetchResource</c> when it is asked for a resource whose
+/// attachment is already in flight. Selectable mainly for experimental A/B/C evaluation of the
+/// deadlock-prevention algorithm.
+/// </summary>
+public enum DeadlockResolutionMode : byte
+{
+    /// <summary>
+    /// Default. Wait for the in-flight attachment to complete, except when a genuine wait-for cycle
+    /// is detected (same dependency chain, or a cross-chain cycle in the wait-for graph), in which
+    /// case a placeholder is returned to break it. Never deadlocks and never returns an unnecessary
+    /// placeholder.
+    /// </summary>
+    WaitWithCycleDetection = 0,
+
+    /// <summary>
+    /// Legacy behaviour: return the not-yet-attached placeholder to any cross-chain requester of an
+    /// in-flight resource. Never deadlocks, but delivers partially-attached resources for non-cyclic
+    /// contention (the bug under study).
+    /// </summary>
+    LegacyCrossChainPlaceholder = 1,
+
+    /// <summary>
+    /// No cycle handling at all: always wait for the in-flight attachment, even within the same
+    /// dependency chain. Genuinely deadlocks whenever the request graph contains a cycle. Used only
+    /// to demonstrate that cycle handling is necessary and that the deadlock detector works.
+    /// </summary>
+    NaiveWait = 2,
+}

Libraries/Esiur/Protocol/EpConnection.cs

@@ -797,7 +797,22 @@ public partial class EpConnection : NetworkConnection, IStore
                 }
                 else if (_authPacket.Command == EpAuthPacketCommand.Acknowledge)
                 {
-                    if (_authPacket.Method == EpAuthPacketMethod.ProceedToHandshake 
+                    // Anonymous (None-mode) success: the responder establishes the session directly
+                    // via SessionEstablished, without a handshake exchange. Complete the connection so
+                    // the pending open request resolves. (Previously this was only handled inside the
+                    // ProceedToHandshake branch, so a direct SessionEstablished left the initiator hung.)
+                    if (_session.AuthenticationMode == AuthenticationMode.None
+                        && _authPacket.Method == EpAuthPacketMethod.SessionEstablished)
+                    {
+                        _session.Authenticated = true;
+                        _session.LocalIdentity = null;
+                        _session.RemoteIdentity = null;
+                        _session.Key = null;
+                        AuthenticatonCompleted();
+                        return offset;
+                    }
+
+                    if (_authPacket.Method == EpAuthPacketMethod.ProceedToHandshake
                         || _authPacket.Method == EpAuthPacketMethod.ProceedToFinalHandshake)
                     {
                         var remoteHeaders
@@ -1948,7 +1963,9 @@ public partial class EpConnection : NetworkConnection, IStore
 
                         _neededResources[id] = r;
 
-                        await FetchResource(id, null);
+                        // Reattach using the last-known age so only properties modified while
+                        // disconnected are transferred and merged, instead of re-fetching all.
+                        await Reattach(id, r.Instance.Age, r);
 
                         Global.Log("EpConnection", LogType.Debug, "Restored " + id);
 

Libraries/Esiur/Protocol/EpConnectionProtocol.cs

@@ -57,6 +57,27 @@ partial class EpConnection
     KeyList<uint, WeakReference<EpResource>> _attachedResources = new KeyList<uint, WeakReference<EpResource>>();
     KeyList<uint, WeakReference<EpResource>> _suspendedResources = new KeyList<uint, WeakReference<EpResource>>();
     KeyList<uint, FetchRequestInfo<EpResource, uint>> _resourceRequests = new KeyList<uint, FetchRequestInfo<EpResource, uint>>();
+
+    // Wait-for graph for in-flight resource fetches: maps a resource id to the set of in-flight
+    // child resource ids its attachment is currently blocked on. Used to detect genuine cycles
+    // (e.g. two concurrent fetches A<->B) so a placeholder can break the deadlock, while
+    // independent/app-facing fetches of an in-flight resource simply wait for full attachment.
+    readonly Dictionary<uint, HashSet<uint>> _fetchBlockedOn = new Dictionary<uint, HashSet<uint>>();
+
+    /// <summary>
+    /// Strategy FetchResource uses for an in-flight resource. Defaults to the new wait + cycle
+    /// detection. Selectable for experimental evaluation (see <see cref="DeadlockResolutionMode"/>).
+    /// </summary>
+    public DeadlockResolutionMode DeadlockResolution { get; set; } = DeadlockResolutionMode.WaitWithCycleDetection;
+
+    // Per-connection diagnostics (free of the cross-connection contamination that the shared
+    // Global.Counters suffer from). Used by the deadlock experiments.
+    /// <summary>Number of resources fully attached on this connection (a monotonic progress signal).</summary>
+    public long AttachedResourceCount { get; private set; }
+    /// <summary>Number of wait-for-cycle breaks (placeholders returned to break a cycle) on this connection.</summary>
+    public long CycleBreakCount { get; private set; }
+    /// <summary>Number of placeholders returned where no genuine cycle existed (legacy resolver only).</summary>
+    public long UnnecessaryPlaceholderCount { get; private set; }
     //KeyList<ulong, AsyncReply<RemoteTypeDef>> _typeDefsByIdRequests = new KeyList<ulong, AsyncReply<RemoteTypeDef>>();
 
     //KeyList<string, AsyncReply<RemoteTypeDef>> _typeDefsByNameRequests = new KeyList<string, AsyncReply<RemoteTypeDef>>();
@@ -1969,6 +1990,11 @@ partial class EpConnection
 
         req.Then(result =>
             {
+                // The resource is being handed to the application: publish its fully-attached
+                // graph so that, if any dependency is only partially attached, it stays unpublished.
+                if (result is EpResource resource)
+                    PublishGraph(resource);
+
                 rt.Trigger(result);
             }).Error(ex => rt.TriggerError(ex));
 
@@ -2026,6 +2052,103 @@ partial class EpConnection
     /// <returns>DistributedResource</returns>
     /// 
     //object fetchResourceLock = new object();
+    // Records that the attachment of `parent` is now blocked waiting on in-flight child `child`.
+    void AddFetchBlock(uint parent, uint child)
+    {
+        if (!_fetchBlockedOn.TryGetValue(parent, out var set))
+            _fetchBlockedOn[parent] = set = new HashSet<uint>();
+        set.Add(child);
+    }
+
+    // Removes a resource from the wait-for graph once it is attached or its fetch failed: it is
+    // no longer blocked on anything and no longer a pending child of anyone.
+    void ClearFetchNode(uint id)
+    {
+        _fetchBlockedOn.Remove(id);
+        foreach (var set in _fetchBlockedOn.Values)
+            set.Remove(id);
+    }
+
+    /// <summary>
+    /// Returns true if completing the fetch of <paramref name="id"/> by waiting for its in-flight
+    /// request would deadlock, i.e. the resource is (transitively) blocked on a resource that the
+    /// current request chain is itself building. In that case the caller should hand back the
+    /// placeholder to break the cycle instead of waiting.
+    /// </summary>
+    internal static bool HasWaitForCycle(uint id, uint[] requestSequence, IReadOnlyDictionary<uint, HashSet<uint>> blockedOn)
+    {
+        if (requestSequence == null || requestSequence.Length == 0)
+            return false;
+
+        var chain = new HashSet<uint>(requestSequence);
+        var visited = new HashSet<uint>();
+        var stack = new Stack<uint>();
+        stack.Push(id);
+
+        while (stack.Count > 0)
+        {
+            var current = stack.Pop();
+            if (!visited.Add(current))
+                continue;
+
+            if (!blockedOn.TryGetValue(current, out var children))
+                continue;
+
+            foreach (var child in children)
+            {
+                // Reaching a node that the current chain is attaching closes the cycle.
+                if (chain.Contains(child))
+                    return true;
+                stack.Push(child);
+            }
+        }
+
+        return false;
+    }
+
+    /// <summary>
+    /// Publishes a fully-attached object graph to the application: every resource reachable from
+    /// <paramref name="root"/> is marked <see cref="ResourceStatus.Published"/>, but only if the
+    /// entire reachable graph is already attached. If any reachable resource is still being
+    /// attached (e.g. a placeholder handed out to break a cycle), the graph is left unpublished —
+    /// exactly the partially-attached delivery that the wait-by-default resolver prevents and the
+    /// legacy resolver does not.
+    /// </summary>
+    internal void PublishGraph(EpResource root)
+    {
+        if (root == null)
+            return;
+
+        var seen = new HashSet<uint>();
+        var reachable = new List<EpResource>();
+        var queue = new Queue<EpResource>();
+        queue.Enqueue(root);
+
+        var fullyAttached = true;
+
+        while (queue.Count > 0)
+        {
+            var node = queue.Dequeue();
+            if (node == null || !seen.Add(node.ResourceInstanceId))
+                continue;
+
+            reachable.Add(node);
+
+            if (node.Status != ResourceStatus.Attached)
+            {
+                fullyAttached = false;
+                continue; // do not traverse into a not-yet-attached node
+            }
+
+            foreach (var child in node.GetReferencedResources())
+                queue.Enqueue(child);
+        }
+
+        if (fullyAttached)
+            foreach (var node in reachable)
+                node.Publish();
+    }
+
     public AsyncReply<EpResource> FetchResource(uint id, uint[] requestSequence)
     {
         //lock (fetchLock)
@@ -2044,27 +2167,64 @@ partial class EpConnection
 
         var requestInfo = _resourceRequests[id];
 
+        // The resource that triggered this fetch (the tail of the chain), if any. Used to record
+        // wait-for edges and to tell graph-internal references from app-facing fetches (no chain).
+        uint? parent = requestSequence != null && requestSequence.Length > 0
+            ? requestSequence[requestSequence.Length - 1]
+            : (uint?)null;
+
         if (requestInfo != null)
         {
-            if (resource != null && (requestSequence?.Contains(id) ?? false))
+            // Same dependency chain (A->B->A): the placeholder is an internal node of the graph
+            // currently being attached. The application only observes the chain's top-level reply,
+            // which fires after full attachment, so returning the not-yet-attached placeholder here
+            // is safe and breaks the reference cycle. NaiveWait skips this so that even same-chain
+            // cycles deadlock (used to demonstrate the protection is necessary).
+            if (DeadlockResolution != DeadlockResolutionMode.NaiveWait
+                && resource != null && (requestSequence?.Contains(id) ?? false))
             {
                 Global.Counters["EpResourceDeadLockSameChain"]++;
-                // dead lock avoidance for loop reference.
+                CycleBreakCount++;
                 return new AsyncReply<EpResource>(resource);
             }
-            else if (resource != null && requestInfo.RequestSequence.Contains(id))
+
+            // Decide whether to break the wait by returning the placeholder:
+            //  - Legacy: hand it to ANY cross-chain requester (over-eager; the bug under study).
+            //  - WaitWithCycleDetection: only on a genuine wait-for cycle.
+            //  - NaiveWait: never — always wait below (deadlocks on cycles).
+            var breakCycle = resource != null && DeadlockResolution switch
+            {
+                DeadlockResolutionMode.LegacyCrossChainPlaceholder => requestInfo.RequestSequence.Contains(id),
+                DeadlockResolutionMode.WaitWithCycleDetection => HasWaitForCycle(id, requestSequence, _fetchBlockedOn),
+                _ => false,
+            };
+
+            if (breakCycle)
             {
                 Global.Counters["EpResourceDeadLockCrossChain"]++;
-                // dead lock avoidance for dependent reference.
+                CycleBreakCount++;
+
+                // Instrumentation: a placeholder handed out where there is no genuine wait-for cycle
+                // is an unnecessary, partial delivery — the new resolver would have waited for full
+                // attachment instead. This counts the legacy resolver's over-eager placeholders.
+                if (DeadlockResolution == DeadlockResolutionMode.LegacyCrossChainPlaceholder
+                    && !HasWaitForCycle(id, requestSequence, _fetchBlockedOn))
+                {
+                    Global.Counters["EpResourceUnnecessaryPlaceholder"]++;
+                    UnnecessaryPlaceholderCount++;
+                }
+
                 return new AsyncReply<EpResource>(resource);
             }
-            else
-            {
-                Global.Counters["EpResourcePendingCacheHit"]++;
-                return requestInfo.Reply;
-            }
+
+            // Otherwise an independent or application-facing requester: wait for the in-flight
+            // attachment to complete fully rather than exposing a partially attached resource.
+            Global.Counters["EpResourcePendingCacheHit"]++;
+            if (parent != null)
+                AddFetchBlock(parent.Value, id);
+            return requestInfo.Reply;
         }
-        else if (resource != null && !resource.ResourceSuspended)
+        else if (resource != null && resource.Status != ResourceStatus.Suspended)
         {
             // @REVIEW: this should never happen
             Global.Log("DCON", LogType.Error, "Resource not moved to attached.");
@@ -2077,6 +2237,10 @@ partial class EpConnection
         var reply = new AsyncReply<EpResource>();
         _resourceRequests.Add(id, new FetchRequestInfo<EpResource, uint>(reply, newSequence));
 
+        // This fetch's parent now waits on `id` until it attaches.
+        if (parent != null)
+            AddFetchBlock(parent.Value, id);
+
         SendRequest(EpPacketRequest.AttachResource, id)
                     .Then((result) =>
                     {
@@ -2113,12 +2277,19 @@ partial class EpConnection
                                 var pvs = results as PropertyValue[];
 
                                 dr._Attach(pvs);
+                                // Progress signal: a resource has fully attached. Used by tests to
+                                // distinguish a true deadlock (no progress while requests pend) from
+                                // merely slow processing (these counters keep advancing).
+                                Global.Counters["EpResourceAttached"]++;
+                                AttachedResourceCount++;
                                 _resourceRequests.Remove(id);
                                 // move from needed to attached.
                                 _neededResources.Remove(id);
                                 _attachedResources[id] = new WeakReference<EpResource>(dr);
+                                // attached: no longer part of the in-flight wait-for graph.
+                                ClearFetchNode(id);
                                 reply.Trigger(dr);
-                            }).Error(ex => reply.TriggerError(ex));
+                            }).Error(ex => { _resourceRequests.Remove(id); ClearFetchNode(id); reply.TriggerError(ex); });
                         };
 
                         if (typeDef == null)
@@ -2135,6 +2306,9 @@ partial class EpConnection
 
                                     resource.ResourceDefinition = td;
                                     typeDef = td;
+                                    // Register the placeholder before parsing properties so cyclic
+                                    // references in the graph can resolve back to this instance.
+                                    _neededResources[id] = resource;
                                     Instance.Warehouse.Put(Instance.Link + "/" + id.ToString(), resource)
                                         .Then(initResource)
                                         .Error(ex => reply.TriggerError(ex));
@@ -2160,6 +2334,9 @@ partial class EpConnection
 
                                 resource.ResourceDefinition = typeDef;
 
+                                // Register the placeholder before parsing properties so cyclic
+                                // references in the graph can resolve back to this instance.
+                                _neededResources[id] = resource;
                                 Instance.Warehouse.Put(this.Instance.Link + "/" + id.ToString(), resource)
                                     .Then(initResource).Error((ex) => reply.TriggerError(ex));
                             }
@@ -2171,6 +2348,10 @@ partial class EpConnection
 
                     }).Error((ex) =>
                     {
+                        // Failed to attach: drop the in-flight request and wait-for edges so a
+                        // later retry is not blocked by a stale entry.
+                        _resourceRequests.Remove(id);
+                        ClearFetchNode(id);
                         reply.TriggerError(ex);
                     });
 
@@ -2187,6 +2368,69 @@ partial class EpConnection
     /// <param name="id">Resource Id</param>
     /// <returns>DistributedResource</returns>
     /// 
+    /// <summary>
+    /// Re-attaches an already-known resource after reconnection using its last-known age. The peer
+    /// returns only the properties modified after <paramref name="age"/> (the delta), which are
+    /// merged into the existing instance instead of re-fetching everything. Falls back to a full
+    /// <see cref="FetchResource"/> if there is no prior state to merge into.
+    /// </summary>
+    public AsyncReply<EpResource> Reattach(uint id, ulong age, EpResource resource)
+    {
+        EpResource attachedResource = null;
+        _attachedResources[id]?.TryGetTarget(out attachedResource);
+        if (attachedResource != null)
+            return new AsyncReply<EpResource>(attachedResource);
+
+        var existing = _resourceRequests[id];
+        if (existing != null)
+            return existing.Reply;
+
+        var reply = new AsyncReply<EpResource>();
+        var sequence = new uint[] { id };
+        _resourceRequests.Add(id, new FetchRequestInfo<EpResource, uint>(reply, sequence));
+
+        SendRequest(EpPacketRequest.ReattachResource, id, age).Then(result =>
+        {
+            if (result == null)
+            {
+                _resourceRequests.Remove(id);
+                reply.TriggerError(new AsyncException(ErrorType.Management,
+                        (ushort)ExceptionCode.ResourceNotFound, "Null response"));
+                return;
+            }
+
+            // typeId, age, link, hops, delta(index -> PropertyValue)
+            var args = (object[])result;
+            var deltaData = (byte[])args[4];
+
+            DataDeserializer.PropertyValueMapParserAsync(deltaData, 0, (uint)deltaData.Length, this, sequence)
+                .Then(delta =>
+                {
+                    if (!resource._Reattach(delta))
+                    {
+                        // No prior state to merge into — perform a full attach instead.
+                        _resourceRequests.Remove(id);
+                        FetchResource(id, null).Then(r => reply.Trigger(r)).Error(ex => reply.TriggerError(ex));
+                        return;
+                    }
+
+                    _resourceRequests.Remove(id);
+                    _neededResources.Remove(id);
+                    _attachedResources[id] = new WeakReference<EpResource>(resource);
+                    ClearFetchNode(id);
+                    reply.Trigger(resource);
+                })
+                .Error(ex => { _resourceRequests.Remove(id); ClearFetchNode(id); reply.TriggerError(ex); });
+        }).Error(ex =>
+        {
+            _resourceRequests.Remove(id);
+            ClearFetchNode(id);
+            reply.TriggerError(ex);
+        });
+
+        return reply;
+    }
+
     //object fetchResourceLock = new object();
     public AsyncReply<RemoteTypeDef> FetchTypeDef(ulong id, ulong[] requestSequence)
     {

Libraries/Esiur/Protocol/EpResource.cs

@@ -56,25 +56,29 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     //public event PropertyModifiedEvent PropertyModified;
     public event PropertyChangedEventHandler PropertyChanged;
 
-    uint instanceId;
-    TypeDef typeDef;
-    EpConnection connection;
+    uint _instanceId;
+    TypeDef _typeDef;
+    EpConnection _connection;
 
 
-    bool attached = false;
-    bool destroyed = false;
-    bool suspended = false;
+    // Single explicit lifecycle state, replacing the former attached/destroyed/suspended booleans.
+    Resource.ResourceStatus _status = Resource.ResourceStatus.Pending;
+
+    // Internal read-only views kept so the existing guard checks read naturally.
+    //bool attached => status == Resource.ResourceStatus.Attached || status == Resource.ResourceStatus.Published;
+    //bool destroyed => status == Resource.ResourceStatus.Destroyed;
+    //bool suspended => status == Resource.ResourceStatus.Suspended;
 
     //Structure properties = new Structure();
 
-    string link;
-    ulong age;
+    string _link;
+    ulong _age;
 
-    protected object[] properties;
-    internal List<EpResource> parents = new List<EpResource>();
-    internal List<EpResource> children = new List<EpResource>();
+    protected object[] _properties;
+    //internal List<EpResource> parents = new List<EpResource>();
+    //internal List<EpResource> children = new List<EpResource>();
 
-    EpResourceEvent[] events;
+    EpResourceEvent[] _events;
 
 
 
@@ -83,7 +87,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     /// </summary>
     public EpConnection ResourceConnection
     {
-        get { return connection; }
+        get { return _connection; }
     }
 
     /// <summary>
@@ -91,7 +95,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     /// </summary>
     public string ResourceLink
     {
-        get { return link; }
+        get { return _link; }
     }
 
     /// <summary>
@@ -99,8 +103,8 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     /// </summary>
     public uint ResourceInstanceId
     {
-        get { return instanceId; }
-        internal set { instanceId = value; }
+        get { return _instanceId; }
+        internal set { _instanceId = value; }
     }
 
     /// <summary>
@@ -108,9 +112,8 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     /// </summary>
     public void Destroy()
     {
-        destroyed = true;
-        attached = false;
-        connection.SendDetachRequest(instanceId);
+        _status = Resource.ResourceStatus.Destroyed;
+        _connection.SendDetachRequest(_instanceId);
         OnDestroy?.Invoke(this);
     }
 
@@ -120,17 +123,69 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     internal void Suspend()
     {
-        suspended = true;
-        attached = false;
+        _status = Resource.ResourceStatus.Suspended;
     }
 
+    /// <summary>
+    /// Marks the resource as published: attached and delivered to the application as part of a
+    /// fully-attached object graph. A resource only transitions Attached -&gt; Published.
+    /// </summary>
+    internal void Publish()
+    {
+        if (_status == Resource.ResourceStatus.Attached)
+            _status = Resource.ResourceStatus.Published;
+    }
 
     /// <summary>
-    /// Resource is attached when all its properties are received.
+    /// The resource's current lifecycle state. Only <see cref="Resource.ResourceStatus.Published"/>
+    /// guarantees the resource and its whole dependency graph are ready for application use.
     /// </summary>
-    public bool ResourceAttached => attached;
+    public Resource.ResourceStatus Status => _status;
 
-    public bool ResourceSuspended => suspended;
+    /// <summary>
+    /// Resource is attached when all its own properties are received (it may be Published too).
+    /// </summary>
+    //public bool ResourceAttached => attached;
+
+    //public bool ResourceSuspended => suspended;
+
+    /// <summary>True once the resource has been published to the application.</summary>
+    //public bool ResourcePublished => status == Resource.ResourceStatus.Published;
+
+    /// <summary>
+    /// Enumerates the distributed resources directly referenced by this resource's property values
+    /// (including those nested inside arrays/lists/maps). Used to walk the dependency graph when
+    /// publishing a fully-attached graph to the application.
+    /// </summary>
+    internal IEnumerable<EpResource> GetReferencedResources()
+    {
+        if (_properties == null)
+            yield break;
+
+        foreach (var value in _properties)
+            foreach (var resource in FlattenResources(value))
+                yield return resource;
+    }
+
+    static IEnumerable<EpResource> FlattenResources(object value)
+    {
+        if (value is EpResource resource)
+        {
+            yield return resource;
+        }
+        else if (value is System.Collections.IDictionary dictionary)
+        {
+            foreach (var item in dictionary.Values)
+                foreach (var r in FlattenResources(item))
+                    yield return r;
+        }
+        else if (value is System.Collections.IEnumerable sequence && !(value is string))
+        {
+            foreach (var item in sequence)
+                foreach (var r in FlattenResources(item))
+                    yield return r;
+        }
+    }
 
 
     // public DistributedResourceStack Stack
@@ -146,40 +201,65 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     /// <param name="age">Resource age.</param>
     public EpResource(EpConnection connection, uint instanceId, ulong age, string link)
     {
-        this.link = link;
-        this.connection = connection;
-        this.instanceId = instanceId;
-        this.age = age;
+        this._link = link;
+        this._connection = connection;
+        this._instanceId = instanceId;
+        this._age = age;
     }
 
     internal bool _Attach(PropertyValue[] properties)
     {
-        if (attached)
+        if (_status == ResourceStatus.Attached)
             return false;
-        else
+
+        _properties = new object[properties.Length];
+
+        _events = new EpResourceEvent[Instance.Definition.Events.Length];
+
+        for (byte i = 0; i < properties.Length; i++)
         {
-            suspended = false;
-
-            this.properties = new object[properties.Length];
-
-            this.events = new EpResourceEvent[Instance.Definition.Events.Length];
-
-            for (byte i = 0; i < properties.Length; i++)
-            {
-                Instance.SetAge(i, properties[i].Age);
-                Instance.SetModificationDate(i, properties[i].Date);
-                this.properties[i] = properties[i].Value;
-            }
-
-            // trigger holded events/property updates.
-            //foreach (var r in afterAttachmentTriggers)
-            //    r.Key.Trigger(r.Value);
-
-            //afterAttachmentTriggers.Clear();
-
-            attached = true;
-
+            Instance.SetAge(i, properties[i].Age);
+            Instance.SetModificationDate(i, properties[i].Date);
+            this._properties[i] = properties[i].Value;
         }
+
+        // trigger holded events/property updates.
+        //foreach (var r in afterAttachmentTriggers)
+        //    r.Key.Trigger(r.Value);
+
+        //afterAttachmentTriggers.Clear();
+
+        _status = Resource.ResourceStatus.Attached;
+
+
+        return true;
+    }
+
+    /// <summary>
+    /// Re-attaches a previously attached (then suspended) resource after reconnection by merging
+    /// only the properties that changed while disconnected. The peer returns just the delta — the
+    /// properties whose age is newer than the age this side last knew — so unchanged properties
+    /// keep their existing value/age/date. Returns false if the resource was never attached (no
+    /// prior state to merge into), in which case the caller should perform a full attach.
+    /// </summary>
+    /// <param name="delta">Modified properties keyed by their property index.</param>
+    internal bool _Reattach(Map<byte, PropertyValue> delta)
+    {
+        if (_properties == null || _events == null)
+            return false; // no prior state — caller should perform a full attach instead.
+
+        foreach (var kv in delta)
+        {
+            var index = kv.Key;
+            if (index >= _properties.Length)
+                continue;
+
+            Instance.SetAge(index, kv.Value.Age);
+            Instance.SetModificationDate(index, kv.Value.Date);
+            _properties[index] = kv.Value.Value;
+        }
+
+        _status = Resource.ResourceStatus.Attached;
         return true;
     }
 
@@ -187,16 +267,17 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     protected internal virtual void _EmitEventByIndex(byte index, object args)
     {
         var et = Instance.Definition.GetEventDefByIndex(index);
-        events[index]?.Invoke(this, args);
+        _events[index]?.Invoke(this, args);
         Instance.EmitResourceEvent(et, args);
     }
 
     public AsyncReply _Invoke(byte index, object args)
     {
-        if (destroyed)
+
+        if (_status == ResourceStatus.Destroyed)
             throw new Exception("Trying to access a destroyed object.");
 
-        if (suspended)
+        if (_status == ResourceStatus.Suspended)
             throw new Exception("Trying to access a suspended object.");
 
         if (index >= Instance.Definition.Functions.Length)
@@ -208,9 +289,9 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
             throw new Exception("Function definition not found.");
 
         if (ft.IsStatic)
-            return connection.StaticCall(Instance.Definition.Id, index, args);
+            return _connection.StaticCall(Instance.Definition.Id, index, args);
         else
-            return connection.SendInvoke(instanceId, index, args);
+            return _connection.SendInvoke(_instanceId, index, args);
     }
 
     public AsyncReply Subscribe(EventDef et)
@@ -229,7 +310,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
             return rt;
         }
 
-        return connection.SendSubscribeRequest(instanceId, et.Index);
+        return _connection.SendSubscribeRequest(_instanceId, et.Index);
     }
 
     public AsyncReply Subscribe(string eventName)
@@ -244,7 +325,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     {
         if (et == null)
         {
-            var rt = new  AsyncReply();
+            var rt = new AsyncReply();
             rt.TriggerError(new AsyncException(ErrorType.Management, (ushort)ExceptionCode.MethodNotFound, ""));
             return rt;
         }
@@ -256,7 +337,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
             return rt;
         }
 
-        return connection.SendUnsubscribeRequest(instanceId, et.Index);
+        return _connection.SendUnsubscribeRequest(_instanceId, et.Index);
     }
 
     public AsyncReply Unsubscribe(string eventName)
@@ -269,61 +350,63 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     public override bool TryInvokeMember(InvokeMemberBinder binder, object[] args, out object result)
     {
-        if (destroyed)
+        if (_status == ResourceStatus.Destroyed)
             throw new Exception("Trying to access a destroyed object.");
 
-        if (suspended)
+        if (_status == ResourceStatus.Suspended)
             throw new Exception("Trying to access a suspended object.");
 
-        var ft = Instance.Definition.GetFunctionDefByName(binder.Name);
 
-        var reply = new AsyncReply<object>();
-
-        if (attached && ft != null)
-        {
-
-            if (args.Length == 1)
-            {
-                // Detect anonymous types
-                var type = args[0].GetType();
-
-
-                if (Codec.IsAnonymous(type))
-                {
-                    var indexedArgs = new Map<byte, object>();
-
-                    var pis = type.GetProperties();
-
-                    for (byte i = 0; i < ft.Arguments.Length; i++)
-                    {
-                        var pi = pis.FirstOrDefault(x => x.Name == ft.Arguments[i].Name);
-                        if (pi != null)
-                            indexedArgs.Add(i, pi.GetValue(args[0]));
-                    }
-
-                    result = _Invoke(ft.Index, indexedArgs);
-                }
-                else if (args[0] is object[] || args[0] is Map<byte, object>)
-                {
-                    result = _Invoke(ft.Index, new object[] { args });
-                }
-                else
-                {
-                    result = _Invoke(ft.Index, args);
-                }
-            }
-            else
-            {
-
-                result = _Invoke(ft.Index, args);
-            }
-            return true;
-        }
-        else
+        if (_status != ResourceStatus.Attached)
         {
             result = null;
             return false;
         }
+
+        var ft = Instance.Definition.GetFunctionDefByName(binder.Name) 
+            ?? throw new Exception($"{binder.Name} does not exist");
+
+        var reply = new AsyncReply<object>();
+
+
+        if (args.Length == 1)
+        {
+            // Detect anonymous types
+            var type = args[0].GetType();
+
+
+            if (Codec.IsAnonymous(type))
+            {
+                var indexedArgs = new Map<byte, object>();
+
+                var pis = type.GetProperties();
+
+                for (byte i = 0; i < ft.Arguments.Length; i++)
+                {
+                    var pi = pis.FirstOrDefault(x => x.Name == ft.Arguments[i].Name);
+                    if (pi != null)
+                        indexedArgs.Add(i, pi.GetValue(args[0]));
+                }
+
+                result = _Invoke(ft.Index, indexedArgs);
+            }
+            else if (args[0] is object[] || args[0] is Map<byte, object>)
+            {
+                result = _Invoke(ft.Index, new object[] { args });
+            }
+            else
+            {
+                result = _Invoke(ft.Index, args);
+            }
+        }
+        else
+        {
+
+            result = _Invoke(ft.Index, args);
+        }
+
+        return true;
+
     }
 
     ///// <summary>
@@ -337,34 +420,34 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     public bool TryGetPropertyValue(byte index, out object value)
     {
-        if (index >= properties.Length)
+        if (index >= _properties.Length)
         {
             value = null;
             return false;
         }
         else
         {
-            value = properties[index];
+            value = _properties[index];
             return true;
         }
     }
 
     public override bool TryGetMember(GetMemberBinder binder, out object result)
     {
-        if (destroyed)
+        if (_status == ResourceStatus.Destroyed)
             throw new Exception("Trying to access a destroyed object.");
 
 
         result = null;
 
-        if (!attached)
+        if (_status != ResourceStatus.Attached)
             return false;
 
         var pt = Instance.Definition.GetPropertyDefByName(binder.Name);
 
         if (pt != null)
         {
-            result = properties[pt.Index];
+            result = _properties[pt.Index];
             return true;
         }
         else
@@ -373,7 +456,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
             if (et == null)
                 return false;
 
-            result = events[et.Index];
+            result = _events[et.Index];
 
             return true;
         }
@@ -383,7 +466,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     internal void _UpdatePropertyByIndex(byte index, object value)
     {
         var pt = Instance.Definition.GetPropertyDefByIndex(index);
-        properties[index] = value;
+        _properties[index] = value;
         Instance.EmitModification(pt, value);
     }
 
@@ -409,13 +492,13 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     public override bool TrySetMember(SetMemberBinder binder, object value)
     {
-        if (destroyed)
+        if (_status == ResourceStatus.Destroyed)
             throw new Exception("Trying to access a destroyed object.");
 
-        if (suspended)
+        if (_status == ResourceStatus.Suspended)
             throw new Exception("Trying to access a suspended object.");
 
-        if (!attached)
+        if (_status != ResourceStatus.Attached)
             return false;
 
         var pt = Instance.Definition.GetPropertyDefByName(binder.Name);
@@ -431,7 +514,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
             if (et == null)
                 return false;
 
-            events[et.Index] = (EpResourceEvent)value;
+            _events[et.Index] = (EpResourceEvent)value;
 
             return true;
         }
@@ -452,11 +535,11 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     {
         get
         {
-            return typeDef;
+            return _typeDef;
         }
         internal set
         {
-            typeDef = value;
+            _typeDef = value;
         }
     }
 
@@ -494,10 +577,10 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     public PropertyValue[] SerializeResource()
     {
-        var props = new PropertyValue[properties.Length];
+        var props = new PropertyValue[_properties.Length];
 
-        for (byte i = 0; i < properties.Length; i++)
-            props[i] = new PropertyValue(properties[i],
+        for (byte i = 0; i < _properties.Length; i++)
+            props[i] = new PropertyValue(_properties[i],
                                         Instance.GetAge(i),
                                         Instance.GetModificationDate(i));
 
@@ -508,9 +591,9 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     {
         var rt = new Map<byte, PropertyValue>();
 
-        for (byte i = 0; i < properties.Length; i++)
+        for (byte i = 0; i < _properties.Length; i++)
             if (Instance.GetAge(i) > age)
-                rt.Add(i, new PropertyValue(properties[i],
+                rt.Add(i, new PropertyValue(_properties[i],
                                             Instance.GetAge(i),
                                             Instance.GetModificationDate(i)));
 
@@ -523,33 +606,33 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
 
     public object GetResourceProperty(byte index)
     {
-        if (index >= properties.Length)
+        if (index >= _properties.Length)
             return null;
-        return properties[index];
+        return _properties[index];
     }
 
     public AsyncReply SetResourcePropertyAsync(byte index, object value)
     {
-        if (destroyed)
+        if (_status == ResourceStatus.Destroyed)
             throw new Exception("Trying to access a destroyed object.");
 
-        if (suspended)
+        if (_status == ResourceStatus.Suspended)
             throw new Exception("Trying to access a suspended object.");
 
-        if (!attached)
+        if (_status != ResourceStatus.Attached)
             throw new Exception("Resource is not attached.");
 
-        if (index >= properties.Length)
+        if (index >= _properties.Length)
             throw new Exception("Property index not found."); ;
 
         var reply = new AsyncReply<object>();
 
-        connection.SendSetProperty(instanceId, index, value)
+        _connection.SendSetProperty(_instanceId, index, value)
                     .Then((res) =>
                     {
                         // not really needed, server will always send property modified, 
                         // this only happens if the programmer forgot to emit in property setter
-                        properties[index] = value;
+                        _properties[index] = value;
                         reply.Trigger(null);
                     });
 
@@ -560,7 +643,7 @@ public class EpResource : DynamicObject, IResource, INotifyPropertyChanged, IDyn
     public void SetResourceProperty(byte index, object value)
     {
         // Don't set the same current value
-        if (properties[index] == value)
+        if (_properties[index] == value)
             return;
 
         SetResourcePropertyAsync(index, value).Wait();

Libraries/Esiur/Resource/ResourceStatus.cs

@@ -0,0 +1,31 @@
+namespace Esiur.Resource;
+
+/// <summary>
+/// Lifecycle state of a distributed (remote) resource on the consuming side. Replaces the former
+/// separate attached/suspended/destroyed booleans with a single explicit state machine.
+/// </summary>
+public enum ResourceStatus : byte
+{
+    /// <summary>Created as a placeholder; its properties have not been received yet.</summary>
+    Pending = 0,
+
+    /// <summary>
+    /// Its own properties have been received and merged, but its dependency graph may still be
+    /// incomplete (e.g. it was used to break a reference cycle). Not yet safe to hand to the
+    /// application as fully ready.
+    /// </summary>
+    Attached = 1,
+
+    /// <summary>
+    /// Attached and delivered to the application as part of a fully-attached object graph. This is
+    /// the only state in which a resource — including every resource it depends on — is guaranteed
+    /// ready for application use.
+    /// </summary>
+    Published = 2,
+
+    /// <summary>The connection was lost; the resource is awaiting reattachment.</summary>
+    Suspended = 3,
+
+    /// <summary>The resource has been detached/destroyed and must not be accessed.</summary>
+    Destroyed = 4,
+}