LLM

Tests/Annotations/LlmRunner.cs

@@ -1,24 +1,51 @@
-using Esiur.Schema.Llm;
+using Esiur.Resource;
+using Esiur.Schema.Llm;
+using Esiur.Stores;
 using OpenAI;
 using OpenAI.Chat;
 using System;
 using System.ClientModel;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Net.NetworkInformation;
 using System.Text;
 using System.Text.Json;
 
-namespace Esiur.Tests.Annotations
+namespace Esiur.Tests.Annotations;
+
+//public sealed class TickState
+//{
+//    public int Load { get; set; }
+//    public int ErrorCount { get; set; }
+//    public bool Enabled { get; set; }
+//}
+
+//public sealed class LlmDecision
+//{
+//    public string? Function { get; set; }
+//    public string? Reason { get; set; }
+//}
+
+
+
+public sealed class LlmRunner
 {
-    public class LlmRunner
+    private static readonly HashSet<string?> ValidFunctions = new(StringComparer.OrdinalIgnoreCase)
     {
-        public async Task RunAsync(ServiceNode node, string endpoint, ApiKeyCredential apiKey, string modelName,
+        null, "Restart", "ResetErrors", "Enable", "Disable"
+    };
+
+    public async Task<(List<TickResult> Results, List<ModelSummary> Summary)> RunAsync(
+        
+        IReadOnlyList<ModelConfig> models,
         int tickDelayMs = 1000)
     {
-            var client = new OpenAIClient(apiKey, new OpenAIClientOptions() { Endpoint = new Uri(endpoint) });
-            var chat = client.GetChatClient("microsoft/phi-4");
 
-            var typeModel = LlmTypeModel.FromTypeDef(node.Instance?.Definition);
+        var wh = new Warehouse();
+
+        await wh.Put("store", new MemoryStore());
+
+        var allResults = new List<TickResult>();
 
         var ticks = new List<TickState>
         {
@@ -30,102 +57,209 @@ namespace Esiur.Tests.Annotations
             new() { Load = 25, ErrorCount = 0, Enabled = true  }
         };
 
+        var expectations = new List<TickExpectation>
+        {
+            new() { Tick = 1, AllowedFunctions = new HashSet<string?> { null }, Note = "Stable service; no action expected." },
+            new() { Tick = 2, AllowedFunctions = new HashSet<string?> { "Restart" }, Note = "Overload; restart expected." },
+            new() { Tick = 3, AllowedFunctions = new HashSet<string?> { "Restart", "ResetErrors" }, Note = "High error count; restart or reset is acceptable." },
+            new() { Tick = 4, AllowedFunctions = new HashSet<string?> { "Enable" }, Note = "Service disabled; enable expected." },
+            new() { Tick = 5, AllowedFunctions = new HashSet<string?> { "Restart" }, Note = "Overload and instability; restart expected." },
+            new() { Tick = 6, AllowedFunctions = new HashSet<string?> { null }, Note = "Stable service; no action expected." }
+        };
+
+        foreach (var model in models)
+        {
+            Console.WriteLine($"=== Model: {model.Name} ({model.ModelName}) ===");
+
+            var client = new OpenAIClient(
+                model.ApiKey,
+                new OpenAIClientOptions { Endpoint = new Uri(model.Endpoint) });
+
+            var chat = client.GetChatClient(model.ModelName);
+
+            Console.WriteLine($"Warming up {model.Name}...");
+
+            await InferAsync(chat,
+                "Return {\"function\":null,\"reason\":\"warmup\"}");
+
+            Console.WriteLine("Warmup done");
+
+            // Fresh node instance per model so results are independent.
+            var node = await wh.Put("store/service-" + model.Name, new ServiceNode());
+
+            var typeModel = LlmTypeModel.FromTypeDef(node.Instance?.Definition);
+
             for (int i = 0; i < ticks.Count; i++)
             {
                 var tick = ticks[i];
+                var expected = expectations[i];
 
-                // Simulate property changes for this tick
+                // Apply tick state before inference
                 node.Load = tick.Load;
                 node.ErrorCount = tick.ErrorCount;
                 node.Enabled = tick.Enabled;
 
+                var loadBefore = node.Load;
+                var errorBefore = node.ErrorCount;
+                var enabledBefore = node.Enabled;
+
                 var jsonModel = typeModel.ToJson(node);
-                Console.WriteLine($"Tick {i + 1}");
-                Console.WriteLine($"State: Load={node.Load}, ErrorCount={node.ErrorCount}, Enabled={node.Enabled}");
+                var prompt = BuildPrompt(jsonModel, i + 1);
 
-                var prompt = BuildPrompt(jsonModel, node, i + 1);
+                var sw = Stopwatch.StartNew();
+                string raw = await InferAsync(chat, prompt);
+                sw.Stop();
 
-                string llmRaw = await InferAsync(chat, prompt);
-                var decision = ParseDecision(llmRaw);
+                var parsedResult = ParseDecisionWithRepair(raw);
 
-                bool invoked = InvokeIfValid(node, decision?.Function);
+                var firstDecision = parsedResult.First;
+                var finalDecision = parsedResult.Final;
 
-                Console.WriteLine($"LLM: {llmRaw}");
-                Console.WriteLine($"Invoked: {invoked}");
-                Console.WriteLine($"After: Load={node.Load}, ErrorCount={node.ErrorCount}, Enabled={node.Enabled}");
-                Console.WriteLine(new string('-', 60));
+                var parsed = finalDecision != null;
+                var repaired = parsedResult.Repaired;
+                var jsonObjectCount = parsedResult.Count;
 
+                var firstPredicted = NormalizeFunction(firstDecision?.Function);
+                var predicted = NormalizeFunction(finalDecision?.Function);
+
+                var allowed = ValidFunctions.Contains(predicted);
+                var correct = expected.AllowedFunctions.Contains(predicted);
+
+                var invoked = false;
+                if (allowed)
+                    invoked = InvokeIfValid(node, predicted);
+
+                var result = new TickResult
+                {
+                    Model = model.Name,
+                    Tick = i + 1,
+
+                    LoadBefore = loadBefore,
+                    ErrorCountBefore = errorBefore,
+                    EnabledBefore = enabledBefore,
+
+                    RawResponse = raw,
+                    FirstPredictedFunction = firstPredicted,
+                    PredictedFunction = predicted,
+                    Reason = finalDecision?.Reason,
+
+                    Parsed = parsed,
+                    Allowed = allowed,
+                    Correct = correct,
+                    Repaired = repaired,
+                    JsonObjectCount = jsonObjectCount,
+                    Invoked = invoked,
+                    LatencyMs = sw.Elapsed.TotalMilliseconds,
+
+                    LoadAfter = node.Load,
+                    ErrorCountAfter = node.ErrorCount,
+                    EnabledAfter = node.Enabled,
+
+                    ExpectedText = string.Join(" | ", expected.AllowedFunctions.Select(x => x ?? "null"))
+                };
+
+                allResults.Add(result);
+
+                Console.WriteLine($"Tick {result.Tick}");
+                Console.WriteLine($"Before: Load={result.LoadBefore}, ErrorCount={result.ErrorCountBefore}, Enabled={result.EnabledBefore}");
+                Console.WriteLine($"Expected: {result.ExpectedText}");
+                Console.WriteLine($"LLM: {result.RawResponse}");
+                Console.WriteLine($"First: {result.FirstPredictedFunction ?? "null"}");
+                Console.WriteLine($"Final: {result.PredictedFunction ?? "null"}");
+                Console.WriteLine($"Parsed={result.Parsed}, Allowed={result.Allowed}, Correct={result.Correct}, Repaired={result.Repaired}, Invoked={result.Invoked}, Latency={result.LatencyMs:F1} ms");
+                Console.WriteLine($"After: Load={result.LoadAfter}, ErrorCount={result.ErrorCountAfter}, Enabled={result.EnabledAfter}");
+                Console.WriteLine(new string('-', 72));
                 await Task.Delay(tickDelayMs);
             }
-
         }
 
-        async Task<string> InferAsync(
-    ChatClient chat,
-    string prompt)
-        {
+        var summary = Summarize(allResults);
+        return (allResults, summary);
+    }
 
-            List<ChatMessage> messages = new List<ChatMessage>
+    private static async Task<string> InferAsync(ChatClient chat, string prompt)
     {
-                new SystemChatMessage("You control a distributed resource. " +
-                "Return only JSON with fields: function and reason."),
+        List<ChatMessage> messages = new()
+        {
+            new SystemChatMessage(
+                "You control a distributed resource. " +
+                "Return raw JSON only with fields: function and reason. " +
+                "Do not wrap the response in markdown or code fences."),
             new UserChatMessage(prompt)
         };
 
         var result = await chat.CompleteChatAsync(messages);
-
         return result.Value.Content[0].Text;
     }
-        private static string BuildPrompt(string typeDefJson, ServiceNode node, int tick)
+
+    private static string BuildPrompt(string typeDefJson, int tick)
     {
         return
 $@"You are given a runtime type definition for a distributed resource and its current state.
 Choose at most one function to call.
 Use only the functions defined in the type definition.
 Do not invent functions.
-If no action is needed, return function as null.
-Return only JSON in this format:
-{{ ""function"": ""Restart|ResetErrors|Enable|Disable|null"", ""reason"": ""short explanation"" }}
+Return ONLY valid JSON in this format:
+{{ ""function"": ""<<name>>"", ""reason"": ""short explanation"" }}
+If the current state is normal and no action is needed, return:
+{{ ""function"": null, ""reason"": ""..."" }}.
 
-Type Definition:
+Input:
 {typeDefJson}";
-
-//Current Tick: {tick}
-//Current State:
-//{{
-//  ""Load"": {node.Load},
-//  ""ErrorCount"": {node.ErrorCount},
-//  ""Enabled"": {(node.Enabled ? "true" : "false")}
-//}}";
     }
 
-        private static LlmDecision? ParseDecision(string text)
-        {
-            try
-            {
-                var json = ExtractJson(text);
+    //private static LlmDecision? ParseDecision(string text)
+    //{
+    //    try
+    //    {
+    //        var json = ExtractJson(text);
 
-                return JsonSerializer.Deserialize<LlmDecision>(
-                    json,
-                    new JsonSerializerOptions
+    //        return JsonSerializer.Deserialize<LlmDecision>(
+    //            json,
+    //            new JsonSerializerOptions
+    //            {
+    //                PropertyNameCaseInsensitive = true
+    //            });
+    //    }
+    //    catch
+    //    {
+    //        return null;
+    //    }
+    //}
+
+    private static (LlmDecision? First, LlmDecision? Final, bool Repaired, int Count) ParseDecisionWithRepair(string text)
+    {
+        var objects = ExtractJsonObjects(text);
+
+        if (objects.Count == 0)
+            return (null, null, false, 0);
+
+        var options = new JsonSerializerOptions
         {
             PropertyNameCaseInsensitive = true
-                    });
-            }
-            catch
-            {
-                return null;
-            }
-        }
+        };
 
-        private static string ExtractJson(string text)
+        LlmDecision? first = null;
+        LlmDecision? final = null;
+
+        try { first = JsonSerializer.Deserialize<LlmDecision>(objects[0], options); } catch { }
+        try { final = JsonSerializer.Deserialize<LlmDecision>(objects[^1], options); } catch { }
+
+        bool repaired = objects.Count > 1 &&
+                        NormalizeFunction(first?.Function) != NormalizeFunction(final?.Function);
+
+        return (first, final, repaired, objects.Count);
+    }
+    private static List<string> ExtractJsonObjects(string text)
     {
+        var results = new List<string>();
+
         if (string.IsNullOrWhiteSpace(text))
-                return "{}";
+            return results;
 
         text = text.Trim();
 
-            if (text.StartsWith("```"))
+        if (text.StartsWith("```", StringComparison.Ordinal))
         {
             var firstNewline = text.IndexOf('\n');
             if (firstNewline >= 0)
@@ -136,13 +270,76 @@ Type Definition:
                 text = text[..lastFence];
         }
 
+        int depth = 0;
+        int start = -1;
+
+        for (int i = 0; i < text.Length; i++)
+        {
+            char c = text[i];
+
+            if (c == '{')
+            {
+                if (depth == 0)
+                    start = i;
+
+                depth++;
+            }
+            else if (c == '}')
+            {
+                if (depth > 0)
+                {
+                    depth--;
+
+                    if (depth == 0 && start >= 0)
+                    {
+                        results.Add(text.Substring(start, i - start + 1));
+                        start = -1;
+                    }
+                }
+            }
+        }
+
+        return results;
+    }
+    private static string ExtractJson(string text)
+    {
+        if (string.IsNullOrWhiteSpace(text))
+            return "{}";
+
+        text = text.Trim();
+
+        if (text.StartsWith("```", StringComparison.Ordinal))
+        {
+            var firstNewline = text.IndexOf('\n');
+            if (firstNewline >= 0)
+                text = text[(firstNewline + 1)..];
+
+            var lastFence = text.LastIndexOf("```", StringComparison.Ordinal);
+            if (lastFence >= 0)
+                text = text[..lastFence];
+        }
+
+        // Fallback: extract first JSON object if extra text exists.
+        int start = text.IndexOf('{');
+        int end = text.LastIndexOf('}');
+        if (start >= 0 && end > start)
+            text = text.Substring(start, end - start + 1);
+
         return text.Trim();
     }
 
+    private static string? NormalizeFunction(string? functionName)
+    {
+        if (string.IsNullOrWhiteSpace(functionName) ||
+            string.Equals(functionName, "null", StringComparison.OrdinalIgnoreCase))
+            return null;
+
+        return functionName.Trim();
+    }
+
     private static bool InvokeIfValid(ServiceNode node, string? functionName)
     {
-            if (string.IsNullOrWhiteSpace(functionName) ||
-                string.Equals(functionName, "null", StringComparison.OrdinalIgnoreCase))
+        if (functionName == null)
             return false;
 
         switch (functionName)
@@ -167,5 +364,47 @@ Type Definition:
                 return false;
         }
     }
+
+    private static List<ModelSummary> Summarize(List<TickResult> results)
+    {
+        return results
+            .GroupBy(r => r.Model)
+            .Select(g =>
+            {
+                var latencies = g.Select(x => x.LatencyMs).OrderBy(x => x).ToList();
+
+                return new ModelSummary
+                {
+                    Model = g.Key,
+                    TotalTicks = g.Count(),
+                    ParseRate = 100.0 * g.Count(x => x.Parsed) / g.Count(),
+                    AllowedRate = 100.0 * g.Count(x => x.Allowed) / g.Count(),
+                    CorrectRate = 100.0 * g.Count(x => x.Correct) / g.Count(),
+                    MeanLatencyMs = g.Average(x => x.LatencyMs),
+                    P95LatencyMs = Percentile(latencies, 0.95),
+                    RepairRate = 100.0 * g.Count(x => x.Repaired) / g.Count(),
+                };
+            })
+            .OrderBy(x => x.Model)
+            .ToList();
+    }
+
+    private static double Percentile(List<double> sortedValues, double p)
+    {
+        if (sortedValues.Count == 0)
+            return 0;
+
+        if (sortedValues.Count == 1)
+            return sortedValues[0];
+
+        double index = (sortedValues.Count - 1) * p;
+        int lower = (int)Math.Floor(index);
+        int upper = (int)Math.Ceiling(index);
+
+        if (lower == upper)
+            return sortedValues[lower];
+
+        double weight = index - lower;
+        return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
     }
 }

Tests/Annotations/ModelConfig.cs

@@ -0,0 +1,15 @@
+using System;
+using System.ClientModel;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Esiur.Tests.Annotations
+{
+    public sealed class ModelConfig
+    {
+        public string Name { get; set; } = "";
+        public string Endpoint { get; set; } = "";
+        public ApiKeyCredential ApiKey { get; set; } = default!;
+        public string ModelName { get; set; } = "";
+    }
+}

Tests/Annotations/ModelSummary.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Esiur.Tests.Annotations
+{
+    public sealed class ModelSummary
+    {
+        public string Model { get; set; } = "";
+        public int TotalTicks { get; set; }
+
+        public double ParseRate { get; set; }
+        public double AllowedRate { get; set; }
+        public double CorrectRate { get; set; }
+
+        public double MeanLatencyMs { get; set; }
+        public double P95LatencyMs { get; set; }
+
+        public double RepairRate { get; set; }
+    }
+}

Tests/Annotations/Program.cs

@@ -7,38 +7,75 @@ using OpenAI.Chat;
 using System.ClientModel;
 using System.Data;
 
-var wh = new Warehouse();
-
-await wh.Put("store", new MemoryStore());
-var node = await wh.Put("store/service", new ServiceNode());
 
 var endpoint = "http://localhost:1234/v1";
 var credential = new ApiKeyCredential("lm-studio");
 
-//var client = new OpenAIClient(credential, new OpenAIClientOptions() { Endpoint = new Uri(endpoint) });
+////var client = new OpenAIClient(credential, new OpenAIClientOptions() { Endpoint = new Uri(endpoint) });
 
-//var chat = client.GetChatClient("microsoft/phi-4");
+////var chat = client.GetChatClient("microsoft/phi-4");
 
-var llmRunner = new LlmRunner();
+//var llmRunner = new LlmRunner();
 
-await llmRunner.RunAsync(
-    node,
-    endpoint,
-    credential,
-    "microsoft/phi-4"
-);
-
-//List<ChatMessage> messages = new List<ChatMessage>
-//{
-//    new SystemChatMessage("You are a helpful assistant that only speaks in rhymes."),
-//    new UserChatMessage("What is the capital of France?")
-//};
-
-//// Send the entire conversation history
-//ChatCompletion completion = chat.CompleteChat(messages);
-
-//var response = await chat.CompleteChatAsync(
-//    "Explain what Pi means"
+//await llmRunner.RunAsync(
+//    node,
+//    endpoint,
+//    credential,
+//    "microsoft/phi-4"
 //);
 
-//Console.WriteLine(response.Value.Content[0].Text);
+var runner = new LlmRunner();
+
+var models = new List<ModelConfig>
+{
+    new()
+    {
+        Name = "Phi-4",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "microsoft/phi-4"
+    },
+    new()
+    {
+        Name = "Qwen2.5-7B",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "qwen2.5-7b-instruct"
+    },
+    new()
+    {
+        Name = "gpt-oss",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "openai/gpt-oss-20b"
+    },
+    new()
+    {
+        Name = "qwen2.5-1.5b-instruct",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "qwen2.5-1.5b-instruct"
+    },
+    new()
+    {
+        Name = "ministral-3-3b",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "mistralai/ministral-3-3b"
+    },
+    new()
+    {
+        Name = "deepseek-r1-0528-qwen3-8b",
+        Endpoint = "http://localhost:1234/v1",
+        ApiKey = new ApiKeyCredential("lm-studio"),
+        ModelName = "deepseek/deepseek-r1-0528-qwen3-8b"
+    }
+};
+
+var (results, summary) = await runner.RunAsync( models.Skip(5).Take(1).ToArray(), 
+    250);
+
+foreach (var item in summary)
+{
+    Console.WriteLine($"{item.Model}: Correct={item.CorrectRate:F1}% Repair={item.RepairRate:F1}% Mean={item.MeanLatencyMs:F1} ms P95={item.P95LatencyMs:F1} ms");
+}

Tests/Annotations/ServiceNode.cs

@@ -9,14 +9,15 @@ namespace Esiur.Tests.Annotations
     [Annotation("usage_rules", @"1.Choose at most one function per tick.
     2. Use only functions defined in the functions list.
     3. Do not invent properties or functions.
-    4. Base the decision only on current property values and annotations.")]
+    4. Base the decision only on current property values and annotations.
+    5. Keep the service enabled as much as possible")]
     [Resource]
     public partial class ServiceNode
     {
         [Annotation("Current service load percentage from 0 to 100. Values above 80 indicate overload.")]
         [Export] int load;
 
-        [Annotation("Number of recent errors detected in the service. Values above 3 indicate instability.")]
+        [Annotation("Number of recent errors detected in the service. Values above 3 indicate instability. A value of 0 means no reset is needed")]
         [Export] int errorCount;
 
         [Annotation("True when the service is enabled and allowed to run. False means the service is disabled.")]
@@ -30,13 +31,13 @@ namespace Esiur.Tests.Annotations
             Enabled = true;
         }
 
-        [Annotation("Clear the error counter when errors were temporary and a restart is not required.")]
+        [Annotation("Clear recent errors only when ErrorCount is greater than 0 and the service is otherwise stable.")]
         [Export] public void ResetErrors()
         {
             ErrorCount = 0;
         }
 
-        [Annotation("Enable the service if it is disabled and should be running.")]
+        [Annotation("Enable the service when Enabled is false.")]
         [Export] public void Enable()
         {
             Enabled = true;

Tests/Annotations/TickExpectation.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Esiur.Tests.Annotations
+{
+    public sealed class TickExpectation
+    {
+        public int Tick { get; set; }
+        public HashSet<string?> AllowedFunctions { get; set; } = new();
+        public string Note { get; set; } = "";
+    }
+}

Tests/Annotations/TickResult.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Esiur.Tests.Annotations
+{
+    public sealed class TickResult
+    {
+        public string Model { get; set; } = "";
+        public int Tick { get; set; }
+
+        public int LoadBefore { get; set; }
+        public int ErrorCountBefore { get; set; }
+        public bool EnabledBefore { get; set; }
+
+        public string RawResponse { get; set; } = "";
+        public string? PredictedFunction { get; set; }
+        public string? Reason { get; set; }
+
+        public bool Parsed { get; set; }
+        public bool Allowed { get; set; }
+        public bool Invoked { get; set; }
+        public bool Correct { get; set; }
+
+        public double LatencyMs { get; set; }
+
+        public int LoadAfter { get; set; }
+        public int ErrorCountAfter { get; set; }
+        public bool EnabledAfter { get; set; }
+
+        public string ExpectedText { get; set; } = "";
+
+
+        public bool Repaired { get; set; }
+        public int JsonObjectCount { get; set; }
+        public string? FirstFunction { get; set; }
+        public string? FinalFunction { get; set; }
+
+         public string? FirstPredictedFunction { get; set; }
+    }
+
+}

Tests/Annotations/TickState.cs

@@ -10,5 +10,4 @@ namespace Esiur.Tests.Annotations
         public int ErrorCount { get; set; }
         public bool Enabled { get; set; }
     }
-
 }