Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions StabilityMatrix.Core/Inference/ComfyClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ private void OnMessageReceived(ResponseMessage message)
}
}

private void HandleTextMessage(string text)
internal void HandleTextMessage(string text)
{
ComfyWebSocketResponse? json;
try
Expand Down Expand Up @@ -254,10 +254,50 @@ is not { } errorData
Logger.Warn($"Could not find task for prompt {errorData.PromptId}, skipping");
}
}
else
else if (
json.Type
is ComfyWebSocketResponseType.ExecutionStart
or ComfyWebSocketResponseType.ExecutionCached
or ComfyWebSocketResponseType.Executed
or ComfyWebSocketResponseType.ExecutionSuccess
or ComfyWebSocketResponseType.ProgressState
)
{
Logger.Warn($"Unknown message type {json.Type} ({json.Data}), skipping");
// Standard ComfyUI messages this client doesn't use
}
else if (json.Type == ComfyWebSocketResponseType.ExecutionInterrupted)
{
// Kept at Debug to help diagnose reports of generations that stopped
Logger.Debug(
"Execution interrupted (PromptId = {PromptId}, NodeId = {NodeId})",
json.Data["prompt_id"]?.ToString(),
json.Data["node_id"]?.ToString()
);
}
else if (Logger.IsTraceEnabled)
{
// Includes custom node broadcasts (e.g. resource monitors) which can arrive every second,
// full message content is already logged at Trace above
Logger.Trace(
"Skipping unhandled message type {Type}",
GetRawMessageType(text) ?? json.Type.ToString()
);
}
}

/// <summary>
/// Returns the "type" string of a websocket message as sent, since types this client
/// doesn't know (such as custom node broadcasts) all deserialize to
/// <see cref="ComfyWebSocketResponseType.Unknown"/>.
/// </summary>
private static string? GetRawMessageType(string text)
{
using var document = JsonDocument.Parse(text);
return
document.RootElement.TryGetProperty("type", out var type)
&& type.ValueKind == JsonValueKind.String
? type.GetString()
: null;
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,13 @@ public enum ComfyWebSocketResponseType

[EnumMember(Value = "executed")]
Executed,

[EnumMember(Value = "execution_success")]
ExecutionSuccess,

[EnumMember(Value = "execution_interrupted")]
ExecutionInterrupted,

[EnumMember(Value = "progress_state")]
ProgressState,
}
108 changes: 108 additions & 0 deletions StabilityMatrix.Tests/Core/ComfyClientMessageLoggingTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
using System.Runtime.CompilerServices;
using System.Text.Json;
using NLog;
using NLog.Config;
using NLog.Targets;
using StabilityMatrix.Core.Inference;
using StabilityMatrix.Core.Models.Api.Comfy;

namespace StabilityMatrix.Tests.Core;

[TestClass]
[DoNotParallelize]
public class ComfyClientMessageLoggingTests
{
private LoggingConfiguration? previousConfiguration;
private MemoryTarget memory = null!;

[TestInitialize]
public void Setup()
{
previousConfiguration = LogManager.Configuration;

memory = new MemoryTarget("memory") { Layout = "${level}|${message}" };
var config = new LoggingConfiguration();
config.AddRule(LogLevel.Trace, LogLevel.Fatal, memory, typeof(ComfyClient).FullName!);
LogManager.Configuration = config;
}

[TestCleanup]
public void Cleanup()
{
LogManager.Configuration = previousConfiguration;
}

private static ComfyClient CreateClient() =>
(ComfyClient)RuntimeHelpers.GetUninitializedObject(typeof(ComfyClient));

private string[] LogsAtOrAbove(LogLevel level) =>
memory.Logs.Where(l => LogLevel.FromString(l.Split('|')[0]) >= level).ToArray();

[DataTestMethod]
[DataRow("""{"type":"execution_start","data":{"prompt_id":"p1","timestamp":1}}""")]
[DataRow("""{"type":"execution_cached","data":{"nodes":["1","2"],"prompt_id":"p1"}}""")]
[DataRow("""{"type":"executed","data":{"node":"9","output":{"images":[]},"prompt_id":"p1"}}""")]
[DataRow("""{"type":"execution_success","data":{"prompt_id":"p1","timestamp":2}}""")]
[DataRow(
"""{"type":"progress_state","data":{"prompt_id":"p1","nodes":{"3":{"value":1,"max":20,"state":"running"}}}}"""
)]
[DataRow("""{"type":"crystools.monitor","data":{"cpu_utilization":12.5}}""")]
public void HandleTextMessage_UnhandledTypes_DoNotLogAboveTrace(string message)
{
CreateClient().HandleTextMessage(message);
LogManager.Flush();

CollectionAssert.AreEqual(Array.Empty<string>(), LogsAtOrAbove(LogLevel.Debug));
}

[TestMethod]
public void HandleTextMessage_CustomNodeType_LogsTypeWithoutPayloadAtTrace()
{
CreateClient().HandleTextMessage("""{"type":"crystools.monitor","data":{"cpu_utilization":12.5}}""");
LogManager.Flush();

var skipped = memory.Logs.Single(l => l.Contains("Skipping unhandled message type"));
StringAssert.StartsWith(skipped, "Trace|");
StringAssert.Contains(skipped, "crystools.monitor");
Assert.IsFalse(skipped.Contains("cpu_utilization"));
}

[TestMethod]
public void HandleTextMessage_ExecutionInterrupted_LogsAtDebug()
{
CreateClient()
.HandleTextMessage(
"""{"type":"execution_interrupted","data":{"prompt_id":"p1","node_id":"3","node_type":"KSampler"}}"""
);
LogManager.Flush();

var logs = LogsAtOrAbove(LogLevel.Debug);
Assert.AreEqual(1, logs.Length);
StringAssert.StartsWith(logs[0], "Debug|");
StringAssert.Contains(logs[0], "p1");
StringAssert.Contains(logs[0], "3");
}

[TestMethod]
public void HandleTextMessage_InvalidJson_StillWarns()
{
CreateClient().HandleTextMessage("not json");
LogManager.Flush();

Assert.AreEqual(1, LogsAtOrAbove(LogLevel.Warn).Length);
}

[DataTestMethod]
[DataRow("execution_success", ComfyWebSocketResponseType.ExecutionSuccess)]
[DataRow("execution_interrupted", ComfyWebSocketResponseType.ExecutionInterrupted)]
[DataRow("progress_state", ComfyWebSocketResponseType.ProgressState)]
[DataRow("crystools.monitor", ComfyWebSocketResponseType.Unknown)]
public void ResponseType_DeserializesNewTypes(string type, ComfyWebSocketResponseType expected)
{
var response = JsonSerializer.Deserialize<ComfyWebSocketResponse>(
$$$"""{"type":"{{{type}}}","data":{}}"""
);

Assert.AreEqual(expected, response!.Type);
}
}
Loading