diff --git a/StabilityMatrix.Avalonia/Languages/Resources.Designer.cs b/StabilityMatrix.Avalonia/Languages/Resources.Designer.cs index 44bca5b52..7d430b86c 100644 --- a/StabilityMatrix.Avalonia/Languages/Resources.Designer.cs +++ b/StabilityMatrix.Avalonia/Languages/Resources.Designer.cs @@ -5167,5 +5167,32 @@ public static string Error_DocumentationPageLoadFailed { return ResourceManager.GetString("Error_DocumentationPageLoadFailed", resourceCulture); } } + + /// + /// Looks up a localized string similar to Generated with. + /// + public static string Label_GeneratedWith { + get { + return ResourceManager.GetString("Label_GeneratedWith", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No generation data detected. + /// + public static string Text_NoGenerationDataDetected { + get { + return ResourceManager.GetString("Text_NoGenerationDataDetected", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} generation data was found, but its prompt and settings can't be loaded. + /// + public static string Text_GenerationDataNotLoadable { + get { + return ResourceManager.GetString("Text_GenerationDataNotLoadable", resourceCulture); + } + } } } diff --git a/StabilityMatrix.Avalonia/Languages/Resources.resx b/StabilityMatrix.Avalonia/Languages/Resources.resx index ede9d4b94..e9375ca8c 100644 --- a/StabilityMatrix.Avalonia/Languages/Resources.resx +++ b/StabilityMatrix.Avalonia/Languages/Resources.resx @@ -1883,4 +1883,13 @@ Account tokens will not be viewable after saving, please make a note of them if Could not load this page. Check your connection and try again. + + Generated with + + + No generation data detected + + + {0} generation data was found, but its prompt and settings can't be loaded + diff --git a/StabilityMatrix.Avalonia/ViewModels/Base/InferenceTabViewModelBase.cs b/StabilityMatrix.Avalonia/ViewModels/Base/InferenceTabViewModelBase.cs index e4e8a933a..d54d866d3 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Base/InferenceTabViewModelBase.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Base/InferenceTabViewModelBase.cs @@ -19,6 +19,7 @@ using NLog; using StabilityMatrix.Avalonia.Animations; using StabilityMatrix.Avalonia.Extensions; +using StabilityMatrix.Avalonia.Languages; using StabilityMatrix.Avalonia.Models; using StabilityMatrix.Avalonia.Models.Inference; using StabilityMatrix.Avalonia.Services; @@ -27,6 +28,7 @@ using StabilityMatrix.Core.Models; using StabilityMatrix.Core.Models.Database; using StabilityMatrix.Core.Models.FileInterfaces; +using StabilityMatrix.Core.Models.GenerationData; #pragma warning disable CS0657 // Not a valid attribute location for this declaration @@ -317,7 +319,43 @@ private void LoadImageMetadata( return; } - throw new ApplicationException("File does not contain SMProject or Parameters Metadata"); + var generationData = GenerationDataParser.TryRead(imageFilePath); + if (generationData?.Parameters is { PositivePrompt: not null } generationParameters) + { + Logger.Info("Loading Parameters from {Source} metadata", generationData.DisplayName); + + if (this is IParametersLoadableState paramsLoadableVm) + { + Dispatcher.UIThread.Invoke(() => + paramsLoadableVm.LoadStateFromParameters(generationParameters) + ); + } + else + { + Logger.Warn( + "Load parameters target {Type} does not implement IParametersLoadableState, skipping", + GetType().Name + ); + } + + if (this is IImageGalleryComponent imageGalleryComponent) + { + Dispatcher.UIThread.Invoke(() => + imageGalleryComponent.LoadImagesToGallery(new ImageSource(imageFilePath)) + ); + } + + return; + } + + if (generationData is not null) + { + throw new ApplicationException( + string.Format(Resources.Text_GenerationDataNotLoadable, generationData.DisplayName) + ); + } + + throw new ApplicationException(Resources.Text_NoGenerationDataDetected); } /// diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/ImageViewerViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/ImageViewerViewModel.cs index 10d5b73de..bccbecbd8 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/ImageViewerViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/ImageViewerViewModel.cs @@ -11,6 +11,7 @@ using StabilityMatrix.Avalonia.Controls; using StabilityMatrix.Avalonia.Extensions; using StabilityMatrix.Avalonia.Helpers; +using StabilityMatrix.Avalonia.Languages; using StabilityMatrix.Avalonia.Models; using StabilityMatrix.Avalonia.ViewModels.Base; using StabilityMatrix.Avalonia.Views; @@ -39,6 +40,8 @@ ISettingsManager settingsManager [ObservableProperty] [NotifyPropertyChangedFor(nameof(HasLocalGenerationParameters))] + [NotifyPropertyChangedFor(nameof(HasLocalImageFile))] + [NotifyPropertyChangedFor(nameof(LocalGenerationDataText))] private LocalImageFile? localImageFile; [ObservableProperty] @@ -62,6 +65,19 @@ ISettingsManager settingsManager /// public bool HasLocalGenerationParameters => LocalImageFile?.GenerationParameters is not null; + /// + /// Whether a local file is shown, which always gets the info button to tell where it came from. + /// + public bool HasLocalImageFile => LocalImageFile is not null; + + /// + /// The tool or format of the local file's generation data, or that none was found. + /// + public string? LocalGenerationDataText => + LocalImageFile is null + ? null + : LocalImageFile.GenerationDataLabel ?? Resources.Text_NoGenerationDataDetected; + /// /// Whether Civitai image metadata is available. /// diff --git a/StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml b/StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml index d9e3f66d7..dda603fda 100644 --- a/StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml +++ b/StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml @@ -128,10 +128,11 @@ VerticalAlignment="Top" icons:Attached.Icon="fa-solid fa-info" Classes="transparent-full" - Tapped="InfoButton_OnTapped"> + Tapped="InfoButton_OnTapped" + ToolTip.Tip="{Binding LocalGenerationDataText}"> - + @@ -326,43 +327,52 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/StabilityMatrix.Core/Helper/ImageMetadata.cs b/StabilityMatrix.Core/Helper/ImageMetadata.cs index 088d709fd..547b24e14 100644 --- a/StabilityMatrix.Core/Helper/ImageMetadata.cs +++ b/StabilityMatrix.Core/Helper/ImageMetadata.cs @@ -98,6 +98,11 @@ public static ( { var file = ImageFile.FromFile(filePath.Info.FullName); var userComment = file.Properties.Get(ExifTag.UserComment); + if (userComment is null) + { + return (null, null, null, null, null); + } + var bytes = userComment.Interoperability.Data.Skip(8).ToArray(); var userCommentString = Encoding.BigEndianUnicode.GetString(bytes); diff --git a/StabilityMatrix.Core/Models/Database/LocalImageFile.cs b/StabilityMatrix.Core/Models/Database/LocalImageFile.cs index b5bd70cf5..f361e625a 100644 --- a/StabilityMatrix.Core/Models/Database/LocalImageFile.cs +++ b/StabilityMatrix.Core/Models/Database/LocalImageFile.cs @@ -4,6 +4,7 @@ using SkiaSharp; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.FileInterfaces; +using StabilityMatrix.Core.Models.GenerationData; using JsonSerializer = System.Text.Json.JsonSerializer; using Size = System.Drawing.Size; @@ -36,6 +37,21 @@ public record LocalImageFile /// public GenerationParameters? GenerationParameters { get; init; } + /// + /// Format of the generation data embedded in the file, if any was found. + /// + public GenerationDataSource? GenerationDataSource { get; init; } + + /// + /// The tool that made the file, when it can be told apart from the format of its generation data. + /// + public string? GenerationSoftware { get; init; } + + /// + /// Name of the tool or format the generation data came from, or null when none was found. + /// + public string? GenerationDataLabel => GenerationSoftware ?? GenerationDataSource?.GetDisplayName(); + /// /// Dimensions of the image /// @@ -99,6 +115,23 @@ public record LocalImageFile } public static LocalImageFile FromPath(FilePath filePath) + { + var imageFile = FromPathWithoutGenerationData(filePath); + var generationData = GenerationDataParser.TryRead(filePath); + if (generationData is null) + { + return imageFile; + } + + return imageFile with + { + GenerationDataSource = generationData.Source, + GenerationSoftware = generationData.Software, + GenerationParameters = imageFile.GenerationParameters ?? generationData.Parameters, + }; + } + + private static LocalImageFile FromPathWithoutGenerationData(FilePath filePath) { // TODO: Support other types const LocalImageFileType imageType = LocalImageFileType.Inference | LocalImageFileType.TextToImage; diff --git a/StabilityMatrix.Core/Models/GenerationData/ComfyPromptReader.cs b/StabilityMatrix.Core/Models/GenerationData/ComfyPromptReader.cs new file mode 100644 index 000000000..07266f1ec --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/ComfyPromptReader.cs @@ -0,0 +1,267 @@ +using System.Text.Json.Nodes; + +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Reads the prompts and sampler settings of a ComfyUI prompt (the API format node graph). +/// +internal class ComfyPromptReader +{ + private static readonly string[] SamplerParams = ["sampler_name", "steps", "cfg"]; + private static readonly string[] SamplerTypes = ["WanVideoSampler"]; + + /// + /// Samplers that take their settings from separate guider, sampler, sigmas and noise nodes + /// + private static readonly string[] CustomSamplerTypes = ["SamplerCustomAdvanced", "SamplerCustom"]; + private static readonly string[] CustomSamplerInputs = ["guider", "sampler", "sigmas", "noise"]; + + /// + /// Text inputs of prompt nodes: CLIPTextEncode and the text, string and prompt nodes of popular node packs + /// + private static readonly string[] TextKeys = + [ + "text", + "Text", + "prompt", + "text_g", + "text_l", + "value", + "string", + ]; + private static readonly string[] PositiveKeys = [.. TextKeys, "positive"]; + private static readonly string[] NegativeKeys = [.. TextKeys, "negative"]; + + private static readonly string[] ValueKeys = + [ + "value", + "seed", + "noise_seed", + "int", + "float", + "number", + "Number", + ]; + + private static readonly string[] ModelKeys = ["ckpt_name", "unet_name", "model"]; + + private readonly Dictionary nodes = new(StringComparer.Ordinal); + + private ComfyPromptReader(JsonObject prompt) + { + foreach (var (id, node) in prompt) + { + if (node is JsonObject obj && obj["class_type"] is not null && obj["inputs"] is JsonObject) + nodes[id] = obj; + } + } + + public static bool IsPrompt(JsonObject? prompt) => + prompt is not null + && prompt.Any(n => + n.Value is JsonObject node && node["class_type"] is not null && node["inputs"] is JsonObject + ); + + public static GenerationParameters? Read(JsonObject prompt) + { + var reader = new ComfyPromptReader(prompt); + GenerationParameters? first = null; + + foreach (var (_, node) in reader.nodes) + { + if (reader.TryReadSampler(node) is not { } parameters) + continue; + first ??= parameters; + if (!string.IsNullOrEmpty(parameters.PositivePrompt)) + return parameters; + } + + return first; + } + + private GenerationParameters? TryReadSampler(JsonObject node) + { + var classType = JsonValueText(node["class_type"]); + if (node["inputs"] is not JsonObject nodeInputs) + return null; + + var inputs = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in nodeInputs) + inputs[key] = value; + + var isCustomSampler = CustomSamplerTypes.Contains(classType); + if (isCustomSampler) + { + foreach (var name in CustomSamplerInputs) + { + if (LinkTarget(inputs.GetValueOrDefault(name)) is not { } targetId) + continue; + if ( + !nodes.TryGetValue(targetId, out var target) + || target["inputs"] is not JsonObject targetInputs + ) + continue; + inputs.Remove(name); + foreach (var (key, value) in targetInputs) + inputs.TryAdd(key, value); + } + if (inputs.TryGetValue("noise_seed", out var noiseSeed)) + inputs.TryAdd("seed", noiseSeed); + } + else if (!SamplerTypes.Contains(classType) && !SamplerParams.All(inputs.ContainsKey)) + { + return null; + } + + var parameters = new GenerationParameters + { + Sampler = JsonValueText( + inputs.GetValueOrDefault("sampler_name") ?? inputs.GetValueOrDefault("scheduler") + ), + Steps = (int)(GenerationDataJson.GetDouble(ResolveValue(inputs, "steps")) ?? 0), + CfgScale = GenerationDataJson.GetDouble(ResolveValue(inputs, "cfg")) ?? 0, + Seed = GenerationDataJson.GetSeed( + ResolveValue(inputs, "seed") ?? ResolveValue(inputs, "noise_seed") + ), + }; + + if (LinkTarget(inputs.GetValueOrDefault("model")) is { } modelId) + parameters.ModelName = FindModel(modelId); + + // WanVideoSampler takes its prompts from a text embeds node + var promptInputs = inputs; + if ( + LinkTarget(inputs.GetValueOrDefault("text_embeds")) is { } embedsId + && nodes.TryGetValue(embedsId, out var embeds) + && embeds["inputs"] is JsonObject embedsInputs + ) + { + promptInputs = embedsInputs.ToDictionary(p => p.Key, p => p.Value, StringComparer.Ordinal); + } + + // BasicGuider takes a single conditioning input + var positiveId = LinkTarget( + promptInputs.GetValueOrDefault("positive") ?? promptInputs.GetValueOrDefault("conditioning") + ); + if (positiveId is not null) + parameters.PositivePrompt = JoinPrompts(FindPrompts(positiveId, PositiveKeys, "negative")); + + if (LinkTarget(promptInputs.GetValueOrDefault("negative")) is { } negativeId) + parameters.NegativePrompt = JoinPrompts(FindPrompts(negativeId, NegativeKeys, "positive")); + + return parameters; + } + + /// + /// Get an input value, following a link to a primitive or seed node when the value is linked + /// + private JsonNode? ResolveValue(Dictionary inputs, string key) + { + var value = inputs.GetValueOrDefault(key); + for (var depth = 0; depth < 8 && LinkTarget(value) is { } targetId; depth++) + { + if ( + !nodes.TryGetValue(targetId, out var target) + || target["inputs"] is not JsonObject targetInputs + ) + return null; + value = + targetInputs[key] + ?? ValueKeys.Select(k => targetInputs[k]).FirstOrDefault(v => v is not null); + } + return value is JsonValue ? value : null; + } + + private string? FindModel(string startId) + { + foreach (var (_, node) in Traverse(startId, _ => true)) + { + if (node["inputs"] is not JsonObject inputs) + continue; + foreach (var key in ModelKeys) + { + // "model" is also the name of the link input of LoRA loaders and patches: only names count + if (inputs[key] is JsonValue value && value.TryGetValue(out var name)) + return name; + } + } + return null; + } + + /// + /// Find the prompts reached from a conditioning input. Nodes that take both conditionings + /// (WanImageToVideo, ControlNet apply nodes) are only followed through the matching input. + /// + private List FindPrompts(string startId, string[] textKeys, string otherConditioning) + { + var prompts = new List(); + foreach ( + var _ in Traverse(startId, inputs => HasPrompt(inputs, textKeys, prompts), otherConditioning) + ) { } + return prompts; + } + + /// + /// Collects the prompts of a node; returns whether to keep looking in the node's inputs. + /// + private static bool HasPrompt(JsonObject inputs, string[] textKeys, List prompts) + { + var found = false; + foreach (var key in textKeys) + { + if (inputs[key] is JsonValue value && value.TryGetValue(out var text)) + { + found = true; + if (!string.IsNullOrWhiteSpace(text)) + prompts.Add(text.Trim()); + } + } + return !found; + } + + private IEnumerable<(string Id, JsonObject Node)> Traverse( + string startId, + Func descend, + string? ignoredInput = null + ) + { + var visited = new HashSet(StringComparer.Ordinal); + var stack = new Stack(); + stack.Push(startId); + + while (stack.Count > 0) + { + var id = stack.Pop(); + if (!visited.Add(id) || !nodes.TryGetValue(id, out var node)) + continue; + + yield return (id, node); + + if (node["inputs"] is not JsonObject inputs || !descend(inputs)) + continue; + + var targets = new List(); + foreach (var (name, value) in inputs) + { + if (name == "clip" || name == ignoredInput) + continue; + if ( + value is JsonArray { Count: 2 } + && LinkTarget(value) is { } target + && nodes.ContainsKey(target) + ) + targets.Add(target); + } + for (var i = targets.Count - 1; i >= 0; i--) + stack.Push(targets[i]); + } + } + + private static string? LinkTarget(JsonNode? input) => + input is JsonArray { Count: > 0 } array ? JsonValueText(array[0]) : null; + + private static string? JsonValueText(JsonNode? node) => GenerationDataJson.GetString(node); + + private static string? JoinPrompts(List prompts) => + prompts.Count == 0 ? null : string.Join(", ", prompts.Distinct()); +} diff --git a/StabilityMatrix.Core/Models/GenerationData/GenerationDataInfo.cs b/StabilityMatrix.Core/Models/GenerationData/GenerationDataInfo.cs new file mode 100644 index 000000000..56c7d6d75 --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/GenerationDataInfo.cs @@ -0,0 +1,21 @@ +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Generation data found in an image or video file. +/// +public record GenerationDataInfo +{ + public required GenerationDataSource Source { get; init; } + + /// + /// The tool that made the file when it can be told apart from the format, + /// e.g. "Forge Neo" or "SD.Next" for files in the AUTOMATIC1111 format. + /// + public string? Software { get; init; } + + public GenerationParameters? Parameters { get; init; } + + public string DisplayName => Software ?? Source.GetDisplayName(); + + public bool IsProvenanceOnly => Source is GenerationDataSource.C2pa or GenerationDataSource.Iptc; +} diff --git a/StabilityMatrix.Core/Models/GenerationData/GenerationDataJson.cs b/StabilityMatrix.Core/Models/GenerationData/GenerationDataJson.cs new file mode 100644 index 000000000..05f25d36c --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/GenerationDataJson.cs @@ -0,0 +1,113 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +namespace StabilityMatrix.Core.Models.GenerationData; + +internal static partial class GenerationDataJson +{ + private static readonly JsonDocumentOptions DocumentOptions = new() + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + }; + + /// + /// Parse a JSON object, also accepting the NaN and Infinity values Python writes, + /// and the trailing null characters some tools add. Returns null for anything else. + /// + public static JsonObject? ParseObject(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + return null; + text = text.TrimEnd('\0', ' ', '\n', '\r', '\t'); + if (!text.TrimStart().StartsWith('{')) + return null; + + try + { + return JsonNode.Parse(text, documentOptions: DocumentOptions) as JsonObject; + } + catch (JsonException) + { + try + { + return JsonNode.Parse( + PythonConstantRegex().Replace(text, "null"), + documentOptions: DocumentOptions + ) as JsonObject; + } + catch (JsonException) + { + return null; + } + } + } + + public static JsonObject? AsObject(JsonNode? node) => + node switch + { + JsonObject obj => obj, + JsonValue value when value.TryGetValue(out var text) => ParseObject(text), + _ => null, + }; + + public static string? GetString(JsonNode? node) + { + if (node is not JsonValue value) + return null; + if (value.TryGetValue(out var text)) + return text; + return value.GetValueKind() is JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False + ? value.ToJsonString() + : null; + } + + public static string? AsText(JsonNode? node) => + node switch + { + null => null, + JsonValue value when value.TryGetValue(out var text) => text, + _ => node.ToJsonString(), + }; + + public static double? GetDouble(JsonNode? node) + { + if (node is not JsonValue value) + return null; + if ( + value.GetValueKind() == JsonValueKind.Number + && double.TryParse( + value.ToJsonString(), + NumberStyles.Float, + CultureInfo.InvariantCulture, + out var n + ) + ) + return n; + if ( + value.TryGetValue(out var text) + && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var number) + ) + return number; + return null; + } + + public static ulong GetSeed(JsonNode? node) + { + if (node is not JsonValue value) + return 0; + if (value.TryGetValue(out var seed)) + return seed; + var text = value.GetValueKind() == JsonValueKind.Number ? value.ToJsonString() : GetString(value); + if (ulong.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out seed)) + return seed; + return GetDouble(value) is { } number and >= 0 and < 1.8e19 ? (ulong)number : 0; + } + + public static JsonNode? Get(JsonNode? node, string key) => node is JsonObject obj ? obj[key] : null; + + [GeneratedRegex(@"(?<=[:\[,]\s*)-?(?:NaN|Infinity)(?=\s*[,\]}])")] + private static partial Regex PythonConstantRegex(); +} diff --git a/StabilityMatrix.Core/Models/GenerationData/GenerationDataParser.cs b/StabilityMatrix.Core/Models/GenerationData/GenerationDataParser.cs new file mode 100644 index 000000000..3fb7ea5a7 --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/GenerationDataParser.cs @@ -0,0 +1,848 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using NLog; + +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Recognizes the generation data that image generators embed in their images and videos, +/// and reads the prompt and settings where the format allows it. +/// +public static partial class GenerationDataParser +{ + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + + private delegate GenerationDataInfo? Parser(MetadataEntries entries); + + /// + /// Parsers in order of precedence: specific formats first, since many tools also write + /// AUTOMATIC1111 parameters or a ComfyUI node graph next to their own data. + /// + private static readonly Parser[] Parsers = + [ + ParseSwarmUI, + ParseRuinedFooocus, + ParseHuggingFaceSpace, + ParseEasyDiffusion, + ParseDrawThings, + ParseDezgo, + ParseKusArt, + ParseTensorArt, + ParseStabilityMatrix, + ParseCivitai, + ParseFooocus, + ParseAutomatic1111, + ParseComfyUI, + ParseInvokeAI, + ParseNovelAI, + ]; + + private static readonly HashSet AiSourceTypes = + [ + "trainedAlgorithmicMedia", + "compositeWithTrainedAlgorithmicMedia", + "compositeSynthetic", + ]; + + public static GenerationDataInfo? TryRead(string path) + { + try + { + return Parse(MetadataEntries.Read(path)); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + Logger.Debug(e, "Failed to read generation data of {Path}", path); + return null; + } + catch (Exception e) + { + Logger.Warn(e, "Failed to parse generation data of {Path}", path); + return null; + } + } + + public static GenerationDataInfo? Parse(MetadataEntries entries) + { + if (entries.IsEmpty) + return null; + + // JPEG and WEBP files of AUTOMATIC1111 and Fooocus keep their parameters in the UserComment + if ( + entries.Container is MetadataContainer.Jpeg or MetadataContainer.Webp + && entries["UserComment"] is { } userComment + ) + { + var userCommentEntries = MetadataEntries.FromDictionary( + [new KeyValuePair("parameters", userComment)], + entries.Container + ); + var info = ParseFooocus(userCommentEntries) ?? ParseAutomatic1111(userCommentEntries); + if (info is not null) + return WithSoftware(info, entries); + } + + foreach (var parser in Parsers) + { + if (parser(entries) is { } info) + return WithSoftware(info, entries); + } + + return ParseProvenance(entries); + } + + #region Software + + private static GenerationDataInfo WithSoftware(GenerationDataInfo info, MetadataEntries entries) + { + if (info.Software is not null) + return info; + + string? software = null; + if (entries["smproj"] is not null || entries["parameters-json"] is not null) + software = "Stability Matrix"; + else if (entries["generation_data"] is not null) + software = "TensorArt"; + else if ( + (entries["Software"]?.StartsWith("Fooocus") ?? false) + || entries["fooocus_scheme"] is not null + || entries["MakerNote"] == "a1111" + ) + software = "Fooocus"; + else if (info.Source == GenerationDataSource.Automatic1111) + { + software = DetectA1111Software(entries["parameters"] ?? entries["UserComment"] ?? string.Empty); + // ComfyUI save nodes that write AUTOMATIC1111 parameters next to the node graph + if (software is null && (entries["prompt"]?.Contains("\"class_type\"") ?? false)) + software = "ComfyUI"; + } + + return software is null || software == info.Source.GetDisplayName() + ? info + : info with + { + Software = software, + }; + } + + /// + /// Name the tool behind an AUTOMATIC1111 style parameter string, from its Version and App fields + /// + public static string? DetectA1111Software(string parameters) + { + if (SdNextAppRegex().IsMatch(parameters)) + return "SD.Next"; + if (parameters.Contains("Civitai resources:") || parameters.Contains("Civitai metadata:")) + return "Civitai"; + + // a standalone "Version:" field, not one with a prefix like "ADetailer version:" + var match = A1111VersionRegex().Match(parameters); + if (!match.Success) + return null; + var version = match.Groups[1].Value; + + if (Regex.IsMatch(version, @"^v\d")) + return "Stable Diffusion WebUI"; + if (version == "classic") + return "Forge Classic"; + if (version.StartsWith("neo")) + return "Forge Neo"; + if (Regex.IsMatch(version, @"^f\d+\.\d+(?:\.\d+)?-v")) + return "EasyReforge"; + if (Regex.IsMatch(version, @"^f\d+\.\d+(?:\.\d+)?v\d+-v")) + return "reForge"; + if (Regex.IsMatch(version, @"^f\d")) + return "Forge"; + if (version.StartsWith("Fooocus")) + return "Fooocus"; + if (version == "ComfyUI") + return "ComfyUI"; + return null; + } + + #endregion + + #region Parsers + + private static GenerationDataInfo Result(GenerationDataSource source, GenerationParameters? parameters) => + new() { Source = source, Parameters = parameters }; + + private static GenerationParameters FromJson( + JsonObject json, + string prompt, + string negativePrompt, + string? model, + string seed, + string steps, + string cfgScale, + string? sampler + ) + { + return new GenerationParameters + { + PositivePrompt = TrimToNull(GenerationDataJson.GetString(json[prompt])), + NegativePrompt = TrimToNull(GenerationDataJson.GetString(json[negativePrompt])), + ModelName = model is null ? null : GenerationDataJson.GetString(json[model]), + Seed = GenerationDataJson.GetSeed(json[seed]), + Steps = (int)(GenerationDataJson.GetDouble(json[steps]) ?? 0), + CfgScale = GenerationDataJson.GetDouble(json[cfgScale]) ?? 0, + Sampler = sampler is null ? null : GenerationDataJson.GetString(json[sampler]), + Width = (int)(GenerationDataJson.GetDouble(json["width"]) ?? 0), + Height = (int)(GenerationDataJson.GetDouble(json["height"]) ?? 0), + }; + } + + private static GenerationDataInfo? ParseSwarmUI(MetadataEntries entries) + { + var source = new[] { entries["parameters"], entries["UserComment"] }.FirstOrDefault(v => + v?.Contains("sui_image_params") ?? false + ); + if (GenerationDataJson.ParseObject(source)?["sui_image_params"] is not JsonObject json) + return null; + + return Result( + GenerationDataSource.SwarmUI, + FromJson(json, "prompt", "negativeprompt", "model", "seed", "steps", "cfgscale", "sampler") + ); + } + + private static GenerationDataInfo? ParseRuinedFooocus(MetadataEntries entries) + { + var source = entries["parameters"] ?? entries["UserComment"]; + if (source is null || !source.Contains("RuinedFooocus")) + return null; + if (GenerationDataJson.ParseObject(source) is not { } json) + return null; + if (GenerationDataJson.GetString(json["software"]) != "RuinedFooocus") + return null; + + return Result( + GenerationDataSource.RuinedFooocus, + FromJson(json, "Prompt", "Negative", "base_model_name", "seed", "steps", "cfg", "sampler_name") + ); + } + + private static GenerationDataInfo? ParseHuggingFaceSpace(MetadataEntries entries) + { + var json = GenerationDataJson.ParseObject(entries["parameters"] ?? entries["UserComment"]); + if ( + json is null + || !json.ContainsKey("resolution") + || !json.ContainsKey("Model") + || !json.ContainsKey("prompt") + ) + return null; + + var parameters = FromJson( + json, + "prompt", + "negative_prompt", + "Model", + "seed", + "num_inference_steps", + "guidance_scale", + "sampler" + ); + parameters.ModelHash = GenerationDataJson.GetString(json["Model hash"]); + return Result(GenerationDataSource.HuggingFaceSpace, parameters); + } + + private static GenerationDataInfo? ParseEasyDiffusion(MetadataEntries entries) + { + const string marker = "use_stable_diffusion_model"; + + JsonObject? json; + if (entries[marker] is not null) + { + // PNG files hold one text chunk per setting, as Python literals + json = new JsonObject(); + foreach (var (key, value) in entries.Entries) + json[key] = value; + } + else + { + var source = new[] { entries["UserComment"], entries["parameters"] }.FirstOrDefault(v => + v?.Contains(marker) ?? false + ); + json = GenerationDataJson.ParseObject(source); + } + if (json is null || !json.ContainsKey(marker)) + return null; + + var parameters = FromJson( + json, + "prompt", + "negative_prompt", + marker, + "seed", + "num_inference_steps", + "guidance_scale", + "sampler_name" + ); + if (parameters.NegativePrompt == "None") + parameters.NegativePrompt = null; + return Result(GenerationDataSource.EasyDiffusion, parameters); + } + + private static GenerationDataInfo? ParseDrawThings(MetadataEntries entries) + { + if (!(entries["CreatorTool"]?.StartsWith("Draw Things") ?? false)) + return null; + var json = GenerationDataJson.ParseObject(entries["UserComment"]); + if (json is null || (!json.ContainsKey("c") && !json.ContainsKey("sampler"))) + return null; + + return Result( + GenerationDataSource.DrawThings, + FromJson(json, "c", "uc", "model", "seed", "steps", "scale", "sampler") + ); + } + + private static GenerationDataInfo? ParseDezgo(MetadataEntries entries) + { + JsonObject? job = null; + if (entries["GenInfo"] is { } genInfo) + { + job = GenerationDataJson.ParseObject(genInfo); + } + else if (entries["UserComment"] is { } userComment && DezgoJobRegex().IsMatch(userComment)) + { + job = SimpleYaml.Parse(userComment); + } + if (job?["Input"] is not JsonObject input || job["JobId"] is null) + return null; + + return Result( + GenerationDataSource.Dezgo, + FromJson(input, "prompt", "negative_prompt", "model", "seed", "steps", "guidance", "sampler") + ); + } + + private static GenerationDataInfo? ParseKusArt(MetadataEntries entries) + { + if (entries["Copyright"] is not { } copyright || !KusArtRegex().IsMatch(copyright)) + return null; + if (GenerationDataJson.ParseObject(entries["extra"]) is not { } json) + return null; + + return Result( + GenerationDataSource.KusArt, + new GenerationParameters + { + PositivePrompt = TrimToNull(GenerationDataJson.GetString(json["prompt"])), + NegativePrompt = TrimToNull(GenerationDataJson.GetString(json["negative_prompt"])), + } + ); + } + + private static GenerationDataInfo? ParseTensorArt(MetadataEntries entries) + { + if (GenerationDataJson.ParseObject(entries["generation_data"]) is not { } json) + return null; + + var parameters = FromJson( + json, + "prompt", + "negativePrompt", + null, + "seed", + "steps", + "cfgScale", + json.ContainsKey("ksamplerName") ? "ksamplerName" : "samplerName" + ); + if (json["baseModel"] is JsonObject baseModel) + { + parameters.ModelName = GenerationDataJson.GetString(baseModel["modelFileName"]); + parameters.ModelHash = GenerationDataJson.GetString(baseModel["hash"]); + } + + // -1 is written for random seeds, the node graph has the real one + if ( + GenerationDataJson.GetDouble(json["seed"]) is -1 + && GenerationDataJson.ParseObject(entries["prompt"]) is { } graph + ) + { + foreach (var (_, node) in graph) + { + var inputs = GenerationDataJson.Get(node, "inputs"); + var seed = + GenerationDataJson.Get(inputs, "seed") ?? GenerationDataJson.Get(inputs, "noise_seed"); + if (seed is JsonValue value && value.GetValueKind() == JsonValueKind.Number) + { + parameters.Seed = GenerationDataJson.GetSeed(seed); + break; + } + } + } + + return Result(GenerationDataSource.TensorArt, parameters); + } + + private static GenerationDataInfo? ParseStabilityMatrix(MetadataEntries entries) + { + var json = entries["parameters-json"]; + // WEBP files written by Stability Matrix keep the parameters in the ImageDescription + if ( + json is null + && entries["smproj"] is null + && entries.Container == MetadataContainer.Webp + && entries["ImageDescription"] is { } description + && description.Contains("\"PositivePrompt\"") + ) + { + json = description; + } + if (json is null) + return null; + + GenerationParameters? parameters; + try + { + parameters = JsonSerializer.Deserialize(json); + } + catch (JsonException) + { + return null; + } + + return parameters is null ? null : Result(GenerationDataSource.StabilityMatrix, parameters); + } + + private static GenerationDataInfo? ParseCivitai(MetadataEntries entries) + { + var source = new[] + { + entries["UserComment"], + entries["prompt"], + entries["parameters"], + }.FirstOrDefault(v => v?.Contains("\"extraMetadata\"") ?? false); + if (GenerationDataJson.ParseObject(source) is not { } graph) + return null; + if (GenerationDataJson.AsObject(graph["extraMetadata"]) is not { } json) + return null; + + var parameters = FromJson( + json, + "prompt", + "negativePrompt", + null, + "seed", + "steps", + "cfgScale", + "sampler" + ); + foreach (var (_, node) in graph) + { + if ( + GenerationDataJson.Get(GenerationDataJson.Get(node, "inputs"), "ckpt_name") + is JsonValue checkpoint + && checkpoint.TryGetValue(out var name) + ) + { + parameters.ModelName = name; + break; + } + } + + return Result(GenerationDataSource.Civitai, parameters); + } + + private static GenerationDataInfo? ParseFooocus(MetadataEntries entries) + { + var json = GenerationDataJson.ParseObject(entries["parameters"]); + if (json is null || json.ContainsKey("num_inference_steps")) + return null; + if (!json.ContainsKey("sampler") || !json.ContainsKey("prompt")) + return null; + + var parameters = FromJson( + json, + "prompt", + "negative_prompt", + "base_model", + "seed", + "steps", + "guidance_scale", + "sampler" + ); + parameters.ModelHash = GenerationDataJson.GetString(json["base_model_hash"]); + return Result(GenerationDataSource.Fooocus, parameters); + } + + private static GenerationDataInfo? ParseAutomatic1111(MetadataEntries entries) + { + // CivitAI writes the parameters to the EXIF UserComment of its PNG files + var text = entries["parameters"] ?? entries["UserComment"]; + if (text is null) + return null; + + var lines = text.Replace("\r\n", "\n").Split('\n'); + + // the settings line: searched from the end, it names at least three sampler settings + var infoIndex = -1; + for (var i = lines.Length - 1; i >= 0; i--) + { + var count = A1111SamplerKeyRegex() + .Matches(lines[i]) + .Select(m => m.Groups[1].Value) + .Distinct() + .Count(); + if (count >= 3) + { + infoIndex = i; + break; + } + } + if (infoIndex < 0) + return null; + + var promptLines = lines[..infoIndex]; + // SD.Next repeats the prompts as templates before the settings + var templateIndex = Array.FindIndex(promptLines, l => A1111TemplateRegex().IsMatch(l)); + if (templateIndex > 0) + promptLines = promptLines[..templateIndex]; + + var prompts = string.Join('\n', promptLines).Split("Negative prompt:"); + if (prompts.Length > 2) + return null; + + var infoLine = lines[infoIndex]; + if ( + !infoLine.StartsWith("Steps:") + || !GenerationParameters.TryParse( + string.Join('\n', [.. promptLines, infoLine]), + out var parameters + ) + ) + { + var fields = GenerationParameters.ParseLine(infoLine); + parameters = new GenerationParameters + { + Steps = ParseInt(fields.GetValueOrDefault("Steps")), + Sampler = fields.GetValueOrDefault("Sampler"), + CfgScale = ParseDouble(fields.GetValueOrDefault("CFG scale")), + Seed = ulong.TryParse(fields.GetValueOrDefault("Seed"), out var seed) ? seed : 0, + ModelName = fields.GetValueOrDefault("Model"), + ModelHash = fields.GetValueOrDefault("Model hash"), + }; + if (fields.GetValueOrDefault("Size")?.Split('x', 2) is [var width, var height]) + { + parameters.Width = ParseInt(width); + parameters.Height = ParseInt(height); + } + } + + parameters.PositivePrompt = TrimToNull(prompts[0]); + parameters.NegativePrompt = prompts.Length > 1 ? TrimToNull(prompts[1]) : null; + return Result(GenerationDataSource.Automatic1111, parameters); + } + + private static GenerationDataInfo? ParseComfyUI(MetadataEntries entries) + { + var prompt = GenerationDataJson.ParseObject(entries["prompt"]); + if (!ComfyPromptReader.IsPrompt(prompt)) + return null; + + return Result(GenerationDataSource.ComfyUI, ComfyPromptReader.Read(prompt!)); + } + + private static GenerationDataInfo? ParseInvokeAI(MetadataEntries entries) + { + if (entries["sd-metadata"] is { } sdMetadata) + { + if (GenerationDataJson.ParseObject(sdMetadata) is not { } metadata) + return null; + if (metadata["image"] is not JsonObject image || image["prompt"] is not JsonArray promptList) + return null; + + var parameters = FromJson(image, "_", "_", null, "seed", "steps", "cfg_scale", "sampler"); + parameters.ModelName = GenerationDataJson.GetString(metadata["model_weights"]); + parameters.ModelHash = GenerationDataJson.GetString(metadata["model_hash"]); + var combined = string.Join( + ' ', + promptList + .Select(p => GenerationDataJson.GetString(GenerationDataJson.Get(p, "prompt"))) + .OfType() + ); + (parameters.PositivePrompt, parameters.NegativePrompt) = SplitInvokePrompt(combined); + return Result(GenerationDataSource.InvokeAI, parameters); + } + + // legacy "Dream" format: "prompt [negative]" -s 50 -S 42 -W 512 -H 512 -C 7.5 -A k_lms + if (entries["Dream"] is { } dream) + { + var match = InvokeDreamRegex().Match(dream); + if (!match.Success) + return null; + + var args = InvokeDreamArgRegex() + .Matches(match.Groups[2].Value) + .GroupBy(m => m.Groups[1].Value) + .ToDictionary(g => g.Key, g => g.First().Groups[2].Value); + var parameters = new GenerationParameters + { + Sampler = args.GetValueOrDefault("A"), + CfgScale = ParseDouble(args.GetValueOrDefault("C")), + Steps = ParseInt(args.GetValueOrDefault("s")), + Seed = ulong.TryParse(args.GetValueOrDefault("S"), out var seed) ? seed : 0, + Width = ParseInt(args.GetValueOrDefault("W")), + Height = ParseInt(args.GetValueOrDefault("H")), + }; + (parameters.PositivePrompt, parameters.NegativePrompt) = SplitInvokePrompt(match.Groups[1].Value); + return Result(GenerationDataSource.InvokeAI, parameters); + } + + if ( + GenerationDataJson.ParseObject(entries["invokeai_metadata"]) is { } json + && json.ContainsKey("scheduler") + ) + { + var parameters = FromJson( + json, + "positive_prompt", + "negative_prompt", + null, + "seed", + "steps", + "cfg_scale", + "scheduler" + ); + if (json["model"] is JsonObject model) + { + parameters.ModelName = + GenerationDataJson.GetString(model["model_name"]) + ?? GenerationDataJson.GetString(model["name"]); + } + return Result(GenerationDataSource.InvokeAI, parameters); + } + + // Some tools only keep the workflow graph + if ( + InvokeGraphReader.Read(GenerationDataJson.ParseObject(entries["invokeai_graph"])) is + { } graphParameters + ) + { + return Result(GenerationDataSource.InvokeAI, graphParameters); + } + + return null; + } + + /// + /// Legacy InvokeAI prompts hold the negative prompt in square brackets + /// + private static (string? Positive, string? Negative) SplitInvokePrompt(string combined) + { + var negatives = new List(); + var positive = InvokeNegativeRegex() + .Replace( + combined, + m => + { + negatives.Add(m.Groups[1].Value.Trim(' ', ',')); + return " "; + } + ); + return ( + TrimToNull(Regex.Replace(positive, @"\s+", " ").Trim(' ', ',')), + TrimToNull(string.Join(", ", negatives.Where(n => n.Length > 0))) + ); + } + + private static GenerationDataInfo? ParseNovelAI(MetadataEntries entries) + { + if (entries["Description"] is not { } description || entries["Source"] is not { } source) + return null; + if ( + GenerationDataJson.ParseObject(entries["Comment"]) is not { } json + || !json.ContainsKey("sampler") + ) + return null; + + var parameters = FromJson(json, "prompt", "uc", null, "seed", "steps", "scale", "sampler"); + // WEBP files can carry a Description with leading null characters or broken UTF-8 + var cleaned = description.TrimStart('\0').Trim(); + if (!cleaned.Contains('�') || parameters.PositivePrompt is null) + parameters.PositivePrompt = TrimToNull(cleaned); + + var model = NovelAIModelRegex().Match(source); + if (model.Success) + { + parameters.ModelName = model.Groups[1].Value; + parameters.ModelHash = model.Groups[2].Value; + } + return Result(GenerationDataSource.NovelAI, parameters); + } + + /// + /// Images labelled as AI made that carry no generation data, like those of ChatGPT or Gemini. + /// The labels are not verified. + /// + private static GenerationDataInfo? ParseProvenance(MetadataEntries entries) + { + if (entries.C2pa is { Length: > 0 } c2pa) + { + var text = Encoding.Latin1.GetString(c2pa); + if (AiSourceTypes.Any(text.Contains)) + { + return new GenerationDataInfo + { + Source = GenerationDataSource.C2pa, + Software = C2paSoftware(c2pa, text), + }; + } + } + + var sourceType = entries["DigitalSourceType"]; + if (sourceType is not null && AiSourceTypes.Contains(sourceType.Split('/').Last())) + { + return new GenerationDataInfo + { + Source = GenerationDataSource.Iptc, + Software = entries["CreatorTool"] ?? VendorSoftware(Vendor(entries["Publisher"])), + }; + } + + return null; + } + + private static readonly (Regex Pattern, string Vendor)[] Vendors = + [ + (new Regex("openai", RegexOptions.IgnoreCase), "OpenAI"), + (new Regex("google", RegexOptions.IgnoreCase), "Google"), + (new Regex("adobe", RegexOptions.IgnoreCase), "Adobe"), + (new Regex("microsoft", RegexOptions.IgnoreCase), "Microsoft"), + (new Regex(@"\bxai\b|spacexai|grok", RegexOptions.IgnoreCase), "xAI"), + (new Regex(@"\bmeta\b|facebook|instagram", RegexOptions.IgnoreCase), "Meta"), + (new Regex("stability", RegexOptions.IgnoreCase), "Stability AI"), + (new Regex("midjourney", RegexOptions.IgnoreCase), "Midjourney"), + (new Regex("runway", RegexOptions.IgnoreCase), "Runway"), + (new Regex("black forest labs", RegexOptions.IgnoreCase), "Black Forest Labs"), + ]; + + private static string? Vendor(string? text) => + text is null ? null : Vendors.FirstOrDefault(v => v.Pattern.IsMatch(text)).Vendor; + + private static string? VendorSoftware(string? vendor) => + vendor switch + { + "Google" => "Gemini", + "Meta" => "Meta AI", + "OpenAI" => "ChatGPT", + _ => vendor, + }; + + /// + /// Name the tool of a C2PA manifest from its software agent, or else its signer + /// + private static string? C2paSoftware(byte[] manifest, string text) + { + if (ReadC2paSoftwareAgent(manifest) is { Length: > 0 } name) + { + if (Regex.IsMatch(name, @"^(?:chatgpt\b|gpt-image|dall-e)", RegexOptions.IgnoreCase)) + return "ChatGPT"; + return Regex.Replace(name, @"\s+\d+(?:\.\d+)*$", ""); + } + return VendorSoftware(Vendor(text)); + } + + /// + /// Find the first "softwareAgent" of the CBOR encoded actions, either a text or a map with a name + /// + private static string? ReadC2paSoftwareAgent(byte[] manifest) + { + var key = "\u006DsoftwareAgent"u8; + var index = manifest.AsSpan().IndexOf(key); + if (index < 0) + return null; + var offset = index + key.Length; + + if (offset < manifest.Length && manifest[offset] is >= 0xA0 and <= 0xB7) + { + var nameKey = "\u0064name"u8; + var nameIndex = manifest.AsSpan(offset, Math.Min(256, manifest.Length - offset)).IndexOf(nameKey); + if (nameIndex < 0) + return null; + offset += nameIndex + nameKey.Length; + } + + return ReadCborText(manifest, offset); + } + + private static string? ReadCborText(byte[] data, int offset) + { + if (offset >= data.Length || data[offset] >> 5 != 3) + return null; + var info = data[offset] & 0x1F; + int length; + var start = offset + 1; + if (info < 24) + { + length = info; + } + else if (info == 24 && start < data.Length) + { + length = data[start]; + start += 1; + } + else if (info == 25 && start + 1 < data.Length) + { + length = (data[start] << 8) | data[start + 1]; + start += 2; + } + else + { + return null; + } + return start + length <= data.Length ? Encoding.UTF8.GetString(data, start, length).Trim() : null; + } + + #endregion + + #region Helpers + + private static string? TrimToNull(string? text) => string.IsNullOrWhiteSpace(text) ? null : text.Trim(); + + private static int ParseInt(string? text) => + double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) + ? (int)value + : 0; + + private static double ParseDouble(string? text) => + double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) ? value : 0; + + [GeneratedRegex(@"(?:^|[,\n])\s*App:\s*SD\.Next\b")] + private static partial Regex SdNextAppRegex(); + + [GeneratedRegex(@"(?:^|[,\n])\s*Version:\s*([^\s,]+)")] + private static partial Regex A1111VersionRegex(); + + [GeneratedRegex(@"(?:^|,)\s*(Sampler|CFG scale|Seed|Steps|ENSD|Schedule type):")] + private static partial Regex A1111SamplerKeyRegex(); + + [GeneratedRegex(@"^(?:Negative )?[Tt]emplate:")] + private static partial Regex A1111TemplateRegex(); + + [GeneratedRegex(@"^\s*JobId:", RegexOptions.Multiline)] + private static partial Regex DezgoJobRegex(); + + [GeneratedRegex(@"^Generated by Kus(?:Art|aPics)$")] + private static partial Regex KusArtRegex(); + + [GeneratedRegex(@"^""(.*?)""(.*)$", RegexOptions.Singleline)] + private static partial Regex InvokeDreamRegex(); + + [GeneratedRegex(@"\s+-([\p{L}\p{N}_]+)\s+(\S+)")] + private static partial Regex InvokeDreamArgRegex(); + + [GeneratedRegex(@"\[([^\[]*)\]")] + private static partial Regex InvokeNegativeRegex(); + + [GeneratedRegex(@"^(.*?)\s+([A-Z0-9]+)$")] + private static partial Regex NovelAIModelRegex(); + + #endregion +} diff --git a/StabilityMatrix.Core/Models/GenerationData/GenerationDataSource.cs b/StabilityMatrix.Core/Models/GenerationData/GenerationDataSource.cs new file mode 100644 index 000000000..64cfe554f --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/GenerationDataSource.cs @@ -0,0 +1,61 @@ +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Format of the generation data embedded in an image or video. +/// Tools that write the format of another one (Forge writes the AUTOMATIC1111 format) +/// are reported with that format, names the actual tool. +/// +public enum GenerationDataSource +{ + StabilityMatrix, + Automatic1111, + ComfyUI, + SwarmUI, + Fooocus, + RuinedFooocus, + InvokeAI, + NovelAI, + EasyDiffusion, + DrawThings, + TensorArt, + Civitai, + HuggingFaceSpace, + Dezgo, + KusArt, + + /// + /// No generation data, but C2PA manifests (Content Credentials) that label the image as AI made. + /// + C2pa, + + /// + /// No generation data, but an IPTC digital source type that labels the image as AI made. + /// + Iptc, +} + +public static class GenerationDataSourceExtensions +{ + public static string GetDisplayName(this GenerationDataSource source) => + source switch + { + GenerationDataSource.StabilityMatrix => "Stability Matrix", + GenerationDataSource.Automatic1111 => "AUTOMATIC1111", + GenerationDataSource.ComfyUI => "ComfyUI", + GenerationDataSource.SwarmUI => "SwarmUI", + GenerationDataSource.Fooocus => "Fooocus", + GenerationDataSource.RuinedFooocus => "RuinedFooocus", + GenerationDataSource.InvokeAI => "InvokeAI", + GenerationDataSource.NovelAI => "NovelAI", + GenerationDataSource.EasyDiffusion => "Easy Diffusion", + GenerationDataSource.DrawThings => "Draw Things", + GenerationDataSource.TensorArt => "TensorArt", + GenerationDataSource.Civitai => "Civitai", + GenerationDataSource.HuggingFaceSpace => "Hugging Face Space", + GenerationDataSource.Dezgo => "Dezgo", + GenerationDataSource.KusArt => "KusArt", + GenerationDataSource.C2pa => "C2PA", + GenerationDataSource.Iptc => "IPTC", + _ => source.ToString(), + }; +} diff --git a/StabilityMatrix.Core/Models/GenerationData/InvokeGraphReader.cs b/StabilityMatrix.Core/Models/GenerationData/InvokeGraphReader.cs new file mode 100644 index 000000000..0d72126c0 --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/InvokeGraphReader.cs @@ -0,0 +1,190 @@ +using System.Text.Json.Nodes; + +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Reads the prompts and settings of an InvokeAI invocation graph ("invokeai_graph"). +/// Values may be set on a node or come from another node through an edge. +/// +internal class InvokeGraphReader +{ + private const int MaxDepth = 16; + + private static readonly HashSet NonPromptInputs = + [ + "clip", + "clip2", + "t5_encoder", + "mask", + "unet", + "vae", + ]; + + private readonly Dictionary nodes = new(StringComparer.Ordinal); + private readonly Dictionary<(string Node, string Field), List<(string Node, string Field)>> incoming = []; + + private InvokeGraphReader(JsonObject graphNodes, JsonArray edges) + { + foreach (var (id, node) in graphNodes) + { + if (node is JsonObject obj) + nodes[id] = obj; + } + + foreach (var edge in edges) + { + var source = GenerationDataJson.Get(edge, "source"); + var destination = GenerationDataJson.Get(edge, "destination"); + if ( + GenerationDataJson.GetString(GenerationDataJson.Get(source, "node_id")) is not { } sourceNode + || GenerationDataJson.GetString(GenerationDataJson.Get(source, "field")) + is not { } sourceField + || GenerationDataJson.GetString(GenerationDataJson.Get(destination, "node_id")) + is not { } destNode + || GenerationDataJson.GetString(GenerationDataJson.Get(destination, "field")) + is not { } destField + ) + continue; + + var key = (destNode, destField); + if (!incoming.TryGetValue(key, out var sources)) + incoming[key] = sources = []; + sources.Add((sourceNode, sourceField)); + } + } + + public static GenerationParameters? Read(JsonObject? graph) + { + if (graph?["nodes"] is not JsonObject graphNodes || graph["edges"] is not JsonArray edges) + return null; + + var reader = new InvokeGraphReader(graphNodes, edges); + return reader.ReadCoreMetadata() ?? reader.ReadDenoiseNode(); + } + + private GenerationParameters? ReadCoreMetadata() + { + if (nodes.FirstOrDefault(n => Type(n.Value) == "core_metadata") is not ({ } id, { } node)) + return null; + + var parameters = new GenerationParameters + { + PositivePrompt = Trim(GenerationDataJson.GetString(Resolve(id, "positive_prompt"))), + NegativePrompt = Trim(GenerationDataJson.GetString(Resolve(id, "negative_prompt"))), + Seed = GenerationDataJson.GetSeed(Resolve(id, "seed")), + Steps = (int)(GenerationDataJson.GetDouble(Resolve(id, "steps")) ?? 0), + CfgScale = GenerationDataJson.GetDouble(Resolve(id, "cfg_scale") ?? Resolve(id, "guidance")) ?? 0, + Sampler = GenerationDataJson.GetString(Resolve(id, "scheduler")), + Width = (int)(GenerationDataJson.GetDouble(Resolve(id, "width")) ?? 0), + Height = (int)(GenerationDataJson.GetDouble(Resolve(id, "height")) ?? 0), + ModelName = ModelName(Resolve(id, "model")) ?? FindLoaderModel(), + }; + + // Upscale and canvas graphs leave the prompt out of the metadata node + if (parameters.PositivePrompt is null && ReadDenoiseNode() is { } denoise) + { + parameters.PositivePrompt = denoise.PositivePrompt; + parameters.NegativePrompt ??= denoise.NegativePrompt; + } + + return parameters.PositivePrompt is null && parameters.Steps == 0 ? null : parameters; + } + + private GenerationParameters? ReadDenoiseNode() + { + var denoise = nodes.FirstOrDefault(n => + n.Value.ContainsKey("steps") && n.Value.Any(p => p.Key.StartsWith("positive_")) + ); + if (denoise.Value is null) + return null; + + var id = denoise.Key; + var positiveField = denoise.Value.ContainsKey("positive_conditioning") + ? "positive_conditioning" + : denoise.Value.First(p => p.Key.StartsWith("positive_")).Key; + var negativeField = positiveField.Replace("positive_", "negative_"); + + var parameters = new GenerationParameters + { + PositivePrompt = FindPrompt(id, positiveField), + NegativePrompt = FindPrompt(id, negativeField), + Steps = (int)(GenerationDataJson.GetDouble(Resolve(id, "steps")) ?? 0), + CfgScale = GenerationDataJson.GetDouble(Resolve(id, "cfg_scale") ?? Resolve(id, "guidance")) ?? 0, + Sampler = GenerationDataJson.GetString(Resolve(id, "scheduler")), + ModelName = FindLoaderModel(), + }; + + foreach (var (noiseNode, _) in incoming.GetValueOrDefault((id, "noise")) ?? []) + { + parameters.Seed = GenerationDataJson.GetSeed(Resolve(noiseNode, "seed")); + parameters.Width = (int)(GenerationDataJson.GetDouble(Resolve(noiseNode, "width")) ?? 0); + parameters.Height = (int)(GenerationDataJson.GetDouble(Resolve(noiseNode, "height")) ?? 0); + } + + return parameters; + } + + /// + /// The value of a node's field, following edges to the node that produces it + /// + private JsonNode? Resolve(string nodeId, string field, int depth = 0) + { + if ( + depth < MaxDepth + && incoming.GetValueOrDefault((nodeId, field)) is [var (sourceNode, sourceField), ..] + ) + return Resolve(sourceNode, sourceField, depth + 1); + + return nodes.TryGetValue(nodeId, out var node) ? node[field] : null; + } + + /// + /// Walk upstream from a conditioning input (through collect nodes) to the prompt nodes feeding it + /// + private string? FindPrompt(string nodeId, string field) + { + var prompts = new List(); + var visited = new HashSet(StringComparer.Ordinal); + var queue = new Queue( + (incoming.GetValueOrDefault((nodeId, field)) ?? []).Select(source => source.Node) + ); + + while (queue.Count > 0 && visited.Count < 256) + { + var id = queue.Dequeue(); + if (!visited.Add(id) || !nodes.TryGetValue(id, out var node)) + continue; + + if (node.ContainsKey("prompt")) + { + if (Trim(GenerationDataJson.GetString(Resolve(id, "prompt"))) is { } prompt) + prompts.Add(prompt); + continue; + } + + foreach (var ((destNode, destField), sources) in incoming) + { + if (destNode != id || NonPromptInputs.Contains(destField)) + continue; + foreach (var (sourceNode, _) in sources) + queue.Enqueue(sourceNode); + } + } + + return prompts.Count == 0 ? null : string.Join(", ", prompts.Distinct()); + } + + private string? FindLoaderModel() => + nodes + .Values.Where(n => Type(n)?.EndsWith("model_loader") ?? false) + .Select(n => ModelName(n["model"])) + .FirstOrDefault(name => name is not null); + + private static string? ModelName(JsonNode? model) => + GenerationDataJson.GetString(GenerationDataJson.Get(model, "name")) + ?? GenerationDataJson.GetString(GenerationDataJson.Get(model, "model_name")); + + private static string? Type(JsonObject node) => GenerationDataJson.GetString(node["type"]); + + private static string? Trim(string? text) => string.IsNullOrWhiteSpace(text) ? null : text.Trim(); +} diff --git a/StabilityMatrix.Core/Models/GenerationData/MetadataEntries.cs b/StabilityMatrix.Core/Models/GenerationData/MetadataEntries.cs new file mode 100644 index 000000000..a3c2dd817 --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/MetadataEntries.cs @@ -0,0 +1,948 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; +using System.Text.RegularExpressions; + +namespace StabilityMatrix.Core.Models.GenerationData; + +public enum MetadataContainer +{ + Unknown, + Png, + Jpeg, + Webp, + Mp4, + Matroska, +} + +/// +/// Text metadata of an image or video, keyed by the name of the PNG chunk, EXIF tag, +/// XMP property or video tag it was read from. Only the metadata is read, never the image or media data. +/// +public partial class MetadataEntries +{ + private const int MaxTextLength = 64 * 1024 * 1024; + private const int MaxBoxLength = 64 * 1024 * 1024; + + private static readonly byte[] PngSignature = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + private const string ExifHeader = "Exif\0\0"; + private const string XmpHeader = "http://ns.adobe.com/xap/1.0/\0"; + private const string Mp4XmpUuid = "be7acfcb97a942e89c71999491e3afac"; + private const string Mp4C2paUuid = "d8fec3d61b0e483c92975828877ec481"; + + /// + /// PNG chunk names that tools pack into one JSON object, or label a value with, + /// when they write JPEG or WEBP files (NovelAI, ComfyUI save nodes). + /// + private static readonly string[] PackedKeys = + [ + "prompt", + "workflow", + "parameters", + "Comment", + "Description", + "Software", + "Source", + "Title", + "Generation time", + "generation_data", + "invokeai_metadata", + "smproj", + "parameters-json", + "Copyright", + "extra", + ]; + + private static readonly Dictionary ExifTextTags = new() + { + [0x010e] = "ImageDescription", + [0x010f] = "Make", + [0x0110] = "Model", + [0x0131] = "Software", + [0x013b] = "Artist", + [0x8298] = "Copyright", + [0x927c] = "MakerNote", + [0x9286] = "UserComment", + }; + + private static readonly Dictionary IlstNames = new() + { + ["©cmt"] = "comment", + ["desc"] = "description", + ["ldes"] = "synopsis", + ["©nam"] = "title", + ["©too"] = "encoder", + ["©swr"] = "encoder", + ["©ART"] = "artist", + ["©day"] = "date", + ["©gen"] = "genre", + ["©inf"] = "information", + ["cprt"] = "copyright", + }; + + public MetadataContainer Container { get; private init; } + + public Dictionary Entries { get; } = new(StringComparer.Ordinal); + + public string? Xmp { get; private set; } + + public byte[]? C2pa { get; private set; } + + public string? this[string key] => Entries.GetValueOrDefault(key); + + public bool IsEmpty => Entries.Count == 0 && Xmp is null && C2pa is null; + + private void Add(string key, string? value) + { + if (!string.IsNullOrEmpty(value)) + { + Entries.TryAdd(key, value); + } + } + + public static MetadataEntries FromDictionary( + IEnumerable> entries, + MetadataContainer container = MetadataContainer.Png + ) + { + var result = new MetadataEntries { Container = container }; + foreach (var (key, value) in entries) + { + result.Add(key, value); + } + return result; + } + + public static MetadataEntries Read(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + return Read(stream); + } + + public static MetadataEntries Read(Stream stream) + { + Span head = stackalloc byte[16]; + var headLength = stream.ReadAtLeast(head, head.Length, false); + head = head[..headLength]; + stream.Position = 0; + + if (head.StartsWith(PngSignature)) + return ReadPng(stream); + if (head.Length >= 3 && head[0] == 0xFF && head[1] == 0xD8 && head[2] == 0xFF) + return ReadJpeg(stream); + if (head.Length >= 12 && Ascii(head[..4]) == "RIFF" && Ascii(head[8..12]) == "WEBP") + return ReadWebp(stream); + if (head.Length >= 12 && Ascii(head[4..8]) == "ftyp") + return ReadMp4(stream); + if (head.Length >= 4 && head[0] == 0x1A && head[1] == 0x45 && head[2] == 0xDF && head[3] == 0xA3) + return ReadMatroska(stream); + + return new MetadataEntries(); + } + + #region PNG + + private static MetadataEntries ReadPng(Stream stream) + { + var result = new MetadataEntries { Container = MetadataContainer.Png }; + stream.Position = 8; + Span header = stackalloc byte[8]; + + while (stream.ReadAtLeast(header, 8, false) == 8) + { + var length = BinaryPrimitives.ReadUInt32BigEndian(header); + var type = Ascii(header[4..]); + if (length > int.MaxValue || stream.Position + length > stream.Length) + break; + + if (type is "tEXt" or "zTXt" or "iTXt" or "eXIf" or "caBX" && length <= MaxTextLength) + { + var data = ReadExactly(stream, (int)length); + switch (type) + { + case "eXIf": + // CivitAI writes its metadata to an eXIf chunk + foreach (var (key, value) in ReadExifEntries(data)) + result.Add(key, value); + break; + case "caBX": + result.C2pa ??= data; + break; + default: + if (ReadPngTextChunk(type, data) is var (keyword, text)) + { + result.Add(keyword, text); + if (keyword == "XML:com.adobe.xmp") + result.AddXmp(text); + } + break; + } + } + else if (type == "IEND") + { + break; + } + else + { + stream.Seek(length, SeekOrigin.Current); + } + + stream.Seek(4, SeekOrigin.Current); + } + + return result; + } + + private static (string Keyword, string Text)? ReadPngTextChunk(string type, byte[] data) + { + var keyEnd = Array.IndexOf(data, (byte)0); + if (keyEnd <= 0) + return null; + var keyword = Encoding.Latin1.GetString(data, 0, keyEnd); + + switch (type) + { + case "tEXt": + return (keyword, DecodeText(data.AsSpan(keyEnd + 1))); + case "zTXt": + { + if (keyEnd + 1 >= data.Length || data[keyEnd + 1] != 0) + return null; + var inflated = Inflate(data.AsSpan(keyEnd + 2)); + return inflated is null ? null : (keyword, DecodeText(inflated)); + } + case "iTXt": + { + if (keyEnd + 3 > data.Length) + return null; + var compressed = data[keyEnd + 1] != 0; + var langEnd = Array.IndexOf(data, (byte)0, keyEnd + 3); + if (langEnd < 0) + return null; + var translatedEnd = Array.IndexOf(data, (byte)0, langEnd + 1); + if (translatedEnd < 0) + return null; + var value = data.AsSpan(translatedEnd + 1); + if (compressed) + { + var inflated = Inflate(value); + if (inflated is null) + return null; + return (keyword, Encoding.UTF8.GetString(inflated)); + } + return (keyword, Encoding.UTF8.GetString(value)); + } + default: + return null; + } + } + + private static byte[]? Inflate(ReadOnlySpan data) + { + try + { + using var input = new MemoryStream(data.ToArray()); + using var zlib = new ZLibStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(); + zlib.CopyTo(output); + return output.ToArray(); + } + catch (InvalidDataException) + { + return null; + } + } + + #endregion + + #region JPEG / WEBP + + private static MetadataEntries ReadJpeg(Stream stream) + { + var result = new MetadataEntries { Container = MetadataContainer.Jpeg }; + var comments = new List(); + using var c2pa = new MemoryStream(); + byte[]? exif = null; + + stream.Position = 2; + Span marker = stackalloc byte[4]; + while (stream.ReadAtLeast(marker[..2], 2, false) == 2) + { + if (marker[0] != 0xFF) + break; + var type = marker[1]; + if (type == 0xFF) + { + stream.Seek(-1, SeekOrigin.Current); + continue; + } + // start of scan or end of image: no metadata after this + if (type is 0xDA or 0xD9) + break; + if (type == 0x01 || type is >= 0xD0 and <= 0xD7) + continue; + + if (stream.ReadAtLeast(marker[2..], 2, false) != 2) + break; + var length = BinaryPrimitives.ReadUInt16BigEndian(marker[2..]) - 2; + if (length < 0 || stream.Position + length > stream.Length) + break; + + if (type is 0xE1 or 0xFE or 0xEB) + { + var segment = ReadExactly(stream, length); + if (type == 0xE1) + { + if (exif is null && StartsWithAscii(segment, ExifHeader)) + exif = segment; + else if (result.Xmp is null && StartsWithAscii(segment, XmpHeader)) + result.Xmp = Encoding.UTF8.GetString( + segment, + XmpHeader.Length, + segment.Length - XmpHeader.Length + ); + } + else if (type == 0xFE) + { + var text = DecodeText(segment).TrimEnd('\0'); + if (text.Length > 0) + comments.Add(text); + } + else if (segment.Length > 8 && segment[0] == (byte)'J' && segment[1] == (byte)'P') + { + // APP11 JUMBF: common header, then the box data (repeated box header after the first segment) + var sequence = BinaryPrimitives.ReadUInt32BigEndian(segment.AsSpan(4)); + c2pa.Write(segment, sequence <= 1 ? 8 : 16, segment.Length - (sequence <= 1 ? 8 : 16)); + } + } + else + { + stream.Seek(length, SeekOrigin.Current); + } + } + + if (exif is not null) + { + foreach (var (key, value) in ReadExifEntries(exif)) + result.Add(key, value); + } + if (comments.Count > 0) + result.Add("Comment", string.Join('\n', comments)); + if (c2pa.Length > 0) + result.C2pa = c2pa.ToArray(); + + result.FinishExifImage(); + return result; + } + + private static MetadataEntries ReadWebp(Stream stream) + { + var result = new MetadataEntries { Container = MetadataContainer.Webp }; + stream.Position = 12; + Span header = stackalloc byte[8]; + byte[]? exif = null; + + while (stream.ReadAtLeast(header, 8, false) == 8) + { + var type = Ascii(header[..4]); + var size = BinaryPrimitives.ReadUInt32LittleEndian(header[4..]); + if (size > int.MaxValue || stream.Position + size > stream.Length) + break; + + if (type is "EXIF" or "XMP " or "C2PA" && size <= MaxTextLength) + { + var data = ReadExactly(stream, (int)size); + switch (type) + { + case "EXIF": + exif ??= data; + break; + case "XMP ": + result.Xmp ??= Encoding.UTF8.GetString(data); + break; + default: + result.C2pa ??= data; + break; + } + stream.Seek(size % 2, SeekOrigin.Current); + } + else + { + stream.Seek(size + size % 2, SeekOrigin.Current); + } + } + + if (exif is not null) + { + foreach (var (key, value) in ReadExifEntries(exif)) + result.Add(key, value); + } + + result.FinishExifImage(); + return result; + } + + private void FinishExifImage() + { + if (Xmp is not null) + { + Add("XMP", Xmp); + AddXmp(Xmp); + } + UnpackJsonEntries(this["UserComment"]); + UnpackJsonEntries(this["Comment"]); + } + + /// + /// Add the values of a JSON object holding PNG chunk names as entries of their own + /// + private void UnpackJsonEntries(string? text) + { + if (text is null || !text.TrimStart().StartsWith('{')) + return; + if (GenerationDataJson.ParseObject(text) is not { } json) + return; + + foreach (var key in PackedKeys) + { + if (Entries.ContainsKey(key) || json[key] is not { } value) + continue; + Add(key, GenerationDataJson.AsText(value)); + } + } + + public static Dictionary ReadExifEntries(byte[] exif) + { + var entries = new Dictionary(StringComparer.Ordinal); + var tiff = exif.AsSpan(); + if (StartsWithAscii(exif, ExifHeader)) + tiff = tiff[6..]; + if (tiff.Length < 8) + return entries; + + var byteOrder = Ascii(tiff[..2]); + if (byteOrder is not ("II" or "MM")) + return entries; + var littleEndian = byteOrder == "II"; + var data = tiff.ToArray(); + + var visited = new HashSet(); + ReadIfd(U32(data, 4, littleEndian)); + return entries; + + void ReadIfd(uint ifdOffset) + { + if (!visited.Add(ifdOffset) || ifdOffset + 2 > data.Length) + return; + + var count = U16(data, (int)ifdOffset, littleEndian); + for (var i = 0; i < count; i++) + { + var entry = (int)ifdOffset + 2 + i * 12; + if (entry + 12 > data.Length) + return; + var tag = U16(data, entry, littleEndian); + var type = U16(data, entry + 2, littleEndian); + var size = (long)TypeSize(type) * U32(data, entry + 4, littleEndian); + + if (tag == 0x8769) + { + ReadIfd(U32(data, entry + 8, littleEndian)); + continue; + } + + if (!ExifTextTags.TryGetValue(tag, out var name) || size > data.Length) + continue; + long valueOffset = size <= 4 ? entry + 8 : U32(data, entry + 8, littleEndian); + if (valueOffset + size > data.Length) + continue; + var value = data.AsSpan((int)valueOffset, (int)size); + + string text; + if (name == "UserComment") + text = DecodeUserComment(value, littleEndian); + else if (type is 1 or 2 or 7) + text = DecodeText(value).TrimEnd('\0'); + else + continue; + if (text.Length == 0) + continue; + + // ComfyUI save nodes label values with the PNG chunk they stand in for: "prompt:{...}", "Workflow: {...}" + var label = ChunkLabelRegex().Match(text); + if ( + label.Success + && name != "UserComment" + && PackedKeys.Contains(label.Groups[1].Value.ToLowerInvariant()) + ) + { + entries.TryAdd(label.Groups[1].Value.ToLowerInvariant(), text[label.Length..]); + } + else + { + entries.TryAdd(name, text); + } + } + } + } + + private static int TypeSize(int type) => + type switch + { + 3 or 8 => 2, + 4 or 9 or 11 => 4, + 5 or 10 or 12 => 8, + _ => 1, + }; + + /// + /// Decode an EXIF UserComment value: an 8 byte character code followed by the text + /// + public static string DecodeUserComment(ReadOnlySpan value, bool littleEndianHint = false) + { + var code = value.Length >= 8 ? Encoding.Latin1.GetString(value[..8]) : string.Empty; + var body = value.Length >= 8 ? value[8..] : value; + if (code == "UNICODE\0") + return DecodeUtf16Guess(body, littleEndianHint); + if (code is "ASCII\0\0\0" or "\0\0\0\0\0\0\0\0") + return DecodeText(body).TrimEnd('\0'); + // no character code: some ComfyUI save nodes write plain UTF-8 + return DecodeText(value).TrimEnd('\0'); + } + + /// + /// Tools write UTF-16 in either byte order regardless of the TIFF header, + /// generation data is mostly ASCII so the order that yields far more ASCII is the right one. + /// + private static string DecodeUtf16Guess(ReadOnlySpan data, bool littleEndianHint) + { + data = data[..(data.Length - data.Length % 2)]; + var preferred = (littleEndianHint ? Encoding.Unicode : Encoding.BigEndianUnicode).GetString(data); + var other = (littleEndianHint ? Encoding.BigEndianUnicode : Encoding.Unicode).GetString(data); + var preferredAscii = CountAscii(preferred); + var otherAscii = CountAscii(other); + var text = otherAscii > preferredAscii * 2 && otherAscii * 2 > other.Length ? other : preferred; + return text.TrimStart('').TrimEnd('\0'); + } + + private static int CountAscii(string text) => + text.Count(c => c is >= ' ' and < '\x7f' or '\n' or '\r' or '\t'); + + #endregion + + #region XMP + + private void AddXmp(string xmp) + { + Xmp ??= xmp; + Add("Description", GetXmpProperty(xmp, "dc:description")); + Add("UserComment", GetXmpProperty(xmp, "exif:UserComment")); + Add("CreatorTool", GetXmpProperty(xmp, "xmp:CreatorTool")); + Add("DigitalSourceType", GetXmpProperty(xmp, "*:DigitalSourceType")); + Add("Publisher", GetXmpProperty(xmp, "dc:publisher")); + } + + /// + /// Read a simple XMP property, written as an attribute or as an element + /// (for language alternatives and lists, the first item). + /// + public static string? GetXmpProperty(string xmp, string name) + { + var escaped = name.StartsWith("*:") ? @"[\w.-]+:" + Regex.Escape(name[2..]) : Regex.Escape(name); + + var attribute = Regex.Match(xmp, $@"\s{escaped}\s*=\s*""([^""]*)"""); + if (!attribute.Success) + attribute = Regex.Match(xmp, $@"\s{escaped}\s*=\s*'([^']*)'"); + if (attribute.Success) + return DecodeXmlText(attribute.Groups[1].Value); + + var element = Regex.Match(xmp, $@"<({escaped})(?:\s[^>]*)?>([\s\S]*?)"); + if (!element.Success) + return null; + var item = Regex.Match(element.Groups[2].Value, @"]*)?>([\s\S]*?)"); + return DecodeXmlText((item.Success ? item.Groups[1].Value : element.Groups[2].Value).Trim()); + } + + private static string DecodeXmlText(string text) => System.Net.WebUtility.HtmlDecode(text); + + #endregion + + #region MP4 / MOV + + private static MetadataEntries ReadMp4(Stream stream) + { + var result = new MetadataEntries { Container = MetadataContainer.Mp4 }; + var raw = new Dictionary(StringComparer.Ordinal); + + foreach (var (type, start, length) in ReadBoxHeaders(stream, 0, stream.Length)) + { + if (type == "moov") + { + foreach (var (childType, childStart, childLength) in ReadBoxHeaders(stream, start, length)) + { + if (childType is not ("meta" or "udta") || childLength > MaxBoxLength) + continue; + stream.Position = childStart; + var data = ReadExactly(stream, (int)childLength); + if (childType == "meta") + ReadMp4Meta(data, raw); + else + ReadMp4Udta(data, raw, result); + } + } + else if (type == "uuid" && length >= 16 && length <= MaxBoxLength) + { + stream.Position = start; + var data = ReadExactly(stream, (int)length); + var uuid = Convert.ToHexStringLower(data, 0, 16); + if (uuid == Mp4XmpUuid) + { + result.Xmp ??= Encoding.UTF8.GetString(data, 16, data.Length - 16); + } + else if (uuid == Mp4C2paUuid && data.Length > 20) + { + var body = data.AsSpan(20); + var end = body.IndexOf((byte)0); + if ( + end > 0 + && Encoding.Latin1.GetString(body[..end]) == "manifest" + && end + 9 <= body.Length + ) + result.C2pa ??= body[(end + 9)..].ToArray(); + } + } + } + + result.FinishVideo(raw); + return result; + } + + private static IEnumerable<(string Type, long Start, long Length)> ReadBoxHeaders( + Stream stream, + long offset, + long length + ) + { + var end = Math.Min(offset + length, stream.Length); + var header = new byte[16]; + while (offset + 8 <= end) + { + stream.Position = offset; + if (stream.ReadAtLeast(header.AsSpan(0, 8), 8, false) != 8) + yield break; + long size = BinaryPrimitives.ReadUInt32BigEndian(header); + var type = Encoding.Latin1.GetString(header, 4, 4); + var headerSize = 8; + if (size == 1) + { + if (stream.ReadAtLeast(header.AsSpan(8, 8), 8, false) != 8) + yield break; + size = (long)BinaryPrimitives.ReadUInt64BigEndian(header.AsSpan(8)); + headerSize = 16; + } + else if (size == 0) + { + size = end - offset; + } + if (size < headerSize) + yield break; + + yield return (type, offset + headerSize, Math.Min(size, end - offset) - headerSize); + offset += size; + } + } + + private static List<(string Type, byte[] Data)> ReadBoxes(byte[] data) + { + using var stream = new MemoryStream(data, false); + var boxes = new List<(string, byte[])>(); + foreach (var (type, start, length) in ReadBoxHeaders(stream, 0, data.Length)) + { + boxes.Add((type, data.AsSpan((int)start, (int)length).ToArray())); + } + return boxes; + } + + private static void ReadMp4Meta(byte[] meta, Dictionary raw) + { + // a full box in MP4 files (version and flags first) but not always in QuickTime files + var boxes = ReadBoxes(meta); + if (!boxes.Any(b => b.Type == "hdlr") && meta.Length > 4) + boxes = ReadBoxes(meta[4..]); + + var keys = boxes.FirstOrDefault(b => b.Type == "keys").Data; + var ilst = boxes.FirstOrDefault(b => b.Type == "ilst").Data; + if (ilst is null) + return; + + var keyNames = new List(); + if (keys is not null) + { + var offset = 8; + while (offset + 8 <= keys.Length) + { + var size = (int)BinaryPrimitives.ReadUInt32BigEndian(keys.AsSpan(offset)); + if (size < 8 || offset + size > keys.Length) + break; + keyNames.Add(Encoding.UTF8.GetString(keys, offset + 8, size - 8)); + offset += size; + } + } + + foreach (var (type, item) in ReadBoxes(ilst)) + { + var dataBox = ReadBoxes(item).FirstOrDefault(b => b.Type == "data").Data; + if (dataBox is null || dataBox.Length < 8) + continue; + var dataType = BinaryPrimitives.ReadUInt32BigEndian(dataBox) & 0xFFFFFF; + if (dataType is not (1 or 2)) + continue; + var value = + dataType == 1 + ? Encoding.UTF8.GetString(dataBox, 8, dataBox.Length - 8) + : Encoding.BigEndianUnicode.GetString(dataBox, 8, dataBox.Length - 8); + + string name; + if (type[0] == '\0') + { + var index = type.Aggregate(0, (total, c) => total * 256 + c); + if (index < 1 || index > keyNames.Count) + continue; + name = keyNames[index - 1].Replace("com.apple.quicktime.", ""); + } + else + { + name = IlstNames.GetValueOrDefault(type, type); + } + raw.TryAdd(name, value); + } + } + + private static void ReadMp4Udta(byte[] udta, Dictionary raw, MetadataEntries result) + { + foreach (var (type, data) in ReadBoxes(udta)) + { + if (type == "meta") + { + ReadMp4Meta(data, raw); + } + else if (type == "XMP_") + { + result.Xmp ??= Encoding.UTF8.GetString(data); + } + else if (type[0] == '©' && data.Length >= 4) + { + // QuickTime user data text: 16 bit length, language code, text + var length = (data[0] << 8) | data[1]; + if (length > 0 && 4 + length <= data.Length) + raw.TryAdd( + IlstNames.GetValueOrDefault(type, type), + Encoding.UTF8.GetString(data, 4, length) + ); + } + } + } + + #endregion + + #region Matroska / WebM + + private const uint EbmlSegment = 0x18538067; + private const uint EbmlInfo = 0x1549a966; + private const uint EbmlTitle = 0x7ba9; + private const uint EbmlCluster = 0x1f43b675; + private const uint EbmlTags = 0x1254c367; + private const uint EbmlTag = 0x7373; + private const uint EbmlTargets = 0x63c0; + private const uint EbmlSimpleTag = 0x67c8; + private const uint EbmlTagName = 0x45a3; + private const uint EbmlTagString = 0x4487; + private static readonly HashSet EbmlPartTargets = [0x63c5, 0x63c9, 0x63c4, 0x63c6]; + + private static MetadataEntries ReadMatroska(Stream stream) + { + var result = new MetadataEntries { Container = MetadataContainer.Matroska }; + var raw = new Dictionary(StringComparer.Ordinal); + + var segment = ReadEbmlHeaders(stream, 0, stream.Length).FirstOrDefault(e => e.Id == EbmlSegment); + if (segment.Id == EbmlSegment) + { + foreach ( + var (id, start, length, unknownSize) in ReadEbmlHeaders(stream, segment.Start, segment.Length) + ) + { + // clusters of unknown size can't be skipped, and tags come after them + if (unknownSize && id == EbmlCluster) + break; + if (id is not (EbmlTags or EbmlInfo) || length > MaxBoxLength) + continue; + + stream.Position = start; + var data = ReadExactly(stream, (int)length); + if (id == EbmlInfo) + { + var title = ReadEbml(data).FirstOrDefault(e => e.Id == EbmlTitle); + if (title.Data is not null) + raw.TryAdd("TITLE", Encoding.UTF8.GetString(title.Data)); + continue; + } + + foreach (var tag in ReadEbml(data).Where(e => e.Id == EbmlTag)) + { + var targets = ReadEbml(tag.Data).FirstOrDefault(e => e.Id == EbmlTargets); + if ( + targets.Data is not null + && ReadEbml(targets.Data).Any(t => EbmlPartTargets.Contains(t.Id)) + ) + continue; + ReadSimpleTags(tag.Data, raw); + } + } + } + + result.FinishVideo(raw); + return result; + } + + private static void ReadSimpleTags(byte[] data, Dictionary raw) + { + foreach (var simpleTag in ReadEbml(data).Where(e => e.Id == EbmlSimpleTag)) + { + var children = ReadEbml(simpleTag.Data); + var name = children.FirstOrDefault(e => e.Id == EbmlTagName).Data; + var value = children.FirstOrDefault(e => e.Id == EbmlTagString).Data; + if (name is not null && value is not null) + raw.TryAdd(Encoding.UTF8.GetString(name), Encoding.UTF8.GetString(value)); + ReadSimpleTags(simpleTag.Data, raw); + } + } + + private static (ulong Value, int Length, bool Unknown)? ReadVint(ReadOnlySpan data, bool keepMarker) + { + if (data.Length == 0 || data[0] == 0) + return null; + var length = System.Numerics.BitOperations.LeadingZeroCount((uint)data[0]) - 23; + if (length > data.Length || length > 8) + return null; + ulong value = keepMarker ? data[0] : (ulong)(data[0] & (0xFF >> length)); + var allOnes = value == (ulong)(0xFF >> length); + for (var i = 1; i < length; i++) + { + value = value * 256 + data[i]; + if (data[i] != 0xFF) + allOnes = false; + } + return (value, length, !keepMarker && allOnes); + } + + private static IEnumerable<(uint Id, long Start, long Length, bool UnknownSize)> ReadEbmlHeaders( + Stream stream, + long offset, + long length + ) + { + var end = Math.Min(offset + length, stream.Length); + var header = new byte[16]; + while (offset < end) + { + stream.Position = offset; + var read = stream.ReadAtLeast(header, header.Length, false); + var span = header.AsSpan(0, read); + if (ReadVint(span, true) is not { } id || ReadVint(span[id.Length..], false) is not { } size) + yield break; + var start = offset + id.Length + size.Length; + var elementEnd = size.Unknown ? end : Math.Min(start + (long)size.Value, end); + yield return ((uint)id.Value, start, elementEnd - start, size.Unknown); + offset = elementEnd; + } + } + + private static List<(uint Id, byte[] Data)> ReadEbml(byte[] data) + { + var elements = new List<(uint, byte[])>(); + var offset = 0; + while (offset < data.Length) + { + var span = data.AsSpan(offset); + if (ReadVint(span, true) is not { } id || ReadVint(span[id.Length..], false) is not { } size) + break; + var start = offset + id.Length + size.Length; + var end = size.Unknown + ? data.Length + : (int)Math.Min((ulong)start + size.Value, (ulong)data.Length); + if (start > end) + break; + elements.Add(((uint)id.Value, data[start..end])); + offset = end; + } + return elements; + } + + #endregion + + /// + /// Video tags under lower-case names. AUTOMATIC1111 style parameters, which Forge writes + /// to the description, are also stored under "parameters". + /// + private void FinishVideo(Dictionary raw) + { + foreach (var (key, value) in raw) + Add(key.ToLowerInvariant(), value); + + var parameters = new[] { this["parameters"], this["description"], this["comment"] }.FirstOrDefault( + v => v is not null && A1111StepsRegex().IsMatch(v) + ); + if (parameters is not null) + Entries["parameters"] = parameters; + + if (Xmp is not null) + AddXmp(Xmp); + } + + #region Helpers + + private static byte[] ReadExactly(Stream stream, int length) + { + var buffer = new byte[length]; + var read = stream.ReadAtLeast(buffer, length, false); + return read == length ? buffer : buffer[..read]; + } + + private static string Ascii(ReadOnlySpan bytes) => Encoding.Latin1.GetString(bytes); + + private static bool StartsWithAscii(byte[] data, string prefix) => + data.Length >= prefix.Length && Encoding.Latin1.GetString(data, 0, prefix.Length) == prefix; + + private static ushort U16(byte[] data, int offset, bool littleEndian) => + littleEndian + ? BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(offset)) + : BinaryPrimitives.ReadUInt16BigEndian(data.AsSpan(offset)); + + private static uint U32(byte[] data, int offset, bool littleEndian) => + offset + 4 > data.Length ? uint.MaxValue + : littleEndian ? BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(offset)) + : BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(offset)); + + /// + /// Text that is not valid UTF-8 is read as Latin-1, as the PNG specification asks for. + /// + private static string DecodeText(ReadOnlySpan data) + { + try + { + return new UTF8Encoding(false, true).GetString(data); + } + catch (DecoderFallbackException) + { + return Encoding.Latin1.GetString(data); + } + } + + [GeneratedRegex(@"^([A-Za-z]+):(?:\s|(?=[{\[]))")] + private static partial Regex ChunkLabelRegex(); + + [GeneratedRegex(@"(?:^|\n)Steps: \d")] + private static partial Regex A1111StepsRegex(); + + #endregion +} diff --git a/StabilityMatrix.Core/Models/GenerationData/SimpleYaml.cs b/StabilityMatrix.Core/Models/GenerationData/SimpleYaml.cs new file mode 100644 index 000000000..0224991d0 --- /dev/null +++ b/StabilityMatrix.Core/Models/GenerationData/SimpleYaml.cs @@ -0,0 +1,122 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +namespace StabilityMatrix.Core.Models.GenerationData; + +/// +/// Parses the small subset of YAML that tools use to write flat or nested "key: value" records. +/// +internal static partial class SimpleYaml +{ + public static JsonObject? Parse(string text) + { + var root = new JsonObject(); + var stack = new Stack<(int Indent, JsonObject Object)>(); + stack.Push((-1, root)); + var lines = text.Replace("\r\n", "\n").Split('\n'); + + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i]; + if (string.IsNullOrWhiteSpace(line) || line.TrimStart().StartsWith('#')) + continue; + var indent = IndentOf(line); + var match = KeyValueRegex().Match(line.Trim()); + if (!match.Success) + return null; + + while (stack.Count > 1 && indent <= stack.Peek().Indent) + stack.Pop(); + var parent = stack.Peek().Object; + var key = match.Groups[1].Value; + var rawValue = match.Groups[2].Success ? match.Groups[2].Value : string.Empty; + + if (rawValue.Length == 0) + { + var child = new JsonObject(); + parent[key] = child; + stack.Push((indent, child)); + continue; + } + + var block = BlockScalarRegex().Match(rawValue); + if (!block.Success) + { + parent[key] = ParseScalar(rawValue); + continue; + } + + // block scalar: the following lines that are indented deeper, or empty + var body = new List(); + while ( + i + 1 < lines.Length + && (string.IsNullOrWhiteSpace(lines[i + 1]) || IndentOf(lines[i + 1]) > indent) + ) + body.Add(lines[++i]); + while (body.Count > 0 && string.IsNullOrWhiteSpace(body[^1])) + body.RemoveAt(body.Count - 1); + + var nonEmpty = body.Where(l => !string.IsNullOrWhiteSpace(l)).ToList(); + var bodyIndent = nonEmpty.Count == 0 ? 0 : nonEmpty.Min(IndentOf); + var content = body.Select(l => l.Length >= bodyIndent ? l[bodyIndent..] : string.Empty).ToList(); + + string value; + if (block.Groups[1].Value == "|") + { + value = string.Join('\n', content); + } + else + { + var folded = new StringBuilder(); + for (var j = 0; j < content.Count; j++) + { + if (content[j].Length == 0) + folded.Append('\n'); + else + folded.Append(j > 0 && content[j - 1].Length > 0 ? " " : "").Append(content[j]); + } + value = folded.ToString(); + } + parent[key] = block.Groups[2].Value == "-" ? value : value + "\n"; + } + + return root; + } + + private static int IndentOf(string line) => line.Length - line.TrimStart().Length; + + private static JsonNode? ParseScalar(string value) + { + if (value.Length >= 2 && value[0] == '\'' && value[^1] == '\'') + return value[1..^1].Replace("''", "'"); + if (value.Length >= 2 && value[0] == '"' && value[^1] == '"') + { + try + { + return JsonSerializer.Deserialize(value); + } + catch (JsonException) + { + return value[1..^1]; + } + } + if (value is "true" or "false") + return value == "true"; + if (value is "null" or "~") + return null; + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var integer)) + return integer; + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var number)) + return number; + return value; + } + + [GeneratedRegex(@"^([^:]+?):(?:\s+(.*))?$")] + private static partial Regex KeyValueRegex(); + + [GeneratedRegex(@"^([>|])([+-]?)$")] + private static partial Regex BlockScalarRegex(); +} diff --git a/StabilityMatrix.Tests/Models/GenerationDataParserTests.cs b/StabilityMatrix.Tests/Models/GenerationDataParserTests.cs new file mode 100644 index 000000000..7e859c93d --- /dev/null +++ b/StabilityMatrix.Tests/Models/GenerationDataParserTests.cs @@ -0,0 +1,455 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; +using StabilityMatrix.Core.Models.Database; +using StabilityMatrix.Core.Models.GenerationData; + +namespace StabilityMatrix.Tests.Models; + +[TestClass] +public class GenerationDataParserTests +{ + private const string ComfyPrompt = """ + { + "3": {"class_type": "KSampler", "inputs": {"seed": 123456789012345, "steps": 25, "cfg": 6.5, + "sampler_name": "euler_ancestral", "scheduler": "normal", "denoise": 1.0, + "model": ["4", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}}, + "4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "sdxl.safetensors"}}, + "5": {"class_type": "EmptyLatentImage", "inputs": {"width": 1024, "height": 1024, "batch_size": 1}}, + "6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a cat in a hat", "clip": ["4", 1]}}, + "7": {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry", "clip": ["4", 1]}}, + "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "ComfyUI", "images": ["8", 0]}} + } + """; + + private const string A1111Parameters = """ + a cat in a hat + Negative prompt: blurry + Steps: 20, Sampler: DPM++ 2M, Schedule type: Karras, CFG scale: 7, Seed: 42, Size: 832x1216, Model hash: abc123, Model: sdxl, Version: neo-2.4 + """; + + private static readonly byte[] PngSignature = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + + private static GenerationDataInfo? ParsePng(params (string Type, byte[] Data)[] chunks) + { + using var stream = new MemoryStream(BuildPng(chunks)); + return GenerationDataParser.Parse(MetadataEntries.Read(stream)); + } + + [TestMethod] + public void Png_ComfyUISaveImage_ReadsPromptAndSettings() + { + var info = ParsePng(TextChunk("prompt", ComfyPrompt)); + + Assert.IsNotNull(info); + Assert.AreEqual(GenerationDataSource.ComfyUI, info.Source); + Assert.AreEqual("ComfyUI", info.DisplayName); + Assert.IsNotNull(info.Parameters); + Assert.AreEqual("a cat in a hat", info.Parameters.PositivePrompt); + Assert.AreEqual("blurry", info.Parameters.NegativePrompt); + Assert.AreEqual("sdxl.safetensors", info.Parameters.ModelName); + Assert.AreEqual(123456789012345UL, info.Parameters.Seed); + Assert.AreEqual(25, info.Parameters.Steps); + Assert.AreEqual(6.5, info.Parameters.CfgScale); + Assert.AreEqual("euler_ancestral", info.Parameters.Sampler); + } + + [TestMethod] + public void Png_ComfyUIPromptWithNaN_IsRecognized() + { + var prompt = ComfyPrompt.Replace("\"denoise\": 1.0", "\"denoise\": NaN"); + + var info = ParsePng(InternationalTextChunk("prompt", prompt, compressed: true)); + + Assert.AreEqual(GenerationDataSource.ComfyUI, info?.Source); + Assert.AreEqual("a cat in a hat", info?.Parameters?.PositivePrompt); + } + + [TestMethod] + public void Png_ComfyUILinkedSeed_FollowsPrimitiveNode() + { + var prompt = ComfyPrompt + .Replace("\"seed\": 123456789012345", "\"seed\": [\"10\", 0]") + .Replace( + "\"9\":", + "\"10\": {\"class_type\": \"PrimitiveInt\", \"inputs\": {\"value\": 777}},\n\"9\":" + ); + + var info = ParsePng(TextChunk("prompt", prompt)); + + Assert.AreEqual(777UL, info?.Parameters?.Seed); + } + + [TestMethod] + public void Png_A1111Parameters_NamesForgeNeo() + { + var info = ParsePng(TextChunk("parameters", A1111Parameters)); + + Assert.IsNotNull(info); + Assert.AreEqual(GenerationDataSource.Automatic1111, info.Source); + Assert.AreEqual("Forge Neo", info.DisplayName); + Assert.AreEqual("a cat in a hat", info.Parameters?.PositivePrompt); + Assert.AreEqual("blurry", info.Parameters?.NegativePrompt); + Assert.AreEqual(42UL, info.Parameters?.Seed); + Assert.AreEqual(832, info.Parameters?.Width); + } + + [TestMethod] + public void Png_A1111ParametersNextToComfyGraph_NamesComfyUI() + { + var parameters = A1111Parameters.Replace(", Version: neo-2.4", ""); + + var info = ParsePng(TextChunk("parameters", parameters), TextChunk("prompt", ComfyPrompt)); + + Assert.AreEqual(GenerationDataSource.Automatic1111, info?.Source); + Assert.AreEqual("ComfyUI", info?.DisplayName); + } + + [TestMethod] + public void Png_SwarmUI_IsPreferredOverOtherFormats() + { + const string swarm = """ + {"sui_image_params": {"prompt": "a dog", "negativeprompt": "cat", "model": "flux", "seed": 7, + "steps": 30, "cfgscale": 4.5, "sampler": "euler"}, "sui_models": []} + """; + + var info = ParsePng(CompressedTextChunk("parameters", swarm), TextChunk("prompt", ComfyPrompt)); + + Assert.IsNotNull(info); + Assert.AreEqual(GenerationDataSource.SwarmUI, info.Source); + Assert.AreEqual("a dog", info.Parameters?.PositivePrompt); + Assert.AreEqual("cat", info.Parameters?.NegativePrompt); + Assert.AreEqual("flux", info.Parameters?.ModelName); + Assert.AreEqual(30, info.Parameters?.Steps); + Assert.AreEqual(4.5, info.Parameters?.CfgScale); + } + + [TestMethod] + public void Png_InvokeAI_ReadsMetadata() + { + const string metadata = """ + {"positive_prompt": "a fox", "negative_prompt": "", "seed": 5, "steps": 30, "cfg_scale": 7.5, + "scheduler": "euler", "model": {"name": "juggernaut", "base": "sdxl"}} + """; + + var info = ParsePng(TextChunk("invokeai_metadata", metadata)); + + Assert.AreEqual(GenerationDataSource.InvokeAI, info?.Source); + Assert.AreEqual("a fox", info?.Parameters?.PositivePrompt); + Assert.IsNull(info?.Parameters?.NegativePrompt); + Assert.AreEqual("juggernaut", info?.Parameters?.ModelName); + } + + private const string InvokeGraph = """ + { + "id": "graph", + "nodes": { + "loader": {"id": "loader", "type": "sdxl_model_loader", "model": {"name": "juggernautXL", "base": "sdxl"}}, + "positive_prompt": {"id": "positive_prompt", "type": "string", "value": "a fox in the snow"}, + "pos_cond": {"id": "pos_cond", "type": "sdxl_compel_prompt", "prompt": "", "style": ""}, + "pos_collect": {"id": "pos_collect", "type": "collect", "collection": []}, + "neg_cond": {"id": "neg_cond", "type": "sdxl_compel_prompt", "prompt": "blurry", "style": "blurry"}, + "seed": {"id": "seed", "type": "integer", "value": 3951921344}, + "noise": {"id": "noise", "type": "noise", "seed": 0, "width": 832, "height": 1216}, + "denoise": {"id": "denoise", "type": "denoise_latents", "positive_conditioning": null, + "negative_conditioning": null, "steps": 24, "cfg_scale": 6.0, "scheduler": "euler_a"} + CORE_METADATA + }, + "edges": [ + {"source": {"node_id": "loader", "field": "clip"}, "destination": {"node_id": "pos_cond", "field": "clip"}}, + {"source": {"node_id": "positive_prompt", "field": "value"}, "destination": {"node_id": "pos_cond", "field": "prompt"}}, + {"source": {"node_id": "pos_cond", "field": "conditioning"}, "destination": {"node_id": "pos_collect", "field": "item"}}, + {"source": {"node_id": "pos_collect", "field": "collection"}, "destination": {"node_id": "denoise", "field": "positive_conditioning"}}, + {"source": {"node_id": "neg_cond", "field": "conditioning"}, "destination": {"node_id": "denoise", "field": "negative_conditioning"}}, + {"source": {"node_id": "seed", "field": "value"}, "destination": {"node_id": "noise", "field": "seed"}}, + {"source": {"node_id": "noise", "field": "noise"}, "destination": {"node_id": "denoise", "field": "noise"}}, + {"source": {"node_id": "seed", "field": "value"}, "destination": {"node_id": "core_metadata", "field": "seed"}}, + {"source": {"node_id": "positive_prompt", "field": "value"}, "destination": {"node_id": "core_metadata", "field": "positive_prompt"}} + ] + } + """; + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void Png_InvokeAIGraphOnly_ReadsPromptAndSettings(bool withCoreMetadata) + { + var coreMetadata = withCoreMetadata + ? """ + ,"core_metadata": {"id": "core_metadata", "type": "core_metadata", "positive_prompt": null, + "negative_prompt": "blurry", "seed": null, "steps": 24, "cfg_scale": 6.0, "scheduler": "euler_a", + "width": 832, "height": 1216, "model": {"name": "juggernautXL"}} + """ + : ""; + + var info = ParsePng(TextChunk("invokeai_graph", InvokeGraph.Replace("CORE_METADATA", coreMetadata))); + + Assert.IsNotNull(info); + Assert.AreEqual(GenerationDataSource.InvokeAI, info.Source); + Assert.AreEqual("a fox in the snow", info.Parameters?.PositivePrompt); + Assert.AreEqual("blurry", info.Parameters?.NegativePrompt); + Assert.AreEqual("juggernautXL", info.Parameters?.ModelName); + Assert.AreEqual(3951921344UL, info.Parameters?.Seed); + Assert.AreEqual(24, info.Parameters?.Steps); + Assert.AreEqual(6.0, info.Parameters?.CfgScale); + Assert.AreEqual("euler_a", info.Parameters?.Sampler); + Assert.AreEqual(832, info.Parameters?.Width); + } + + [TestMethod] + public void Png_NovelAI_ReadsDescriptionAndComment() + { + var info = ParsePng( + TextChunk("Description", "1girl, solo"), + TextChunk("Software", "NovelAI"), + TextChunk("Source", "NovelAI Diffusion V4.5 4BDE2A90"), + TextChunk( + "Comment", + """{"prompt": "1girl, solo", "uc": "lowres", "steps": 28, "scale": 5, "seed": 9, "sampler": "k_euler"}""" + ) + ); + + Assert.AreEqual(GenerationDataSource.NovelAI, info?.Source); + Assert.AreEqual("1girl, solo", info?.Parameters?.PositivePrompt); + Assert.AreEqual("lowres", info?.Parameters?.NegativePrompt); + Assert.AreEqual("NovelAI Diffusion V4.5", info?.Parameters?.ModelName); + } + + [TestMethod] + public void Png_WithoutGenerationData_ReturnsNull() + { + Assert.IsNull(ParsePng(TextChunk("Software", "GIMP 2.10"))); + Assert.IsNull(ParsePng()); + } + + [TestMethod] + public void Png_XmpDigitalSourceType_IsLabelledAsAiMade() + { + const string xmp = """ + + + + """; + + var info = ParsePng(InternationalTextChunk("XML:com.adobe.xmp", xmp, compressed: false)); + + Assert.AreEqual(GenerationDataSource.Iptc, info?.Source); + Assert.IsTrue(info?.IsProvenanceOnly); + Assert.IsNull(info?.Parameters); + } + + [TestMethod] + public void Jpeg_UserCommentUnicode_ReadsA1111Parameters() + { + var userComment = "UNICODE\0"u8 + .ToArray() + .Concat(Encoding.BigEndianUnicode.GetBytes(A1111Parameters)) + .ToArray(); + var jpeg = BuildJpeg(BuildExif(littleEndian: true, (0x9286, 7, userComment))); + + using var stream = new MemoryStream(jpeg); + var info = GenerationDataParser.Parse(MetadataEntries.Read(stream)); + + Assert.AreEqual(GenerationDataSource.Automatic1111, info?.Source); + Assert.AreEqual("Forge Neo", info?.DisplayName); + Assert.AreEqual("a cat in a hat", info?.Parameters?.PositivePrompt); + } + + [TestMethod] + public void Jpeg_ComfyUILabelledExifValue_ReadsPrompt() + { + var description = Encoding.UTF8.GetBytes("prompt:" + ComfyPrompt + "\0"); + var jpeg = BuildJpeg(BuildExif(littleEndian: false, (0x010e, 2, description))); + + using var stream = new MemoryStream(jpeg); + var info = GenerationDataParser.Parse(MetadataEntries.Read(stream)); + + Assert.AreEqual(GenerationDataSource.ComfyUI, info?.Source); + Assert.AreEqual("a cat in a hat", info?.Parameters?.PositivePrompt); + } + + [TestMethod] + public void Webp_StabilityMatrixImageDescription_IsRecognized() + { + var json = """{"PositivePrompt":"a lighthouse","NegativePrompt":"fog","Steps":12,"Seed":3}"""; + var exif = BuildExif(littleEndian: true, (0x010e, 2, Encoding.UTF8.GetBytes(json + "\0"))); + var webp = BuildRiff("WEBP", ("VP8L", new byte[6]), ("EXIF", exif)); + + using var stream = new MemoryStream(webp); + var info = GenerationDataParser.Parse(MetadataEntries.Read(stream)); + + Assert.AreEqual(GenerationDataSource.StabilityMatrix, info?.Source); + Assert.AreEqual("a lighthouse", info?.Parameters?.PositivePrompt); + Assert.AreEqual(12, info?.Parameters?.Steps); + } + + [TestMethod] + [DataRow("Version: v1.10.1", "Stable Diffusion WebUI")] + [DataRow("Version: f2.0.1v1.10.1-previous-669-gdfdcbab6", "Forge")] + [DataRow("Version: classic", "Forge Classic")] + [DataRow("Version: neo", "Forge Neo")] + [DataRow("Version: f1.3.0-v1.10.1RC-latest-1038-g19395bf9", "EasyReforge")] + [DataRow("Version: f1.0.0v2-v1.10.1RC-latest-2498-g5120e679", "reForge")] + [DataRow("App: SD.Next, Version: 2c4d0751", "SD.Next")] + [DataRow("Civitai resources: []", "Civitai")] + [DataRow("ADetailer version: 24.1.0", null)] + public void DetectA1111Software(string fields, string? expected) + { + var parameters = $"prompt\nSteps: 20, Sampler: Euler, CFG scale: 7, Seed: 1, {fields}"; + + Assert.AreEqual(expected, GenerationDataParser.DetectA1111Software(parameters)); + } + + [TestMethod] + public void LocalImageFile_FromPath_RecordsGenerationSource() + { + var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.png"); + File.WriteAllBytes(path, BuildPng([TextChunk("prompt", ComfyPrompt)])); + try + { + var file = LocalImageFile.FromPath(path); + + Assert.AreEqual(GenerationDataSource.ComfyUI, file.GenerationDataSource); + Assert.AreEqual("ComfyUI", file.GenerationDataLabel); + Assert.AreEqual("a cat in a hat", file.GenerationParameters?.PositivePrompt); + } + finally + { + File.Delete(path); + } + } + + #region File builders + + private static (string, byte[]) TextChunk(string keyword, string text) => + ("tEXt", [.. Encoding.Latin1.GetBytes(keyword), 0, .. Encoding.UTF8.GetBytes(text)]); + + private static (string, byte[]) CompressedTextChunk(string keyword, string text) => + ("zTXt", [.. Encoding.Latin1.GetBytes(keyword), 0, 0, .. Deflate(Encoding.UTF8.GetBytes(text))]); + + private static (string, byte[]) InternationalTextChunk(string keyword, string text, bool compressed) + { + var body = Encoding.UTF8.GetBytes(text); + return ( + "iTXt", + [ + .. Encoding.Latin1.GetBytes(keyword), + 0, + (byte)(compressed ? 1 : 0), + 0, + 0, + 0, + .. compressed ? Deflate(body) : body, + ] + ); + } + + private static byte[] Deflate(byte[] data) + { + using var output = new MemoryStream(); + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal)) + { + zlib.Write(data); + } + return output.ToArray(); + } + + private static byte[] BuildPng((string Type, byte[] Data)[] chunks) + { + using var stream = new MemoryStream(); + stream.Write(PngSignature); + + var header = new byte[13]; + BinaryPrimitives.WriteUInt32BigEndian(header, 64); + BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(4), 64); + header[8] = 8; + header[9] = 6; + + foreach ( + var (type, data) in chunks + .Prepend(("IHDR", header)) + .Append(("IDAT", new byte[4])) + .Append(("IEND", [])) + ) + { + Span length = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(length, (uint)data.Length); + stream.Write(length); + stream.Write(Encoding.ASCII.GetBytes(type)); + stream.Write(data); + stream.Write(new byte[4]); + } + return stream.ToArray(); + } + + private static byte[] BuildExif(bool littleEndian, params (int Tag, int Type, byte[] Value)[] tags) + { + void U16(Span span, int value) + { + if (littleEndian) + BinaryPrimitives.WriteUInt16LittleEndian(span, (ushort)value); + else + BinaryPrimitives.WriteUInt16BigEndian(span, (ushort)value); + } + + void U32(Span span, int value) + { + if (littleEndian) + BinaryPrimitives.WriteUInt32LittleEndian(span, (uint)value); + else + BinaryPrimitives.WriteUInt32BigEndian(span, (uint)value); + } + + var ifdSize = 2 + tags.Length * 12 + 4; + var tiff = new byte[8 + ifdSize + tags.Sum(t => t.Value.Length)]; + tiff[0] = tiff[1] = (byte)(littleEndian ? 'I' : 'M'); + U16(tiff.AsSpan(2), 42); + U32(tiff.AsSpan(4), 8); + U16(tiff.AsSpan(8), tags.Length); + + var valueOffset = 8 + ifdSize; + for (var i = 0; i < tags.Length; i++) + { + var (tag, type, value) = tags[i]; + var entry = tiff.AsSpan(10 + i * 12); + U16(entry, tag); + U16(entry[2..], type); + U32(entry[4..], value.Length); + U32(entry[8..], valueOffset); + value.CopyTo(tiff, valueOffset); + valueOffset += value.Length; + } + + return [.. "Exif\0\0"u8, .. tiff]; + } + + private static byte[] BuildJpeg(byte[] exif) + { + var length = new byte[2]; + BinaryPrimitives.WriteUInt16BigEndian(length, (ushort)(exif.Length + 2)); + return [0xFF, 0xD8, 0xFF, 0xE1, .. length, .. exif, 0xFF, 0xDA, 0x00, 0x02, 0xFF, 0xD9]; + } + + private static byte[] BuildRiff(string format, params (string Type, byte[] Data)[] chunks) + { + using var body = new MemoryStream(); + body.Write(Encoding.ASCII.GetBytes(format)); + foreach (var (type, data) in chunks) + { + Span size = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)data.Length); + body.Write(Encoding.ASCII.GetBytes(type)); + body.Write(size); + body.Write(data); + if (data.Length % 2 == 1) + body.WriteByte(0); + } + + var riffSize = new byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(riffSize, (uint)body.Length); + return [.. "RIFF"u8, .. riffSize, .. body.ToArray()]; + } + + #endregion +}