Skip to content

Commit 5c8ea6c

Browse files
AkiKurisuAkiKurisu
authored
feat(core): prepare unsupported image inputs (#98)
1 parent d674684 commit 5c8ea6c

9 files changed

Lines changed: 485 additions & 19 deletions

‎src/DotCraft.Core/Agents/ImageContentSanitizingChatClient.cs‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,24 @@ private static List<ChatMessage> SanitizeMessages(IEnumerable<ChatMessage> messa
7272
{
7373
if (isCurrentRoundTool && frc.Result is IEnumerable<AIContent> items)
7474
{
75+
var placeholderTexts = new List<string>();
7576
foreach (var item in items)
7677
{
7778
if (item is DataContent dc &&
78-
dc.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
79-
promotedImages.Add(dc);
79+
ModelImageInputPreparer.IsImageMediaType(dc.MediaType))
80+
{
81+
var prepared = ModelImageInputPreparer.Prepare(dc);
82+
if (prepared.Content != null)
83+
promotedImages.Add(prepared.Content);
84+
else if (!string.IsNullOrWhiteSpace(prepared.PlaceholderText))
85+
placeholderTexts.Add(prepared.PlaceholderText);
86+
}
8087
}
88+
89+
newContents.Add(new FunctionResultContent(
90+
frc.CallId,
91+
AppendPlaceholders(DescribeResult(frc.Result), placeholderTexts)));
92+
continue;
8193
}
8294

8395
newContents.Add(new FunctionResultContent(frc.CallId, DescribeResult(frc.Result)));
@@ -104,6 +116,17 @@ private static List<ChatMessage> SanitizeMessages(IEnumerable<ChatMessage> messa
104116
return result;
105117
}
106118

119+
private static string AppendPlaceholders(string text, IReadOnlyList<string> placeholderTexts)
120+
{
121+
if (placeholderTexts.Count == 0)
122+
return text;
123+
124+
var parts = new List<string> { text };
125+
foreach (var placeholder in placeholderTexts.Distinct(StringComparer.Ordinal))
126+
parts.Add(placeholder);
127+
return string.Join("\n", parts);
128+
}
129+
107130
private static bool HasNonTextContent(object? result)
108131
{
109132
if (result is IEnumerable<AIContent> items)

‎src/DotCraft.Core/Agents/ModelImageInputPreparer.cs‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
using Microsoft.Extensions.AI;
2+
using SixLabors.ImageSharp;
3+
using SixLabors.ImageSharp.Formats;
4+
using SixLabors.ImageSharp.Processing;
5+
6+
namespace DotCraft.Agents;
7+
8+
internal static class ModelImageInputPreparer
9+
{
10+
internal const string CouldNotProcessPlaceholder = "image content omitted because it could not be processed";
11+
internal const string TooLargePlaceholder = "image content omitted because it exceeded the supported size limit";
12+
13+
private const int MaxInputBytes = 64 * 1024 * 1024;
14+
private const int MaxDimension = 2048;
15+
private const int PatchSize = 32;
16+
private const int MaxPatches = 2500;
17+
18+
public static bool IsImageMediaType(string? mediaType) =>
19+
mediaType?.StartsWith("image/", StringComparison.OrdinalIgnoreCase) == true;
20+
21+
public static bool IsSupportedRemoteImageMediaType(string? mediaType) =>
22+
IsSupportedProviderImageMediaType(NormalizeMediaType(mediaType));
23+
24+
public static PreparedModelImageInput Prepare(DataContent source)
25+
{
26+
if (!IsImageMediaType(source.MediaType))
27+
return PreparedModelImageInput.Placeholder(CouldNotProcessPlaceholder);
28+
29+
if (source.Data.Length == 0 || source.Data.Length > MaxInputBytes)
30+
return PreparedModelImageInput.Placeholder(TooLargePlaceholder);
31+
32+
var bytes = source.Data.ToArray();
33+
try
34+
{
35+
var detectedFormat = Image.DetectFormat(bytes);
36+
var detectedMediaType = NormalizeMediaType(detectedFormat.DefaultMimeType);
37+
var info = Image.Identify(bytes);
38+
if (info == null || info.Width <= 0 || info.Height <= 0)
39+
return PreparedModelImageInput.Placeholder(CouldNotProcessPlaceholder);
40+
41+
var targetSize = CalculateTargetSize(info.Width, info.Height);
42+
var canPreserveSource = CanPreserveSourceBytes(detectedMediaType);
43+
if (canPreserveSource && targetSize.Width == info.Width && targetSize.Height == info.Height)
44+
return PreparedModelImageInput.Image(CopyMetadata(source, new DataContent(bytes, detectedMediaType)));
45+
46+
using var image = Image.Load(bytes);
47+
if (targetSize.Width != image.Width || targetSize.Height != image.Height)
48+
{
49+
image.Mutate(context => context.Resize(targetSize.Width, targetSize.Height));
50+
}
51+
52+
var outputMediaType = canPreserveSource ? detectedMediaType : "image/png";
53+
using var output = new MemoryStream();
54+
SaveImage(image, output, outputMediaType);
55+
return PreparedModelImageInput.Image(CopyMetadata(source, new DataContent(output.ToArray(), outputMediaType)));
56+
}
57+
catch (Exception ex) when (IsImagePreparationException(ex))
58+
{
59+
return PreparedModelImageInput.Placeholder(CouldNotProcessPlaceholder);
60+
}
61+
}
62+
63+
private static Size CalculateTargetSize(int width, int height)
64+
{
65+
var scale = 1.0;
66+
var maxSide = Math.Max(width, height);
67+
if (maxSide > MaxDimension)
68+
scale = Math.Min(scale, MaxDimension / (double)maxSide);
69+
70+
var patchBudgetPixels = MaxPatches * PatchSize * PatchSize;
71+
var pixelCount = (long)width * height;
72+
if (pixelCount > patchBudgetPixels)
73+
scale = Math.Min(scale, Math.Sqrt(patchBudgetPixels / (double)pixelCount));
74+
75+
var targetWidth = Math.Max(1, (int)Math.Floor(width * scale));
76+
var targetHeight = Math.Max(1, (int)Math.Floor(height * scale));
77+
78+
while (CountPatches(targetWidth, targetHeight) > MaxPatches)
79+
{
80+
scale *= 0.98;
81+
targetWidth = Math.Max(1, (int)Math.Floor(width * scale));
82+
targetHeight = Math.Max(1, (int)Math.Floor(height * scale));
83+
}
84+
85+
return new Size(targetWidth, targetHeight);
86+
}
87+
88+
private static long CountPatches(int width, int height) =>
89+
((long)(width + PatchSize - 1) / PatchSize) * ((height + PatchSize - 1) / PatchSize);
90+
91+
private static bool CanPreserveSourceBytes(string mediaType) =>
92+
string.Equals(mediaType, "image/png", StringComparison.OrdinalIgnoreCase) ||
93+
string.Equals(mediaType, "image/jpeg", StringComparison.OrdinalIgnoreCase) ||
94+
string.Equals(mediaType, "image/webp", StringComparison.OrdinalIgnoreCase);
95+
96+
private static bool IsSupportedProviderImageMediaType(string mediaType) =>
97+
CanPreserveSourceBytes(mediaType) ||
98+
string.Equals(mediaType, "image/gif", StringComparison.OrdinalIgnoreCase);
99+
100+
private static void SaveImage(Image image, Stream output, string mediaType)
101+
{
102+
switch (mediaType)
103+
{
104+
case "image/jpeg":
105+
image.SaveAsJpeg(output);
106+
break;
107+
case "image/webp":
108+
image.SaveAsWebp(output);
109+
break;
110+
default:
111+
image.SaveAsPng(output);
112+
break;
113+
}
114+
}
115+
116+
private static DataContent CopyMetadata(DataContent source, DataContent prepared)
117+
{
118+
if (source.AdditionalProperties is not { Count: > 0 } additionalProperties)
119+
return prepared;
120+
121+
prepared.AdditionalProperties ??= new AdditionalPropertiesDictionary();
122+
foreach (var (key, value) in additionalProperties)
123+
prepared.AdditionalProperties[key] = value;
124+
return prepared;
125+
}
126+
127+
private static bool IsImagePreparationException(Exception ex) =>
128+
ex is ArgumentException
129+
or InvalidImageContentException
130+
or ImageFormatException
131+
or NotSupportedException
132+
or UnknownImageFormatException;
133+
134+
private static string NormalizeMediaType(string? mediaType) =>
135+
string.IsNullOrWhiteSpace(mediaType)
136+
? "application/octet-stream"
137+
: mediaType.Trim().ToLowerInvariant();
138+
139+
internal sealed record PreparedModelImageInput(DataContent? Content, string? PlaceholderText)
140+
{
141+
public bool HasImage => Content != null;
142+
143+
public static PreparedModelImageInput Image(DataContent content) => new(content, null);
144+
145+
public static PreparedModelImageInput Placeholder(string text) => new(null, text);
146+
}
147+
}

‎src/DotCraft.Core/Agents/ResponsesToolSearchMapper.cs‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -513,24 +513,40 @@ private static JsonObject CreateTextContentPart(ChatRole role, string text)
513513

514514
private static JsonObject CreateContentPartOrPlaceholder(ChatRole role, AIContent content)
515515
{
516-
if (role == ChatRole.User && TryCreateUserImagePart(content, out var imagePart))
517-
return imagePart;
516+
if (role == ChatRole.User)
517+
{
518+
if (TryCreateUserImagePart(content, out var imagePart, out var placeholderText))
519+
return imagePart;
520+
if (!string.IsNullOrWhiteSpace(placeholderText))
521+
return CreateTextContentPart(role, placeholderText);
522+
}
518523

519524
return CreateTextContentPart(role, DescribeUnsupportedContent(content));
520525
}
521526

522-
private static bool TryCreateUserImagePart(AIContent content, out JsonObject part)
527+
private static bool TryCreateUserImagePart(
528+
AIContent content,
529+
out JsonObject part,
530+
out string? placeholderText)
523531
{
524532
part = null!;
533+
placeholderText = null;
525534
string? imageUri = null;
526535

527536
switch (content)
528537
{
529-
case DataContent data when IsImageMediaType(data.MediaType):
530-
imageUri = data.Uri?.ToString();
538+
case DataContent data when ModelImageInputPreparer.IsImageMediaType(data.MediaType):
539+
var prepared = ModelImageInputPreparer.Prepare(data);
540+
if (prepared.Content == null)
541+
{
542+
placeholderText = prepared.PlaceholderText;
543+
return false;
544+
}
545+
546+
imageUri = CreateDataUri(prepared.Content);
531547
break;
532548

533-
case UriContent uri when IsImageMediaType(uri.MediaType):
549+
case UriContent uri when ModelImageInputPreparer.IsSupportedRemoteImageMediaType(uri.MediaType):
534550
imageUri = uri.Uri?.ToString();
535551
break;
536552
}
@@ -546,6 +562,9 @@ private static bool TryCreateUserImagePart(AIContent content, out JsonObject par
546562
return true;
547563
}
548564

565+
private static string CreateDataUri(DataContent content) =>
566+
$"data:{NormalizeMediaType(content.MediaType)};base64,{Convert.ToBase64String(content.Data.ToArray())}";
567+
549568
private static JsonObject CreateFunctionCallItem(FunctionCallContent call)
550569
{
551570
if (string.Equals(call.Name, NativeToolSearchTool.ToolName, StringComparison.Ordinal))
@@ -821,9 +840,6 @@ private static string DescribeUnsupportedContent(AIContent content)
821840
: $"[Unsupported content: {typeName} ({mediaType})]";
822841
}
823842

824-
private static bool IsImageMediaType(string? mediaType) =>
825-
mediaType?.StartsWith("image/", StringComparison.OrdinalIgnoreCase) == true;
826-
827843
private static string NormalizeMediaType(string? mediaType) =>
828844
string.IsNullOrWhiteSpace(mediaType)
829845
? "application/octet-stream"

‎src/DotCraft.Core/DotCraft.Core.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@
2323
<DebugType>None</DebugType>
2424
<EnableCompressionInSingleFile>true</EnableCompressionInSingleFile>
2525
</PropertyGroup>
26-
26+
2727
<ItemGroup>
2828
<PackageReference Include="Alibaba.OpenSandbox" Version="0.1.1" />
2929
<PackageReference Include="Anthropic" Version="12.20.0" />
3030
<PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.4" />
31+
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
3132
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.3" />
3233
<PackageReference Include="Microsoft.Agents.AI" Version="1.6.1" />
3334
<PackageReference Include="Microsoft.Agents.AI.Hosting" Version="1.6.1-preview.260514.1" />

‎src/DotCraft.Core/Protocol/AppServer/Handlers/TurnRequestHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,7 @@ private static string InferMediaType(string pathOrUrl)
595595
".png" => "image/png",
596596
".gif" => "image/gif",
597597
".webp" => "image/webp",
598+
".bmp" => "image/bmp",
598599
_ => "image/jpeg"
599600
};
600601
}

‎tests/DotCraft.Core.Tests/Agents/ImageContentSanitizingChatClientTests.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
using System.Runtime.CompilerServices;
2+
using DotCraft.Agents;
3+
using Microsoft.Extensions.AI;
4+
using SixLabors.ImageSharp;
5+
using SixLabors.ImageSharp.PixelFormats;
6+
7+
namespace DotCraft.Tests.Agents;
8+
9+
public sealed class ImageContentSanitizingChatClientTests
10+
{
11+
[Fact]
12+
public async Task GetStreamingResponseAsync_ToolBmpResult_PromotesPreparedPngImage()
13+
{
14+
using var inner = new CapturingChatClient();
15+
using var client = new ImageContentSanitizingChatClient(inner);
16+
17+
_ = await CollectStreamingAsync(client.GetStreamingResponseAsync(CreateToolResultMessages(
18+
new DataContent(CreateBmpBytes(), "image/bmp"))));
19+
20+
var syntheticUser = Assert.Single(
21+
inner.LastMessages,
22+
message => message.Role == ChatRole.User &&
23+
message.Contents.OfType<DataContent>().Any());
24+
var image = Assert.Single(syntheticUser.Contents.OfType<DataContent>());
25+
Assert.Equal("image/png", image.MediaType);
26+
Assert.Equal("image/png", Image.DetectFormat(image.Data.ToArray()).DefaultMimeType);
27+
}
28+
29+
[Fact]
30+
public async Task GetStreamingResponseAsync_InvalidToolImage_AddsPlaceholderWithoutPromotingImage()
31+
{
32+
using var inner = new CapturingChatClient();
33+
using var client = new ImageContentSanitizingChatClient(inner);
34+
35+
_ = await CollectStreamingAsync(client.GetStreamingResponseAsync(CreateToolResultMessages(
36+
new DataContent(new byte[] { 1, 2, 3 }, "image/bmp"))));
37+
38+
Assert.DoesNotContain(
39+
inner.LastMessages,
40+
message => message.Role == ChatRole.User &&
41+
message.Contents.OfType<DataContent>().Any());
42+
var tool = Assert.Single(inner.LastMessages, message => message.Role == ChatRole.Tool);
43+
var result = Assert.Single(tool.Contents.OfType<FunctionResultContent>());
44+
var text = Assert.IsType<string>(result.Result);
45+
Assert.Contains(ModelImageInputPreparer.CouldNotProcessPlaceholder, text, StringComparison.Ordinal);
46+
}
47+
48+
private static IReadOnlyList<ChatMessage> CreateToolResultMessages(DataContent image) =>
49+
[
50+
new ChatMessage(ChatRole.User, "inspect"),
51+
new ChatMessage(ChatRole.Assistant, (IList<AIContent>)
52+
[
53+
new FunctionCallContent(
54+
"call-1",
55+
"ReadFile",
56+
new Dictionary<string, object?> { ["path"] = "text_object0.bmp" })
57+
]),
58+
new ChatMessage(ChatRole.Tool, (IList<AIContent>)
59+
[
60+
new FunctionResultContent(
61+
"call-1",
62+
(IList<AIContent>)
63+
[
64+
new TextContent("Image: text_object0.bmp"),
65+
image
66+
])
67+
])
68+
];
69+
70+
private static async Task<List<ChatResponseUpdate>> CollectStreamingAsync(
71+
IAsyncEnumerable<ChatResponseUpdate> streaming)
72+
{
73+
var updates = new List<ChatResponseUpdate>();
74+
await foreach (var update in streaming)
75+
updates.Add(update);
76+
return updates;
77+
}
78+
79+
private static byte[] CreateBmpBytes()
80+
{
81+
using var image = new Image<Rgba32>(1, 1, new Rgba32(0xff, 0, 0));
82+
using var stream = new MemoryStream();
83+
image.SaveAsBmp(stream);
84+
return stream.ToArray();
85+
}
86+
87+
private sealed class CapturingChatClient : IChatClient
88+
{
89+
public IReadOnlyList<ChatMessage> LastMessages { get; private set; } = [];
90+
91+
public Task<ChatResponse> GetResponseAsync(
92+
IEnumerable<ChatMessage> chatMessages,
93+
ChatOptions? options = null,
94+
CancellationToken cancellationToken = default)
95+
{
96+
LastMessages = chatMessages.ToList();
97+
return Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "ok")]));
98+
}
99+
100+
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
101+
IEnumerable<ChatMessage> chatMessages,
102+
ChatOptions? options = null,
103+
[EnumeratorCancellation] CancellationToken cancellationToken = default)
104+
{
105+
LastMessages = chatMessages.ToList();
106+
yield return new ChatResponseUpdate(ChatRole.Assistant, "ok");
107+
await Task.CompletedTask;
108+
}
109+
110+
public object? GetService(Type serviceType, object? serviceKey = null) =>
111+
serviceType.IsInstanceOfType(this) ? this : null;
112+
113+
public void Dispose()
114+
{
115+
}
116+
}
117+
}

0 commit comments

Comments
 (0)