Skip to content

Commit 4f636b1

Browse files
waldekmastykarzwaldekmastykarz
authored
Adds support for OAI Content Parts. Closes #1174 (#1175)
1 parent 3ce9cb1 commit 4f636b1

File tree

6 files changed

+157
-19
lines changed

6 files changed

+157
-19
lines changed

‎dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ namespace DevProxy.Abstractions.LanguageModel;
66

77
public interface ILanguageModelChatCompletionMessage
88
{
9-
string Content { get; set; }
9+
object Content { get; set; }
1010
string Role { get; set; }
1111
}

‎dev-proxy-abstractions/LanguageModel/OllamaModels.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public class OllamaLanguageModelChatCompletionResponse : OllamaResponse
8989
public OllamaLanguageModelChatCompletionMessage Message { get; set; } = new();
9090
public override string? Response
9191
{
92-
get => Message.Content;
92+
get => Message.Content.ToString();
9393
set
9494
{
9595
if (value is null)
@@ -118,7 +118,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()
118118
Index = 0,
119119
Message = new()
120120
{
121-
Content = Message.Content,
121+
Content = Message.Content.ToString() ?? string.Empty,
122122
Role = Message.Role
123123
}
124124
}],
@@ -152,7 +152,7 @@ public override OpenAIResponse ConvertToOpenAIResponse()
152152

153153
public class OllamaLanguageModelChatCompletionMessage : ILanguageModelChatCompletionMessage
154154
{
155-
public string Content { get; set; } = string.Empty;
155+
public object Content { get; set; } = string.Empty;
156156
public string Role { get; set; } = string.Empty;
157157

158158
public override bool Equals(object? obj)

‎dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
7+
8+
namespace DevProxy.Abstractions.LanguageModel;
9+
10+
public class OpenAIContentPartJsonConverter : JsonConverter<object>
11+
{
12+
public override bool CanConvert(Type typeToConvert)
13+
{
14+
return typeToConvert == typeof(object);
15+
}
16+
17+
public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
18+
{
19+
if (reader.TokenType == JsonTokenType.String)
20+
{
21+
return reader.GetString();
22+
}
23+
else if (reader.TokenType == JsonTokenType.StartArray)
24+
{
25+
var contentParts = new List<object>();
26+
while (reader.Read())
27+
{
28+
if (reader.TokenType == JsonTokenType.EndArray)
29+
{
30+
break;
31+
}
32+
33+
if (reader.TokenType == JsonTokenType.StartObject)
34+
{
35+
using JsonDocument doc = JsonDocument.ParseValue(ref reader);
36+
37+
var root = doc.RootElement;
38+
if (root.TryGetProperty("type", out var typeProp))
39+
{
40+
var contentType = typeProp.GetString() switch
41+
{
42+
"text" => typeof(OpenAITextContentPart),
43+
"image" => typeof(OpenAIImageContentPart),
44+
"audio" => typeof(OpenAIAudioContentPart),
45+
"file" => typeof(OpenAIFileContentPart),
46+
_ => null
47+
};
48+
if (contentType is not null)
49+
{
50+
var contentPart = JsonSerializer.Deserialize(doc.RootElement.GetRawText(), contentType, options);
51+
if (contentPart is not null)
52+
{
53+
contentParts.Add(contentPart);
54+
}
55+
}
56+
}
57+
}
58+
}
59+
return contentParts.ToArray();
60+
}
61+
return null;
62+
}
63+
64+
public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerOptions options)
65+
{
66+
if (value is string str)
67+
{
68+
writer.WriteStringValue(str);
69+
}
70+
else
71+
{
72+
JsonSerializer.Serialize(writer, value, options);
73+
}
74+
}
75+
}

‎dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using Microsoft.Extensions.Logging;
56
using System.Diagnostics;
67
using System.Net.Http.Json;
7-
using Microsoft.Extensions.Logging;
88

99
namespace DevProxy.Abstractions.LanguageModel;
1010

@@ -170,7 +170,9 @@ private async Task<bool> IsEnabledInternalAsync()
170170
Temperature = options?.Temperature
171171
};
172172

173-
var response = await _httpClient.PostAsJsonAsync(url, payload);
173+
// var payloadString = JsonSerializer.Serialize(payload, ProxyUtils.JsonSerializerOptions);
174+
175+
var response = await _httpClient.PostAsJsonAsync(url, payload, ProxyUtils.JsonSerializerOptions);
174176
_logger.LogDebug("Response: {response}", response.StatusCode);
175177

176178
if (!response.IsSuccessStatusCode)

‎dev-proxy-abstractions/LanguageModel/OpenAIModels.cs

+67-13
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class OpenAIError
3838
public string? Message { get; set; }
3939
}
4040

41-
public abstract class OpenAIResponse: ILanguageModelCompletionResponse
41+
public abstract class OpenAIResponse : ILanguageModelCompletionResponse
4242
{
4343
public long Created { get; set; }
4444
public OpenAIError? Error { get; set; }
@@ -106,9 +106,63 @@ public class OpenAICompletionResponseChoice : OpenAIResponseChoice
106106
public string Text { get; set; } = string.Empty;
107107
}
108108

109-
public class OpenAIChatCompletionMessage: ILanguageModelChatCompletionMessage
109+
#region content parts
110+
111+
public abstract class OpenAIContentPart
110112
{
111-
public string Content { get; set; } = string.Empty;
113+
public string? Type { get; set; }
114+
}
115+
116+
public class OpenAITextContentPart : OpenAIContentPart
117+
{
118+
public string? Text { get; set; }
119+
}
120+
121+
public class OpenAIImageContentPartUrl
122+
{
123+
public string? Detail { get; set; } = "auto";
124+
public string? Url { get; set; }
125+
}
126+
127+
public class OpenAIImageContentPart : OpenAIContentPart
128+
{
129+
[JsonPropertyName("image_url")]
130+
public OpenAIImageContentPartUrl? Url { get; set; }
131+
}
132+
133+
public class OpenAIAudioContentPartInputAudio
134+
{
135+
public string? Data { get; set; }
136+
public string? Format { get; set; }
137+
}
138+
139+
public class OpenAIAudioContentPart : OpenAIContentPart
140+
{
141+
[JsonPropertyName("input_audio")]
142+
public OpenAIAudioContentPartInputAudio? InputAudio { get; set; }
143+
}
144+
145+
public class OpenAIFileContentPartFile
146+
{
147+
[JsonPropertyName("file_data")]
148+
public string? Data { get; set; }
149+
[JsonPropertyName("file_id")]
150+
public string? Id { get; set; }
151+
[JsonPropertyName("filename")]
152+
public string? Name { get; set; }
153+
}
154+
155+
public class OpenAIFileContentPart : OpenAIContentPart
156+
{
157+
public OpenAIFileContentPartFile? File { get; set; }
158+
}
159+
160+
#endregion
161+
162+
public class OpenAIChatCompletionMessage : ILanguageModelChatCompletionMessage
163+
{
164+
[JsonConverter(typeof(OpenAIContentPartJsonConverter))]
165+
public object Content { get; set; } = string.Empty;
112166
public string Role { get; set; } = string.Empty;
113167

114168
public override bool Equals(object? obj)
@@ -204,13 +258,13 @@ public class OpenAIFineTuneRequest : OpenAIRequest
204258
}
205259

206260
public class OpenAIFineTuneResponse : OpenAIResponse
207-
{
261+
{
208262
[JsonPropertyName("fine_tuned_model")]
209-
public string? FineTunedModel { get; set; }
210-
public string Status { get; set; } = string.Empty;
211-
public string? Organization { get; set; }
212-
public long CreatedAt { get; set; }
213-
public long UpdatedAt { get; set; }
263+
public string? FineTunedModel { get; set; }
264+
public string Status { get; set; } = string.Empty;
265+
public string? Organization { get; set; }
266+
public long CreatedAt { get; set; }
267+
public long UpdatedAt { get; set; }
214268
[JsonPropertyName("training_file")]
215269
public string TrainingFile { get; set; } = string.Empty;
216270
[JsonPropertyName("validation_file")]
@@ -233,16 +287,16 @@ public class OpenAIImageRequest : OpenAIRequest
233287
}
234288

235289
public class OpenAIImageResponse : OpenAIResponse
236-
{
237-
public OpenAIImageData[]? Data { get; set; }
290+
{
291+
public OpenAIImageData[]? Data { get; set; }
238292
public override string? Response => null; // Image responses don't have a text response
239293
}
240294

241295
public class OpenAIImageData
242296
{
243-
public string? Url { get; set; }
297+
public string? Url { get; set; }
244298
[JsonPropertyName("b64_json")]
245-
public string? Base64Json { get; set; }
299+
public string? Base64Json { get; set; }
246300
[JsonPropertyName("revised_prompt")]
247301
public string? RevisedPrompt { get; set; }
248302
}

‎dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs

+7
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ public override async Task RegisterAsync()
3434

3535
private async Task OnRequestAsync(object sender, ProxyRequestArgs e)
3636
{
37+
if (UrlsToWatch is null ||
38+
!e.HasRequestUrlMatch(UrlsToWatch))
39+
{
40+
Logger.LogRequest("URL not matched", MessageType.Skipped, new LoggingContext(e.Session));
41+
return;
42+
}
43+
3744
var request = e.Session.HttpClient.Request;
3845
if (request.Method is null ||
3946
!request.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) ||

0 commit comments

Comments
 (0)