using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using OpenMono.Config;
using OpenMono.Session;
using OpenMono.Utils;
namespace OpenMono.Llm;
public sealed class OpenAiCompatClient : ILlmClient, IDisposable
{
private readonly HttpClient _http;
private readonly string _endpoint;
private const int MaxRetries = 3;
private static readonly TimeSpan[] RetryDelays =
[
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(4),
TimeSpan.FromSeconds(16),
];
private static readonly Regex QwenFunctionRegex = new(
@"(.*?)",
RegexOptions.Singleline | RegexOptions.Compiled);
private static readonly Regex QwenParamRegex = new(
@"\s*(.*?)\s*",
RegexOptions.Singleline | RegexOptions.Compiled);
public string? ApiKey { get; init; }
public Action? OnDebug { get; set; }
private readonly string _model;
public OpenAiCompatClient(LlmConfig config)
{
_endpoint = config.Endpoint.TrimEnd('/');
_model = config.Model;
_http = new HttpClient { Timeout = TimeSpan.FromMinutes(10) };
}
public async IAsyncEnumerable StreamChatAsync(
IReadOnlyList messages,
JsonElement? tools,
LlmOptions options,
[EnumeratorCancellation] CancellationToken ct)
{
HttpResponseMessage? response = null;
var lastException = default(Exception);
for (var attempt = 0; attempt <= MaxRetries; attempt++)
{
ct.ThrowIfCancellationRequested();
if (attempt > 0)
{
var delay = RetryDelays[Math.Min(attempt - 1, RetryDelays.Length - 1)];
await Task.Delay(delay, ct);
}
var requestBody = BuildRequestBody(messages, tools, options, _model);
var requestJson = JsonSerializer.Serialize(requestBody, JsonOptions.Default);
if (attempt == 0)
{
var toolCount = tools?.ValueKind == JsonValueKind.Array ? tools.Value.GetArrayLength() : 0;
OnDebug?.Invoke($"[LLM] POST {_endpoint}/v1/chat/completions");
var resolvedModel = string.IsNullOrEmpty(options.Model) ? _model : options.Model;
OnDebug?.Invoke($"[LLM] Model: {resolvedModel} | Messages: {messages.Count} | Tools: {toolCount} | MaxTokens: {options.MaxTokens}");
Log.Debug($"LLM request: model={resolvedModel} messages={messages.Count} tools={toolCount} endpoint={_endpoint}");
}
else
{
var delay = RetryDelays[Math.Min(attempt - 1, RetryDelays.Length - 1)];
OnDebug?.Invoke($"[LLM] Retry {attempt}/{MaxRetries} after {delay.TotalSeconds}s");
Log.Warn($"LLM retry {attempt}/{MaxRetries} after {delay.TotalSeconds}s");
}
var content = new StringContent(requestJson, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, $"{_endpoint}/v1/chat/completions")
{
Content = content,
};
if (ApiKey is not null)
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", ApiKey);
try
{
response = await _http.SendAsync(
request, HttpCompletionOption.ResponseHeadersRead, ct);
if (IsRetryableStatus(response.StatusCode))
{
lastException = new HttpRequestException(
$"HTTP {(int)response.StatusCode} {response.StatusCode}",
inner: null, response.StatusCode);
response.Dispose();
response = null;
continue;
}
response.EnsureSuccessStatusCode();
break;
}
catch (HttpRequestException ex) when (attempt < MaxRetries)
{
lastException = ex;
response?.Dispose();
response = null;
}
catch (TaskCanceledException ex) when (!ct.IsCancellationRequested && attempt < MaxRetries)
{
lastException = ex;
response?.Dispose();
response = null;
}
}
if (response is null)
throw lastException ?? new HttpRequestException("Failed to connect after retries");
var streamStarted = System.Diagnostics.Stopwatch.StartNew();
var chunkCount = 0;
using (response)
{
using var stream = await response.Content.ReadAsStreamAsync(ct);
using var reader = new StreamReader(stream);
var toolCalls = new Dictionary();
var malformedChunks = 0;
var fullText = new StringBuilder();
var suppressText = false;
string? line;
while ((line = await reader.ReadLineAsync(ct)) is not null)
{
ct.ThrowIfCancellationRequested();
if (string.IsNullOrEmpty(line)) continue;
if (!line.StartsWith("data: ", StringComparison.Ordinal)) continue;
var data = line["data: ".Length..];
if (data == "[DONE]")
{
foreach (var tc in toolCalls.Values.Where(t => t.IsComplete))
{
OnDebug?.Invoke($"[SSE] tool_call: {tc.Name} {{ {tc.Arguments.ToString()[..Math.Min(100, tc.Arguments.Length)]} }}");
Log.Debug($"SSE tool_call: {tc.Name} args={tc.Arguments.ToString()[..Math.Min(200, tc.Arguments.Length)]}");
yield return new StreamChunk
{
ToolCallDelta = new ToolCall
{
Id = tc.Id,
Name = tc.Name,
Arguments = tc.Arguments.ToString(),
}
};
}
var elapsed = streamStarted.Elapsed;
OnDebug?.Invoke($"[LLM] Stream complete — {chunkCount} chunks in {elapsed.TotalSeconds:F1}s");
Log.Debug($"LLM stream complete: chunks={chunkCount} elapsed={elapsed.TotalSeconds:F1}s");
if (toolCalls.Count == 0 && suppressText)
{
var idx = 0;
foreach (Match m in QwenFunctionRegex.Matches(fullText.ToString()))
{
var name = m.Groups[1].Value;
var body = m.Groups[2].Value;
var args = new Dictionary();
foreach (Match p in QwenParamRegex.Matches(body))
args[p.Groups[1].Value] = p.Groups[2].Value.Trim();
yield return new StreamChunk
{
ToolCallDelta = new ToolCall
{
Id = $"call_xml_{idx++}",
Name = name,
Arguments = JsonSerializer.Serialize(args),
}
};
}
}
yield return new StreamChunk { IsComplete = true };
yield break;
}
JsonDocument? doc;
try
{
doc = JsonDocument.Parse(data);
}
catch (JsonException)
{
malformedChunks++;
if (malformedChunks > 50)
throw new InvalidOperationException(
$"Too many malformed SSE chunks ({malformedChunks}). Stream is corrupt.");
continue;
}
using (doc)
{
var root = doc.RootElement;
if (root.TryGetProperty("error", out var errorEl))
{
var errorMsg = errorEl.TryGetProperty("message", out var msgEl)
? msgEl.GetString() : "Unknown API error";
throw new HttpRequestException($"LLM API error: {errorMsg}");
}
chunkCount++;
UsageInfo? usage = null;
if (root.TryGetProperty("usage", out var usageEl) &&
usageEl.ValueKind == JsonValueKind.Object)
{
usage = new UsageInfo
{
PromptTokens = usageEl.TryGetProperty("prompt_tokens", out var pt) ? pt.GetInt32() : 0,
CompletionTokens = usageEl.TryGetProperty("completion_tokens", out var cpt) ? cpt.GetInt32() : 0,
};
var usageMsg = $"[SSE] usage: prompt={usage.PromptTokens} completion={usage.CompletionTokens} total={usage.TotalTokens}";
OnDebug?.Invoke(usageMsg);
Log.Info(usageMsg);
}
if (!root.TryGetProperty("choices", out var choices)) continue;
foreach (var choice in choices.EnumerateArray())
{
if (!choice.TryGetProperty("delta", out var delta)) continue;
if (delta.TryGetProperty("reasoning_content", out var reasoningEl) &&
reasoningEl.ValueKind == JsonValueKind.String)
{
var thinking = reasoningEl.GetString();
if (!string.IsNullOrEmpty(thinking))
yield return new StreamChunk { ThinkingDelta = thinking };
}
if (delta.TryGetProperty("content", out var contentEl) &&
contentEl.ValueKind == JsonValueKind.String)
{
var text = contentEl.GetString();
if (!string.IsNullOrEmpty(text))
{
fullText.Append(text);
if (!suppressText)
{
if (fullText.ToString().Contains("
status is System.Net.HttpStatusCode.TooManyRequests
or System.Net.HttpStatusCode.InternalServerError
or System.Net.HttpStatusCode.BadGateway
or System.Net.HttpStatusCode.ServiceUnavailable
or System.Net.HttpStatusCode.GatewayTimeout;
private static object BuildRequestBody(
IReadOnlyList messages,
JsonElement? tools,
LlmOptions options,
string configModel)
{
var model = string.IsNullOrEmpty(options.Model) ? configModel : options.Model;
var apiMessages = messages.Select(m => m.Role switch
{
MessageRole.System => new { role = "system", content = m.Content },
MessageRole.User when m.ContentParts is { Count: > 0 } =>
new { role = "user", content = (object)m.ContentParts.Select(p => p switch {
TextPart t => (object)new { type = "text", text = t.Text },
ImagePart i => new { type = "image_url", image_url = new { url = i.Url } },
_ => new { type = "text", text = "" }
}).ToList() },
MessageRole.User => new { role = "user", content = m.Content },
MessageRole.Assistant when m.ToolCalls is { Count: > 0 } =>
new
{
role = "assistant",
content = m.Content ?? (object)"",
tool_calls = m.ToolCalls.Select(tc => new
{
id = tc.Id,
type = "function",
function = new { name = tc.Name, arguments = tc.Arguments }
})
},
MessageRole.Assistant => new { role = "assistant", content = m.Content },
MessageRole.Tool => (object)new
{
role = "tool",
tool_call_id = m.ToolCallId,
content = m.Content,
},
_ => new { role = "user", content = m.Content },
}).ToList();
var body = new Dictionary
{
["model"] = model,
["messages"] = apiMessages,
["temperature"] = options.Temperature,
["max_tokens"] = options.MaxTokens,
["top_p"] = options.TopP,
["top_k"] = options.TopK,
["presence_penalty"] = options.PresencePenalty,
["min_p"] = options.MinP,
["repetition_penalty"] = options.RepetitionPenalty,
["stream"] = true,
["stream_options"] = new { include_usage = true },
};
if (options.EnableThinking.HasValue)
body["chat_template_kwargs"] = new { enable_thinking = options.EnableThinking.Value };
if (tools.HasValue && tools.Value.ValueKind == JsonValueKind.Array &&
tools.Value.GetArrayLength() > 0)
{
body["tools"] = tools.Value;
body["tool_choice"] = "auto";
}
return body;
}
public void Dispose() => _http.Dispose();
private sealed class ToolCallAccumulator
{
public string Id { get; set; } = "";
public string Name { get; set; } = "";
public StringBuilder Arguments { get; } = new();
public bool IsComplete { get; set; }
}
}