Diagnostics: surface upstream WS close reason + break the null-spin loop

The Pi journal showed 170 000 'upstream silent NxN' messages within one
second — i.e. ReceiveJsonAsync was returning null in zero time, so the
upstream WS to OpenAI was closed. The relay was spinning until idle
watchdog (30 s) fired. Add IRealtimeUpstream.IsClosed + CloseReason,
populated by OpenAIRealtimeUpstream from the close status / WS exception,
and have PumpAsync stop on close and forward
  error code=upstream_closed message=close status=<X> desc=<Y>
to the device. Closes the spin AND gives us the OpenAI close reason.
This commit is contained in:
2026-06-12 07:17:04 +00:00
parent 12be767325
commit 1a36c8096d
5 changed files with 50 additions and 8 deletions
+4
View File
@@ -6,4 +6,8 @@ public interface IRealtimeUpstream : IAsyncDisposable
{
Task SendJsonAsync(JsonObject envelope, CancellationToken ct);
Task<JsonObject?> ReceiveJsonAsync(CancellationToken ct);
// Once a receive returns null because the WebSocket transitioned out of
// the Open state, these surface why. Both stay null until that happens.
bool IsClosed { get; }
string? CloseReason { get; }
}
+24 -4
View File
@@ -7,6 +7,10 @@ namespace backend.Realtime;
public class OpenAIRealtimeUpstream : IRealtimeUpstream
{
private readonly ClientWebSocket _ws = new();
private string? _closeReason;
public bool IsClosed => _ws.State != WebSocketState.Open && _ws.State != WebSocketState.Connecting;
public string? CloseReason => _closeReason;
public async Task ConnectAsync(string apiKey, string model, CancellationToken ct)
{
@@ -31,11 +35,27 @@ public class OpenAIRealtimeUpstream : IRealtimeUpstream
while (true)
{
WebSocketReceiveResult result;
try { result = await _ws.ReceiveAsync(buffer, ct); }
catch (WebSocketException) { return null; }
catch (OperationCanceledException) { return null; }
try
{
result = await _ws.ReceiveAsync(buffer, ct);
}
catch (WebSocketException exc)
{
_closeReason ??= $"WebSocketException:{exc.WebSocketErrorCode}:{exc.Message}";
return null;
}
catch (OperationCanceledException)
{
return null;
}
if (result.MessageType == WebSocketMessageType.Close) return null;
if (result.MessageType == WebSocketMessageType.Close)
{
var status = _ws.CloseStatus?.ToString() ?? "<none>";
var desc = _ws.CloseStatusDescription ?? "<no description>";
_closeReason ??= $"close status={status} desc={desc}";
return null;
}
stream.Write(buffer, 0, result.Count);
if (result.EndOfMessage)
{
+19 -3
View File
@@ -128,9 +128,25 @@ public class RealtimeSession
if (evt is null && ct.IsCancellationRequested) return;
if (evt is null)
{
// Upstream has gone silent. Surface that to the device every
// ~2s so we can tell "OpenAI is up but emitting nothing" from
// "the upstream WS was closed" — Coolify logs aren't reachable.
// If the upstream WS is closed, stop spinning — surface the
// close reason to the device and bail with EndReason=Error.
if (_upstream.IsClosed)
{
var reason = _upstream.CloseReason ?? "upstream closed";
_logger.LogWarning("upstream closed mid-session: {Reason}", reason);
await _output.WriteEnvelopeAsync(new JsonObject
{
["type"] = "error",
["code"] = "upstream_closed",
["message"] = reason,
["fatal"] = false,
}, ct);
_endReason = EndReason.Error;
return;
}
// Surface "alive but silent" every ~2s so we can tell that
// from "the upstream WS was closed". Coolify logs aren't
// reachable from the dev sandbox.
if (++nullPolls % 8 == 0)
{
_logger.LogInformation("upstream silent ({Polls} x 250ms polls)", nullPolls);