Diagnostics: surface upstream WS close reason + break the null-spin loop
The Pi journal showed 170 000 'upstream silent NxN' messages within one second — i.e. ReceiveJsonAsync was returning null in zero time, so the upstream WS to OpenAI was closed. The relay was spinning until idle watchdog (30 s) fired. Add IRealtimeUpstream.IsClosed + CloseReason, populated by OpenAIRealtimeUpstream from the close status / WS exception, and have PumpAsync stop on close and forward error code=upstream_closed message=close status=<X> desc=<Y> to the device. Closes the spin AND gives us the OpenAI close reason.
This commit is contained in:
@@ -13,6 +13,8 @@ public class ScriptedRealtimeUpstream : IRealtimeUpstream
|
|||||||
Channel.CreateUnbounded<JsonObject>(new UnboundedChannelOptions { SingleWriter = true });
|
Channel.CreateUnbounded<JsonObject>(new UnboundedChannelOptions { SingleWriter = true });
|
||||||
|
|
||||||
public IList<JsonObject> Sent { get; } = new List<JsonObject>();
|
public IList<JsonObject> Sent { get; } = new List<JsonObject>();
|
||||||
|
public bool IsClosed => _toRelay.Reader.Completion.IsCompleted;
|
||||||
|
public string? CloseReason => IsClosed ? "scripted upstream closed" : null;
|
||||||
|
|
||||||
public Task SendJsonAsync(JsonObject envelope, CancellationToken ct)
|
public Task SendJsonAsync(JsonObject envelope, CancellationToken ct)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ public static class DeviceHubEndpoint
|
|||||||
|
|
||||||
// Bump on every backend change so we can verify a Coolify deploy landed
|
// Bump on every backend change so we can verify a Coolify deploy landed
|
||||||
// by looking at the hello_ack config.server_version field in the Pi journal.
|
// by looking at the hello_ack config.server_version field in the Pi journal.
|
||||||
public const string ServerVersion = "plan3-debug-2026-06-12-09";
|
public const string ServerVersion = "plan3-debug-2026-06-12-09-closeprobe";
|
||||||
|
|
||||||
private static async Task SendHelloAckAsync(ActiveDevice active, Device device, CancellationToken ct)
|
private static async Task SendHelloAckAsync(ActiveDevice active, Device device, CancellationToken ct)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -6,4 +6,8 @@ public interface IRealtimeUpstream : IAsyncDisposable
|
|||||||
{
|
{
|
||||||
Task SendJsonAsync(JsonObject envelope, CancellationToken ct);
|
Task SendJsonAsync(JsonObject envelope, CancellationToken ct);
|
||||||
Task<JsonObject?> ReceiveJsonAsync(CancellationToken ct);
|
Task<JsonObject?> ReceiveJsonAsync(CancellationToken ct);
|
||||||
|
// Once a receive returns null because the WebSocket transitioned out of
|
||||||
|
// the Open state, these surface why. Both stay null until that happens.
|
||||||
|
bool IsClosed { get; }
|
||||||
|
string? CloseReason { get; }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ namespace backend.Realtime;
|
|||||||
public class OpenAIRealtimeUpstream : IRealtimeUpstream
|
public class OpenAIRealtimeUpstream : IRealtimeUpstream
|
||||||
{
|
{
|
||||||
private readonly ClientWebSocket _ws = new();
|
private readonly ClientWebSocket _ws = new();
|
||||||
|
private string? _closeReason;
|
||||||
|
|
||||||
|
public bool IsClosed => _ws.State != WebSocketState.Open && _ws.State != WebSocketState.Connecting;
|
||||||
|
public string? CloseReason => _closeReason;
|
||||||
|
|
||||||
public async Task ConnectAsync(string apiKey, string model, CancellationToken ct)
|
public async Task ConnectAsync(string apiKey, string model, CancellationToken ct)
|
||||||
{
|
{
|
||||||
@@ -31,11 +35,27 @@ public class OpenAIRealtimeUpstream : IRealtimeUpstream
|
|||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
WebSocketReceiveResult result;
|
WebSocketReceiveResult result;
|
||||||
try { result = await _ws.ReceiveAsync(buffer, ct); }
|
try
|
||||||
catch (WebSocketException) { return null; }
|
{
|
||||||
catch (OperationCanceledException) { return null; }
|
result = await _ws.ReceiveAsync(buffer, ct);
|
||||||
|
}
|
||||||
|
catch (WebSocketException exc)
|
||||||
|
{
|
||||||
|
_closeReason ??= $"WebSocketException:{exc.WebSocketErrorCode}:{exc.Message}";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
if (result.MessageType == WebSocketMessageType.Close) return null;
|
if (result.MessageType == WebSocketMessageType.Close)
|
||||||
|
{
|
||||||
|
var status = _ws.CloseStatus?.ToString() ?? "<none>";
|
||||||
|
var desc = _ws.CloseStatusDescription ?? "<no description>";
|
||||||
|
_closeReason ??= $"close status={status} desc={desc}";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
stream.Write(buffer, 0, result.Count);
|
stream.Write(buffer, 0, result.Count);
|
||||||
if (result.EndOfMessage)
|
if (result.EndOfMessage)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -128,9 +128,25 @@ public class RealtimeSession
|
|||||||
if (evt is null && ct.IsCancellationRequested) return;
|
if (evt is null && ct.IsCancellationRequested) return;
|
||||||
if (evt is null)
|
if (evt is null)
|
||||||
{
|
{
|
||||||
// Upstream has gone silent. Surface that to the device every
|
// If the upstream WS is closed, stop spinning — surface the
|
||||||
// ~2s so we can tell "OpenAI is up but emitting nothing" from
|
// close reason to the device and bail with EndReason=Error.
|
||||||
// "the upstream WS was closed" — Coolify logs aren't reachable.
|
if (_upstream.IsClosed)
|
||||||
|
{
|
||||||
|
var reason = _upstream.CloseReason ?? "upstream closed";
|
||||||
|
_logger.LogWarning("upstream closed mid-session: {Reason}", reason);
|
||||||
|
await _output.WriteEnvelopeAsync(new JsonObject
|
||||||
|
{
|
||||||
|
["type"] = "error",
|
||||||
|
["code"] = "upstream_closed",
|
||||||
|
["message"] = reason,
|
||||||
|
["fatal"] = false,
|
||||||
|
}, ct);
|
||||||
|
_endReason = EndReason.Error;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Surface "alive but silent" every ~2s so we can tell that
|
||||||
|
// from "the upstream WS was closed". Coolify logs aren't
|
||||||
|
// reachable from the dev sandbox.
|
||||||
if (++nullPolls % 8 == 0)
|
if (++nullPolls % 8 == 0)
|
||||||
{
|
{
|
||||||
_logger.LogInformation("upstream silent ({Polls} x 250ms polls)", nullPolls);
|
_logger.LogInformation("upstream silent ({Polls} x 250ms polls)", nullPolls);
|
||||||
|
|||||||
Reference in New Issue
Block a user