tes
12be767325
The previous run showed session_started, then total silence from the relay for the entire conversation — no events forwarded, no errors, nothing. Two changes to disambiguate: 1. Drop the explicit audio.input.format / audio.output.format objects. 24 kHz PCM16 mono is the documented GA default for both directions and the only thing OpenAI ever advertised; an unrecognised value inside the format object is the most likely reason OpenAI silently ignores incoming audio. 2. Forward 'upstream silent NxN250ms' debug envelopes every ~2 s when ReceiveJsonAsync keeps returning null, and add a 'server_version' field to hello_ack so the Pi journal proves which build is running.
93 lines
3.3 KiB
C#
93 lines
3.3 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using backend.Tools;
|
|
|
|
namespace backend.Realtime;
|
|
|
|
public static class RealtimeEvents
|
|
{
|
|
// GA Realtime API shape (the Beta API shape was retired in 2026 with
|
|
// upstream error "beta_api_shape_disabled"). Key differences:
|
|
// - session.type = "realtime" is required.
|
|
// - voice / audio formats / transcription / turn_detection now nested
|
|
// under session.audio.{input,output}.
|
|
// - No OpenAI-Beta header on the upgrade (see OpenAIRealtimeUpstream).
|
|
public static JsonObject SessionUpdate(RealtimeSettings cfg, IEnumerable<ITool> enabledTools)
|
|
{
|
|
var tools = new JsonArray();
|
|
foreach (var t in enabledTools)
|
|
{
|
|
tools.Add(new JsonObject
|
|
{
|
|
["type"] = "function",
|
|
["name"] = t.Name,
|
|
["description"] = t.Description,
|
|
["parameters"] = JsonNode.Parse(t.ParameterSchema.GetRawText())!,
|
|
});
|
|
}
|
|
|
|
// Minimal GA session.update. We drop explicit audio.input/output.format
|
|
// objects (the GA shape accepts them but OpenAI defaults to 24 kHz PCM16
|
|
// mono on both directions, which is what we use). Removing the explicit
|
|
// format objects matches the documented minimal session config and
|
|
// eliminates a known source of OpenAI silently ignoring the audio.
|
|
return new JsonObject
|
|
{
|
|
["type"] = "session.update",
|
|
["session"] = new JsonObject
|
|
{
|
|
["type"] = "realtime",
|
|
["model"] = cfg.Model,
|
|
["instructions"] = cfg.SystemPrompt,
|
|
["audio"] = new JsonObject
|
|
{
|
|
["input"] = new JsonObject
|
|
{
|
|
["transcription"] = new JsonObject { ["model"] = "whisper-1" },
|
|
["turn_detection"] = new JsonObject
|
|
{
|
|
["type"] = "server_vad",
|
|
["threshold"] = 0.5,
|
|
["prefix_padding_ms"] = 300,
|
|
["silence_duration_ms"] = 500,
|
|
["create_response"] = true,
|
|
["interrupt_response"] = true,
|
|
},
|
|
},
|
|
["output"] = new JsonObject
|
|
{
|
|
["voice"] = cfg.Voice,
|
|
},
|
|
},
|
|
["tools"] = tools,
|
|
["tool_choice"] = "auto",
|
|
},
|
|
};
|
|
}
|
|
|
|
public static JsonObject InputAudioBufferAppend(ReadOnlySpan<byte> pcm16)
|
|
{
|
|
return new JsonObject
|
|
{
|
|
["type"] = "input_audio_buffer.append",
|
|
["audio"] = Convert.ToBase64String(pcm16),
|
|
};
|
|
}
|
|
|
|
public static JsonObject FunctionCallOutput(string callId, string outputJson)
|
|
{
|
|
return new JsonObject
|
|
{
|
|
["type"] = "conversation.item.create",
|
|
["item"] = new JsonObject
|
|
{
|
|
["type"] = "function_call_output",
|
|
["call_id"] = callId,
|
|
["output"] = outputJson,
|
|
},
|
|
};
|
|
}
|
|
|
|
public static JsonObject ResponseCreate() => new() { ["type"] = "response.create" };
|
|
}
|