Assistant/backend/Realtime/RealtimeEvents.cs

using System.Text.Json;
using System.Text.Json.Nodes;
using backend.Tools;

namespace backend.Realtime;

public static class RealtimeEvents
{
    // GA Realtime API shape (the Beta API shape was retired in 2026 with
    // upstream error "beta_api_shape_disabled"). Key differences:
    //   - session.type = "realtime" is required.
    //   - voice / audio formats / transcription / turn_detection now nested
    //     under session.audio.{input,output}.
    //   - No OpenAI-Beta header on the upgrade (see OpenAIRealtimeUpstream).
    public static JsonObject SessionUpdate(RealtimeSettings cfg, IEnumerable<ITool> enabledTools)
    {
        var tools = new JsonArray();
        foreach (var t in enabledTools)
        {
            tools.Add(new JsonObject
            {
                ["type"] = "function",
                ["name"] = t.Name,
                ["description"] = t.Description,
                ["parameters"] = JsonNode.Parse(t.ParameterSchema.GetRawText())!,
            });
        }

        return new JsonObject
        {
            ["type"] = "session.update",
            ["session"] = new JsonObject
            {
                ["type"] = "realtime",
                ["model"] = cfg.Model,
                ["instructions"] = cfg.SystemPrompt,
                ["audio"] = new JsonObject
                {
                    ["input"] = new JsonObject
                    {
                        ["format"] = new JsonObject
                        {
                            ["type"] = "audio/pcm",
                            ["rate"] = 24000,
                        },
                        ["transcription"] = new JsonObject { ["model"] = "whisper-1" },
                        ["turn_detection"] = new JsonObject
                        {
                            ["type"] = "server_vad",
                            ["threshold"] = 0.5,
                            ["prefix_padding_ms"] = 300,
                            ["silence_duration_ms"] = 500,
                            // GA requires both explicitly — without them the
                            // model never auto-generates a reply after VAD
                            // detects speech_stopped.
                            ["create_response"] = true,
                            ["interrupt_response"] = true,
                        },
                    },
                    ["output"] = new JsonObject
                    {
                        ["voice"] = cfg.Voice,
                        ["format"] = new JsonObject
                        {
                            ["type"] = "audio/pcm",
                            ["rate"] = 24000,
                        },
                    },
                },
                ["tools"] = tools,
                ["tool_choice"] = "auto",
            },
        };
    }

    public static JsonObject InputAudioBufferAppend(ReadOnlySpan<byte> pcm16)
    {
        return new JsonObject
        {
            ["type"] = "input_audio_buffer.append",
            ["audio"] = Convert.ToBase64String(pcm16),
        };
    }

    public static JsonObject FunctionCallOutput(string callId, string outputJson)
    {
        return new JsonObject
        {
            ["type"] = "conversation.item.create",
            ["item"] = new JsonObject
            {
                ["type"] = "function_call_output",
                ["call_id"] = callId,
                ["output"] = outputJson,
            },
        };
    }

    public static JsonObject ResponseCreate() => new() { ["type"] = "response.create" };
}