diff --git a/backend/Realtime/IRealtimeUpstream.cs b/backend/Realtime/IRealtimeUpstream.cs new file mode 100644 index 0000000..9a5b11f --- /dev/null +++ b/backend/Realtime/IRealtimeUpstream.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Nodes; + +namespace backend.Realtime; + +public interface IRealtimeUpstream : IAsyncDisposable +{ + Task SendJsonAsync(JsonObject envelope, CancellationToken ct); + Task ReceiveJsonAsync(CancellationToken ct); +} diff --git a/backend/Realtime/RealtimeEvents.cs b/backend/Realtime/RealtimeEvents.cs new file mode 100644 index 0000000..1c5bcad --- /dev/null +++ b/backend/Realtime/RealtimeEvents.cs @@ -0,0 +1,71 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using backend.Tools; + +namespace backend.Realtime; + +public static class RealtimeEvents +{ + public static JsonObject SessionUpdate(RealtimeSettings cfg, IEnumerable enabledTools) + { + var tools = new JsonArray(); + foreach (var t in enabledTools) + { + tools.Add(new JsonObject + { + ["type"] = "function", + ["name"] = t.Name, + ["description"] = t.Description, + ["parameters"] = JsonNode.Parse(t.ParameterSchema.GetRawText())!, + }); + } + + return new JsonObject + { + ["type"] = "session.update", + ["session"] = new JsonObject + { + ["voice"] = cfg.Voice, + ["instructions"] = cfg.SystemPrompt, + ["modalities"] = new JsonArray("text", "audio"), + ["input_audio_format"] = "pcm16", + ["output_audio_format"] = "pcm16", + ["input_audio_transcription"] = new JsonObject { ["model"] = "whisper-1" }, + ["turn_detection"] = new JsonObject + { + ["type"] = "server_vad", + ["threshold"] = 0.5, + ["prefix_padding_ms"] = 300, + ["silence_duration_ms"] = 500, + }, + ["tools"] = tools, + ["tool_choice"] = "auto", + }, + }; + } + + public static JsonObject InputAudioBufferAppend(ReadOnlySpan pcm16) + { + return new JsonObject + { + ["type"] = "input_audio_buffer.append", + ["audio"] = Convert.ToBase64String(pcm16), + }; + } + + public static JsonObject FunctionCallOutput(string callId, string outputJson) + { + return new JsonObject + { + ["type"] = "conversation.item.create", + ["item"] = new JsonObject + { + ["type"] = "function_call_output", + ["call_id"] = callId, + ["output"] = outputJson, + }, + }; + } + + public static JsonObject ResponseCreate() => new() { ["type"] = "response.create" }; +}