RealtimeSession: uplink pump + downlink audio.delta forwarding

2026-06-11 21:33:37 +00:00
parent a202725c39
commit 24b8acdf60
2 changed files with 122 additions and 0 deletions
@@ -0,0 +1,81 @@
+using System.Text.Json.Nodes;
+using backend.Conversations;
+using backend.Realtime;
+using backend.Tools;
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace backend.tests;
+
+public class RealtimeSessionAudioTests(ApiFactory factory) : IClassFixture<ApiFactory>
+{
+    private readonly ApiFactory _factory = factory;
+
+    [Fact]
+    public async Task Uplink_bytes_become_input_audio_buffer_append()
+    {
+        using var scope = _factory.Services.CreateScope();
+        var log = scope.ServiceProvider.GetRequiredService<ConversationLog>();
+        var reg = new ToolRegistry(Array.Empty<ITool>());
+        var upstream = new ScriptedRealtimeUpstream();
+        var sink = new RecordingSink();
+        var cfg = new RealtimeSettings("m", "v", "p", 30, new HashSet<string>());
+        var session = new RealtimeSession(Guid.NewGuid(), upstream, sink, log, reg, cfg,
+            new FakeDeviceChannel(), NullLogger.Instance);
+
+        upstream.Push(new JsonObject { ["type"] = "session.updated" });
+        var cts = new CancellationTokenSource();
+        var run = session.RunAsync(cts.Token);
+        await sink.WaitForAsync("session_started", TimeSpan.FromSeconds(2));
+
+        var frame = new byte[3840];
+        for (int i = 0; i < frame.Length; i++) frame[i] = (byte)(i % 256);
+        await session.WriteUplinkAsync(frame, default);
+
+        await Task.Delay(100);
+
+        var appended = upstream.Sent
+            .FirstOrDefault(e => (string?)e["type"] == "input_audio_buffer.append");
+        appended.Should().NotBeNull();
+        var b64 = (string?)appended!["audio"];
+        Convert.FromBase64String(b64!).Length.Should().Be(3840);
+
+        cts.Cancel();
+        await run;
+    }
+
+    [Fact]
+    public async Task Audio_delta_from_upstream_becomes_binary_to_device()
+    {
+        using var scope = _factory.Services.CreateScope();
+        var log = scope.ServiceProvider.GetRequiredService<ConversationLog>();
+        var reg = new ToolRegistry(Array.Empty<ITool>());
+        var upstream = new ScriptedRealtimeUpstream();
+        var sink = new RecordingSink();
+        var cfg = new RealtimeSettings("m", "v", "p", 30, new HashSet<string>());
+        var session = new RealtimeSession(Guid.NewGuid(), upstream, sink, log, reg, cfg,
+            new FakeDeviceChannel(), NullLogger.Instance);
+
+        upstream.Push(new JsonObject { ["type"] = "session.updated" });
+        var cts = new CancellationTokenSource();
+        var run = session.RunAsync(cts.Token);
+        await sink.WaitForAsync("session_started", TimeSpan.FromSeconds(2));
+
+        var payload = new byte[3840];
+        for (int i = 0; i < payload.Length; i++) payload[i] = (byte)((i + 1) % 256);
+        upstream.Push(new JsonObject
+        {
+            ["type"] = "response.audio.delta",
+            ["delta"] = Convert.ToBase64String(payload),
+        });
+
+        await Task.Delay(100);
+        sink.Binary.Should().NotBeEmpty();
+        sink.Binary[0].Length.Should().Be(3840);
+        sink.Binary[0][0].Should().Be(1);
+
+        cts.Cancel();
+        await run;
+    }
+}