Compare commits
40 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cd07b52f8f | |||
| 4cfb0d4b9a | |||
| ac8a3f012c | |||
| a590c96d07 | |||
| a1a3b5c981 | |||
| 43b3a1a0f1 | |||
| e96b6ffd95 | |||
| dea126f994 | |||
| cbb47d1465 | |||
| a7d408e903 | |||
| 1d9a417cfc | |||
| 070cc0767a | |||
| 5a498e9c91 | |||
| 222d58baeb | |||
| 6fa2779ea1 | |||
| 14690484de | |||
| e9730c7e50 | |||
| b6d9f9a1cf | |||
| 7d808e6ade | |||
| 9b63e298d3 | |||
| 14e862f50a | |||
| 2e830eaed4 | |||
| b43a1ce17f | |||
| e18aed53d7 | |||
| bea35c8f1c | |||
| 849ad47230 | |||
| fe15a009b3 | |||
| 815389ecc2 | |||
| bfd8a05734 | |||
| 8fdbedd28f | |||
| 7c4621b3b7 | |||
| 87a85db38d | |||
| e2bd845abe | |||
| 1d08f49204 | |||
| fec222b6dc | |||
| 9e8cee7a16 | |||
| d744a6318f | |||
| 7b1417e885 | |||
| 91a1f04b44 | |||
| b7fda48e91 |
@@ -1,13 +0,0 @@
|
|||||||
**/bin
|
|
||||||
**/obj
|
|
||||||
**/.vs
|
|
||||||
.git
|
|
||||||
.gitignore
|
|
||||||
.vscode
|
|
||||||
.idea
|
|
||||||
docs
|
|
||||||
tests
|
|
||||||
backend.tests
|
|
||||||
*.md
|
|
||||||
.dockerignore
|
|
||||||
Dockerfile
|
|
||||||
-12
@@ -3,15 +3,3 @@ __pycache__/
|
|||||||
.venv/
|
.venv/
|
||||||
*.wav
|
*.wav
|
||||||
.openwakeword-cache/
|
.openwakeword-cache/
|
||||||
|
|
||||||
# .NET build outputs (don't catch root-level bin/ scripts)
|
|
||||||
backend/bin/
|
|
||||||
backend/obj/
|
|
||||||
backend.tests/bin/
|
|
||||||
backend.tests/obj/
|
|
||||||
|
|
||||||
# SQLite dev database (created on first run)
|
|
||||||
backend/*.db
|
|
||||||
backend/*.db-shm
|
|
||||||
backend/*.db-wal
|
|
||||||
deploy.json
|
|
||||||
|
|||||||
@@ -1,48 +0,0 @@
|
|||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
|
||||||
# Visual Studio Version 17
|
|
||||||
VisualStudioVersion = 17.0.31903.59
|
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "backend", "backend\backend.csproj", "{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "backend.tests", "backend.tests\backend.tests.csproj", "{63423BEA-4ECF-45CF-B0E8-611AC65AE463}"
|
|
||||||
EndProject
|
|
||||||
Global
|
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
|
||||||
Debug|Any CPU = Debug|Any CPU
|
|
||||||
Debug|x64 = Debug|x64
|
|
||||||
Debug|x86 = Debug|x86
|
|
||||||
Release|Any CPU = Release|Any CPU
|
|
||||||
Release|x64 = Release|x64
|
|
||||||
Release|x86 = Release|x86
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Debug|x86.Build.0 = Debug|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
{98CFCD4D-61C0-4FCA-BB6A-C6F68D5F4E40}.Release|x86.Build.0 = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Debug|x86.Build.0 = Debug|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
{63423BEA-4ECF-45CF-B0E8-611AC65AE463}.Release|x86.Build.0 = Release|Any CPU
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
|
||||||
HideSolutionNode = FALSE
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
|
||||||
@@ -24,3 +24,7 @@ Copy a file to the Pi:
|
|||||||
```sh
|
```sh
|
||||||
sshpass -p 'assistant' scp <local> pi@192.168.50.115:<remote>
|
sshpass -p 'assistant' scp <local> pi@192.168.50.115:<remote>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Prototype + probe findings
|
||||||
|
|
||||||
|
`findings.md` is the canonical record of what we proved during the prototype + C# probe phase. Read it before starting any audio, wakeword, or Pi-client work — it captures hardware constraints (USB Speaker Phone is 48 kHz-native, mic and speaker are the same device), the .NET-on-Linux env-var gotcha (`Environment.SetEnvironmentVariable` does NOT reach libc `getenv()`; you must P/Invoke `setenv` for libportaudio and onnxruntime to see env knobs), and the punch list of items to redesign before the main assistant. The Python prototypes live at `tests/01-record-play/` and `tests/02-wakeword/`; the C# Test 1 port at `tests/01-record-play-cs/` (deployed via `bin/probe-cs`).
|
||||||
|
|||||||
-32
@@ -1,32 +0,0 @@
|
|||||||
# syntax=docker/dockerfile:1.7
|
|
||||||
|
|
||||||
# ---- Backend build ----
|
|
||||||
FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
|
|
||||||
WORKDIR /src
|
|
||||||
COPY Assistant.sln ./
|
|
||||||
COPY backend/backend.csproj backend/
|
|
||||||
COPY backend.tests/backend.tests.csproj backend.tests/
|
|
||||||
RUN dotnet restore backend/backend.csproj
|
|
||||||
COPY backend/ backend/
|
|
||||||
RUN dotnet publish backend/backend.csproj -c Release -o /publish
|
|
||||||
|
|
||||||
# ---- Client bundle ----
|
|
||||||
FROM alpine:3.20 AS bundle
|
|
||||||
WORKDIR /src
|
|
||||||
RUN apk add --no-cache tar gzip
|
|
||||||
COPY client/ client/
|
|
||||||
COPY requirements.txt requirements-nodeps.txt ./
|
|
||||||
RUN tar czf /client.tar.gz client/ requirements.txt requirements-nodeps.txt
|
|
||||||
|
|
||||||
# ---- Final ----
|
|
||||||
FROM mcr.microsoft.com/dotnet/aspnet:9.0
|
|
||||||
WORKDIR /app
|
|
||||||
COPY --from=build /publish/ ./
|
|
||||||
RUN mkdir -p ./wwwroot && mkdir -p /data /keys
|
|
||||||
COPY --from=bundle /client.tar.gz ./wwwroot/client.tar.gz
|
|
||||||
|
|
||||||
ENV ASPNETCORE_URLS=http://+:8080
|
|
||||||
ENV ConnectionStrings__Default="Data Source=/data/assistant.db"
|
|
||||||
ENV Install__ClientTarballPath="/app/wwwroot/client.tar.gz"
|
|
||||||
EXPOSE 8080
|
|
||||||
ENTRYPOINT ["dotnet", "backend.dll"]
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
using Microsoft.AspNetCore.Hosting;
|
|
||||||
using Microsoft.AspNetCore.Mvc.Testing;
|
|
||||||
using Microsoft.Data.Sqlite;
|
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class ApiFactory : WebApplicationFactory<Program>
|
|
||||||
{
|
|
||||||
private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"assistant-test-{Guid.NewGuid():N}.db");
|
|
||||||
|
|
||||||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
|
||||||
{
|
|
||||||
builder.UseEnvironment("Test");
|
|
||||||
builder.ConfigureAppConfiguration((_, cfg) =>
|
|
||||||
{
|
|
||||||
cfg.AddInMemoryCollection(new Dictionary<string, string?>
|
|
||||||
{
|
|
||||||
["ConnectionStrings:Default"] = $"Data Source={_dbPath}",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void Dispose(bool disposing)
|
|
||||||
{
|
|
||||||
base.Dispose(disposing);
|
|
||||||
SqliteConnection.ClearAllPools();
|
|
||||||
foreach (var suffix in new[] { "", "-shm", "-wal", "-journal" })
|
|
||||||
{
|
|
||||||
var p = _dbPath + suffix;
|
|
||||||
try { if (File.Exists(p)) File.Delete(p); } catch { /* best effort */ }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
using System.Net;
|
|
||||||
using System.Net.Http.Json;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class AuthTests
|
|
||||||
{
|
|
||||||
[Fact]
|
|
||||||
public async Task Register_then_login_then_me_returns_user()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
|
|
||||||
var reg = await client.PostAsJsonAsync("/api/auth/register",
|
|
||||||
new { email = "first@example.com", password = "Passw0rd!" });
|
|
||||||
reg.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
|
|
||||||
var login = await client.PostAsJsonAsync("/api/auth/login",
|
|
||||||
new { email = "first@example.com", password = "Passw0rd!" });
|
|
||||||
login.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
|
|
||||||
var me = await client.GetAsync("/api/auth/me");
|
|
||||||
me.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
(await me.Content.ReadAsStringAsync()).Should().Contain("first@example.com");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Me_without_login_returns_401()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
var res = await client.GetAsync("/api/auth/me");
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.Unauthorized);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task First_user_is_admin()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
await client.PostAsJsonAsync("/api/auth/register",
|
|
||||||
new { email = "admin@example.com", password = "Passw0rd!" });
|
|
||||||
await client.PostAsJsonAsync("/api/auth/login",
|
|
||||||
new { email = "admin@example.com", password = "Passw0rd!" });
|
|
||||||
|
|
||||||
var me = await client.GetAsync("/api/auth/me");
|
|
||||||
(await me.Content.ReadAsStringAsync()).Should().Contain("\"isAdmin\":true");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
using backend.Devices;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class DeviceTokenServiceTests
|
|
||||||
{
|
|
||||||
private readonly DeviceTokenService _svc = new();
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Generate_returns_url_safe_token_of_expected_length()
|
|
||||||
{
|
|
||||||
var (plaintext, hash) = _svc.Generate();
|
|
||||||
plaintext.Should().MatchRegex("^[A-Za-z0-9_-]+$");
|
|
||||||
plaintext.Length.Should().BeGreaterThanOrEqualTo(40);
|
|
||||||
hash.Should().NotBeNullOrEmpty();
|
|
||||||
hash.Should().NotBe(plaintext);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Hash_is_deterministic_for_same_input()
|
|
||||||
{
|
|
||||||
var (plaintext, hash) = _svc.Generate();
|
|
||||||
_svc.Hash(plaintext).Should().Be(hash);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Verify_accepts_matching_plaintext_and_rejects_others()
|
|
||||||
{
|
|
||||||
var (plaintext, hash) = _svc.Generate();
|
|
||||||
_svc.Verify(plaintext, hash).Should().BeTrue();
|
|
||||||
_svc.Verify(plaintext + "x", hash).Should().BeFalse();
|
|
||||||
_svc.Verify("totally-different", hash).Should().BeFalse();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
using System.Net;
|
|
||||||
using System.Net.Http.Json;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class DevicesEndpointsTests
|
|
||||||
{
|
|
||||||
private static async Task<(HttpClient signedIn, string code)> SetupWithPairCode(ApiFactory factory, string email = "owner@example.com")
|
|
||||||
{
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
await client.PostAsJsonAsync("/api/auth/register",
|
|
||||||
new { email, password = "Passw0rd!" });
|
|
||||||
await client.PostAsJsonAsync("/api/auth/login",
|
|
||||||
new { email, password = "Passw0rd!" });
|
|
||||||
var codeRes = await client.PostAsJsonAsync("/api/pair-code", new { name = "kitchen" });
|
|
||||||
var body = await codeRes.Content.ReadFromJsonAsync<Dictionary<string, object>>();
|
|
||||||
return (client, body!["code"].ToString()!);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task User_sees_only_their_own_devices()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var (alice, codeA) = await SetupWithPairCode(factory, "alice@example.com");
|
|
||||||
await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code = codeA, hostname = "rpi", client_version = "0.1" });
|
|
||||||
|
|
||||||
var (bob, codeB) = await SetupWithPairCode(factory, "bob@example.com");
|
|
||||||
await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code = codeB, hostname = "rpi", client_version = "0.1" });
|
|
||||||
|
|
||||||
var aliceList = await alice.GetStringAsync("/api/devices");
|
|
||||||
var bobList = await bob.GetStringAsync("/api/devices");
|
|
||||||
|
|
||||||
aliceList.Should().Contain("kitchen");
|
|
||||||
bobList.Should().Contain("kitchen");
|
|
||||||
aliceList.Should().NotBe(bobList);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Rename_device_updates_name()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var (client, code) = await SetupWithPairCode(factory);
|
|
||||||
await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code, hostname = "rpi", client_version = "0.1" });
|
|
||||||
|
|
||||||
var listJson = await client.GetFromJsonAsync<List<Dictionary<string, object>>>("/api/devices");
|
|
||||||
var id = listJson![0]["id"].ToString();
|
|
||||||
|
|
||||||
var patchRes = await client.PatchAsJsonAsync($"/api/devices/{id}", new { name = "living-room" });
|
|
||||||
patchRes.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
|
|
||||||
var after = await client.GetStringAsync($"/api/devices/{id}");
|
|
||||||
after.Should().Contain("living-room");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Revoke_device_sets_is_revoked_true()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var (client, code) = await SetupWithPairCode(factory);
|
|
||||||
await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code, hostname = "rpi", client_version = "0.1" });
|
|
||||||
|
|
||||||
var listJson = await client.GetFromJsonAsync<List<Dictionary<string, object>>>("/api/devices");
|
|
||||||
var id = listJson![0]["id"].ToString();
|
|
||||||
|
|
||||||
var del = await client.DeleteAsync($"/api/devices/{id}");
|
|
||||||
del.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
|
|
||||||
var after = await client.GetStringAsync($"/api/devices/{id}");
|
|
||||||
after.Should().Contain("\"isRevoked\":true");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
using System.Net;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class HealthTests(ApiFactory factory) : IClassFixture<ApiFactory>
|
|
||||||
{
|
|
||||||
private readonly ApiFactory _factory = factory;
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Health_returns_ok()
|
|
||||||
{
|
|
||||||
var client = _factory.CreateClient();
|
|
||||||
var res = await client.GetAsync("/health");
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
(await res.Content.ReadAsStringAsync()).Should().Contain("\"ok\":true");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
using System.Net;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Microsoft.AspNetCore.Hosting;
|
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class InstallEndpointsTests
|
|
||||||
{
|
|
||||||
[Fact]
|
|
||||||
public async Task Install_sh_returns_shell_script_with_request_host()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
var res = await client.GetAsync("/install.sh");
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
res.Content.Headers.ContentType!.MediaType.Should().Be("text/x-shellscript");
|
|
||||||
|
|
||||||
var body = await res.Content.ReadAsStringAsync();
|
|
||||||
body.Should().StartWith("#!/usr/bin/env bash");
|
|
||||||
body.Should().Contain("BACKEND_URL=\"http://localhost\"");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Client_tarball_returns_octet_stream()
|
|
||||||
{
|
|
||||||
var tmp = Path.Combine(Path.GetTempPath(), $"client-test-{Guid.NewGuid():N}.tar.gz");
|
|
||||||
File.WriteAllBytes(tmp, new byte[] { 0x1f, 0x8b }); // tiny gzip-ish marker
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.WithWebHostBuilder(b =>
|
|
||||||
{
|
|
||||||
b.ConfigureAppConfiguration((_, cfg) =>
|
|
||||||
{
|
|
||||||
cfg.AddInMemoryCollection(new Dictionary<string, string?>
|
|
||||||
{
|
|
||||||
["Install:ClientTarballPath"] = tmp,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).CreateClient();
|
|
||||||
|
|
||||||
var res = await client.GetAsync("/client.tar.gz");
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
res.Content.Headers.ContentType!.MediaType.Should().Be("application/gzip");
|
|
||||||
(await res.Content.ReadAsByteArrayAsync()).Length.Should().Be(2);
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (File.Exists(tmp)) File.Delete(tmp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
using backend.Install;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class InstallScriptBuilderTests
|
|
||||||
{
|
|
||||||
[Fact]
|
|
||||||
public void Build_inlines_the_backend_url_into_the_script()
|
|
||||||
{
|
|
||||||
var script = new InstallScriptBuilder().Build("https://example.test");
|
|
||||||
script.Should().Contain("BACKEND_URL=\"https://example.test\"");
|
|
||||||
script.Should().Contain("#!/usr/bin/env bash");
|
|
||||||
script.Should().Contain("$BACKEND_URL/client.tar.gz");
|
|
||||||
script.Should().Contain("client.pair");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
using System.Net;
|
|
||||||
using System.Net.Http.Json;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class PairingEndpointsTests
|
|
||||||
{
|
|
||||||
private static async Task<HttpClient> SignedInClient(ApiFactory factory, string email = "owner@example.com")
|
|
||||||
{
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
await client.PostAsJsonAsync("/api/auth/register",
|
|
||||||
new { email, password = "Passw0rd!" });
|
|
||||||
await client.PostAsJsonAsync("/api/auth/login",
|
|
||||||
new { email, password = "Passw0rd!" });
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Anonymous_user_cannot_request_a_pairing_code()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = factory.CreateClient();
|
|
||||||
var res = await client.PostAsJsonAsync("/api/pair-code", new { name = "kitchen" });
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.Unauthorized);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Pair_code_then_pair_returns_token_and_device_id()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = await SignedInClient(factory);
|
|
||||||
var codeRes = await client.PostAsJsonAsync("/api/pair-code", new { name = "kitchen" });
|
|
||||||
codeRes.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
var codeBody = await codeRes.Content.ReadFromJsonAsync<Dictionary<string, object>>();
|
|
||||||
var code = codeBody!["code"].ToString()!;
|
|
||||||
|
|
||||||
var pairRes = await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code, hostname = "rpi", client_version = "0.1" });
|
|
||||||
pairRes.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
var body = await pairRes.Content.ReadAsStringAsync();
|
|
||||||
body.Should().Contain("device_id");
|
|
||||||
body.Should().Contain("device_token");
|
|
||||||
body.Should().Contain("backend_ws");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Pair_with_invalid_code_returns_404()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var res = await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code = "ZZZZZZZZ", hostname = "rpi", client_version = "0.1" });
|
|
||||||
res.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Pair_with_consumed_code_returns_404()
|
|
||||||
{
|
|
||||||
using var factory = new ApiFactory();
|
|
||||||
var client = await SignedInClient(factory);
|
|
||||||
var codeRes = await client.PostAsJsonAsync("/api/pair-code", new { name = "kitchen" });
|
|
||||||
var code = (await codeRes.Content.ReadFromJsonAsync<Dictionary<string, object>>())!["code"].ToString()!;
|
|
||||||
|
|
||||||
var first = await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code, hostname = "rpi", client_version = "0.1" });
|
|
||||||
first.StatusCode.Should().Be(HttpStatusCode.OK);
|
|
||||||
|
|
||||||
var second = await factory.CreateClient().PostAsJsonAsync("/api/pair",
|
|
||||||
new { code, hostname = "rpi", client_version = "0.1" });
|
|
||||||
second.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
using backend.Pairing;
|
|
||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace backend.tests;
|
|
||||||
|
|
||||||
public class PairingServiceTests
|
|
||||||
{
|
|
||||||
[Fact]
|
|
||||||
public void Generated_codes_use_only_safe_alphabet_and_are_8_chars()
|
|
||||||
{
|
|
||||||
var svc = new PairingService();
|
|
||||||
for (var i = 0; i < 50; i++)
|
|
||||||
{
|
|
||||||
var code = svc.GenerateCode();
|
|
||||||
code.Length.Should().Be(8);
|
|
||||||
code.Should().MatchRegex("^[A-HJ-KM-NP-Z2-9]+$");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Generated_codes_are_not_all_identical()
|
|
||||||
{
|
|
||||||
var svc = new PairingService();
|
|
||||||
var codes = Enumerable.Range(0, 20).Select(_ => svc.GenerateCode()).ToHashSet();
|
|
||||||
codes.Count.Should().BeGreaterThan(15);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net9.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
<IsPackable>false</IsPackable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="coverlet.collector" Version="6.0.2" />
|
|
||||||
<PackageReference Include="FluentAssertions" Version="8.10.0" />
|
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="9.0.8" />
|
|
||||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
|
|
||||||
<PackageReference Include="xunit" Version="2.9.2" />
|
|
||||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<Using Include="Xunit" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\backend\backend.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
using Microsoft.AspNetCore.Identity;
|
|
||||||
|
|
||||||
namespace backend.Auth;
|
|
||||||
|
|
||||||
public class ApplicationUser : IdentityUser<Guid>
|
|
||||||
{
|
|
||||||
}
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
using Microsoft.AspNetCore.Identity;
|
|
||||||
using Microsoft.AspNetCore.Mvc;
|
|
||||||
|
|
||||||
namespace backend.Auth;
|
|
||||||
|
|
||||||
public record RegisterRequest(string Email, string Password);
|
|
||||||
public record LoginRequest(string Email, string Password);
|
|
||||||
|
|
||||||
public static class AuthEndpoints
|
|
||||||
{
|
|
||||||
public static IEndpointRouteBuilder MapAuthEndpoints(this IEndpointRouteBuilder app)
|
|
||||||
{
|
|
||||||
var grp = app.MapGroup("/api/auth");
|
|
||||||
|
|
||||||
grp.MapPost("/register", async (
|
|
||||||
[FromBody] RegisterRequest req,
|
|
||||||
UserManager<ApplicationUser> users) =>
|
|
||||||
{
|
|
||||||
var user = new ApplicationUser { UserName = req.Email, Email = req.Email };
|
|
||||||
var create = await users.CreateAsync(user, req.Password);
|
|
||||||
if (!create.Succeeded)
|
|
||||||
return Results.BadRequest(new { errors = create.Errors.Select(e => e.Description) });
|
|
||||||
|
|
||||||
var anyAdmin = await users.GetUsersInRoleAsync(Roles.Admin);
|
|
||||||
var role = anyAdmin.Count == 0 ? Roles.Admin : Roles.User;
|
|
||||||
await users.AddToRoleAsync(user, role);
|
|
||||||
return Results.Ok(new { id = user.Id, email = user.Email, role });
|
|
||||||
});
|
|
||||||
|
|
||||||
grp.MapPost("/login", async (
|
|
||||||
[FromBody] LoginRequest req,
|
|
||||||
SignInManager<ApplicationUser> signIn) =>
|
|
||||||
{
|
|
||||||
var res = await signIn.PasswordSignInAsync(req.Email, req.Password,
|
|
||||||
isPersistent: true, lockoutOnFailure: false);
|
|
||||||
return res.Succeeded ? Results.Ok() : Results.Unauthorized();
|
|
||||||
});
|
|
||||||
|
|
||||||
grp.MapPost("/logout", async (SignInManager<ApplicationUser> signIn) =>
|
|
||||||
{
|
|
||||||
await signIn.SignOutAsync();
|
|
||||||
return Results.Ok();
|
|
||||||
}).RequireAuthorization();
|
|
||||||
|
|
||||||
grp.MapGet("/me", async (
|
|
||||||
UserManager<ApplicationUser> users,
|
|
||||||
HttpContext http) =>
|
|
||||||
{
|
|
||||||
var user = await users.GetUserAsync(http.User);
|
|
||||||
if (user is null) return Results.Unauthorized();
|
|
||||||
var rolesList = await users.GetRolesAsync(user);
|
|
||||||
return Results.Ok(new
|
|
||||||
{
|
|
||||||
id = user.Id,
|
|
||||||
email = user.Email,
|
|
||||||
isAdmin = rolesList.Contains(Roles.Admin),
|
|
||||||
});
|
|
||||||
}).RequireAuthorization();
|
|
||||||
|
|
||||||
return app;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
using backend.Data;
|
|
||||||
using Microsoft.AspNetCore.Identity;
|
|
||||||
|
|
||||||
namespace backend.Auth;
|
|
||||||
|
|
||||||
public static class IdentitySetup
|
|
||||||
{
|
|
||||||
public static IServiceCollection AddAppIdentity(this IServiceCollection services)
|
|
||||||
{
|
|
||||||
services.AddIdentity<ApplicationUser, IdentityRole<Guid>>(opt =>
|
|
||||||
{
|
|
||||||
opt.Password.RequiredLength = 8;
|
|
||||||
opt.Password.RequireNonAlphanumeric = false;
|
|
||||||
opt.User.RequireUniqueEmail = true;
|
|
||||||
opt.SignIn.RequireConfirmedAccount = false;
|
|
||||||
})
|
|
||||||
.AddEntityFrameworkStores<AppDbContext>()
|
|
||||||
.AddDefaultTokenProviders();
|
|
||||||
|
|
||||||
services.ConfigureApplicationCookie(opt =>
|
|
||||||
{
|
|
||||||
opt.Cookie.HttpOnly = true;
|
|
||||||
opt.Cookie.SameSite = SameSiteMode.Lax;
|
|
||||||
opt.Cookie.SecurePolicy = CookieSecurePolicy.SameAsRequest;
|
|
||||||
opt.ExpireTimeSpan = TimeSpan.FromDays(30);
|
|
||||||
opt.SlidingExpiration = true;
|
|
||||||
opt.Events.OnRedirectToLogin = ctx =>
|
|
||||||
{
|
|
||||||
ctx.Response.StatusCode = 401;
|
|
||||||
return Task.CompletedTask;
|
|
||||||
};
|
|
||||||
opt.Events.OnRedirectToAccessDenied = ctx =>
|
|
||||||
{
|
|
||||||
ctx.Response.StatusCode = 403;
|
|
||||||
return Task.CompletedTask;
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
return services;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Task SeedRolesAsync(IServiceProvider sp)
|
|
||||||
{
|
|
||||||
var roleMgr = sp.GetRequiredService<RoleManager<IdentityRole<Guid>>>();
|
|
||||||
foreach (var name in Roles.All)
|
|
||||||
{
|
|
||||||
if (!await roleMgr.RoleExistsAsync(name))
|
|
||||||
await roleMgr.CreateAsync(new IdentityRole<Guid>(name));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
namespace backend.Auth;
|
|
||||||
|
|
||||||
public static class Roles
|
|
||||||
{
|
|
||||||
public const string Admin = "Admin";
|
|
||||||
public const string User = "User";
|
|
||||||
|
|
||||||
public static readonly string[] All = { Admin, User };
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
using backend.Auth;
|
|
||||||
using backend.Devices;
|
|
||||||
using backend.Pairing;
|
|
||||||
using backend.Settings;
|
|
||||||
using Microsoft.AspNetCore.Identity;
|
|
||||||
using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
|
|
||||||
namespace backend.Data;
|
|
||||||
|
|
||||||
public class AppDbContext(DbContextOptions<AppDbContext> options)
|
|
||||||
: IdentityDbContext<ApplicationUser, IdentityRole<Guid>, Guid>(options)
|
|
||||||
{
|
|
||||||
public DbSet<Device> Devices => Set<Device>();
|
|
||||||
public DbSet<DeviceConfig> DeviceConfigs => Set<DeviceConfig>();
|
|
||||||
public DbSet<SystemSettings> SystemSettings => Set<SystemSettings>();
|
|
||||||
public DbSet<PairingCode> PairingCodes => Set<PairingCode>();
|
|
||||||
|
|
||||||
protected override void OnModelCreating(ModelBuilder b)
|
|
||||||
{
|
|
||||||
base.OnModelCreating(b);
|
|
||||||
|
|
||||||
b.Entity<Device>()
|
|
||||||
.HasOne<ApplicationUser>()
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey(d => d.OwnerUserId)
|
|
||||||
.OnDelete(DeleteBehavior.Cascade);
|
|
||||||
b.Entity<Device>().HasIndex(d => d.TokenHash).IsUnique();
|
|
||||||
b.Entity<Device>()
|
|
||||||
.HasOne(d => d.Config)
|
|
||||||
.WithOne()
|
|
||||||
.HasForeignKey<DeviceConfig>(c => c.DeviceId)
|
|
||||||
.OnDelete(DeleteBehavior.Cascade);
|
|
||||||
b.Entity<DeviceConfig>().HasKey(c => c.DeviceId);
|
|
||||||
b.Entity<SystemSettings>().HasKey(s => s.Id);
|
|
||||||
b.Entity<SystemSettings>().Property(s => s.Id).ValueGeneratedNever();
|
|
||||||
b.Entity<PairingCode>().HasKey(p => p.Code);
|
|
||||||
b.Entity<PairingCode>().HasIndex(p => p.ExpiresAt);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
namespace backend.Devices;
|
|
||||||
|
|
||||||
public class Device
|
|
||||||
{
|
|
||||||
public Guid Id { get; set; } = Guid.NewGuid();
|
|
||||||
public Guid OwnerUserId { get; set; }
|
|
||||||
public string Name { get; set; } = "";
|
|
||||||
public DateTimeOffset PairedAt { get; set; }
|
|
||||||
public DateTimeOffset? LastSeenAt { get; set; }
|
|
||||||
public string TokenHash { get; set; } = "";
|
|
||||||
public bool IsRevoked { get; set; }
|
|
||||||
public DeviceConfig? Config { get; set; }
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace backend.Devices;
|
|
||||||
|
|
||||||
public class DeviceConfig
|
|
||||||
{
|
|
||||||
public Guid DeviceId { get; set; }
|
|
||||||
public string SystemPrompt { get; set; } = "";
|
|
||||||
public string Voice { get; set; } = "";
|
|
||||||
public string Model { get; set; } = "";
|
|
||||||
public int IdleTimeoutSeconds { get; set; } = 30;
|
|
||||||
public string EnabledToolsJson { get; set; } = "[\"end_session\",\"get_current_time\",\"set_volume\"]";
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
using System.Security.Cryptography;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
namespace backend.Devices;
|
|
||||||
|
|
||||||
public class DeviceTokenService
|
|
||||||
{
|
|
||||||
public (string Plaintext, string Hash) Generate()
|
|
||||||
{
|
|
||||||
Span<byte> bytes = stackalloc byte[32];
|
|
||||||
RandomNumberGenerator.Fill(bytes);
|
|
||||||
var plaintext = Base64UrlEncode(bytes);
|
|
||||||
return (plaintext, Hash(plaintext));
|
|
||||||
}
|
|
||||||
|
|
||||||
public string Hash(string plaintext)
|
|
||||||
{
|
|
||||||
Span<byte> hash = stackalloc byte[32];
|
|
||||||
SHA256.HashData(Encoding.UTF8.GetBytes(plaintext), hash);
|
|
||||||
return Convert.ToHexString(hash);
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool Verify(string plaintext, string expectedHash)
|
|
||||||
{
|
|
||||||
var actual = Hash(plaintext);
|
|
||||||
return CryptographicOperations.FixedTimeEquals(
|
|
||||||
Encoding.UTF8.GetBytes(actual),
|
|
||||||
Encoding.UTF8.GetBytes(expectedHash));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string Base64UrlEncode(ReadOnlySpan<byte> bytes)
|
|
||||||
=> Convert.ToBase64String(bytes)
|
|
||||||
.TrimEnd('=')
|
|
||||||
.Replace('+', '-')
|
|
||||||
.Replace('/', '_');
|
|
||||||
}
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
using System.Security.Claims;
|
|
||||||
using backend.Data;
|
|
||||||
using Microsoft.AspNetCore.Mvc;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
|
|
||||||
namespace backend.Devices;
|
|
||||||
|
|
||||||
public record RenameRequest(string Name);
|
|
||||||
|
|
||||||
public static class DevicesEndpoints
|
|
||||||
{
|
|
||||||
public static IEndpointRouteBuilder MapDevicesEndpoints(this IEndpointRouteBuilder app)
|
|
||||||
{
|
|
||||||
var grp = app.MapGroup("/api/devices").RequireAuthorization();
|
|
||||||
|
|
||||||
grp.MapGet("/", async (AppDbContext db, HttpContext http, CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var uid = GetUserId(http);
|
|
||||||
var devs = await db.Devices
|
|
||||||
.Where(d => d.OwnerUserId == uid)
|
|
||||||
.Select(d => new
|
|
||||||
{
|
|
||||||
id = d.Id,
|
|
||||||
name = d.Name,
|
|
||||||
pairedAt = d.PairedAt,
|
|
||||||
lastSeenAt = d.LastSeenAt,
|
|
||||||
isRevoked = d.IsRevoked,
|
|
||||||
})
|
|
||||||
.ToListAsync(ct);
|
|
||||||
return Results.Ok(devs);
|
|
||||||
});
|
|
||||||
|
|
||||||
grp.MapGet("/{id:guid}", async (Guid id, AppDbContext db, HttpContext http, CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var uid = GetUserId(http);
|
|
||||||
var d = await db.Devices.Include(x => x.Config)
|
|
||||||
.FirstOrDefaultAsync(x => x.Id == id && x.OwnerUserId == uid, ct);
|
|
||||||
return d is null
|
|
||||||
? Results.NotFound()
|
|
||||||
: Results.Ok(new
|
|
||||||
{
|
|
||||||
id = d.Id,
|
|
||||||
name = d.Name,
|
|
||||||
pairedAt = d.PairedAt,
|
|
||||||
lastSeenAt = d.LastSeenAt,
|
|
||||||
isRevoked = d.IsRevoked,
|
|
||||||
config = d.Config,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
grp.MapPatch("/{id:guid}", async (
|
|
||||||
Guid id, [FromBody] RenameRequest req,
|
|
||||||
AppDbContext db, HttpContext http, CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var uid = GetUserId(http);
|
|
||||||
var d = await db.Devices.FirstOrDefaultAsync(x => x.Id == id && x.OwnerUserId == uid, ct);
|
|
||||||
if (d is null) return Results.NotFound();
|
|
||||||
if (string.IsNullOrWhiteSpace(req.Name))
|
|
||||||
return Results.BadRequest(new { error = "name is required" });
|
|
||||||
d.Name = req.Name.Trim();
|
|
||||||
await db.SaveChangesAsync(ct);
|
|
||||||
return Results.Ok();
|
|
||||||
});
|
|
||||||
|
|
||||||
grp.MapDelete("/{id:guid}", async (Guid id, AppDbContext db, HttpContext http, CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var uid = GetUserId(http);
|
|
||||||
var d = await db.Devices.FirstOrDefaultAsync(x => x.Id == id && x.OwnerUserId == uid, ct);
|
|
||||||
if (d is null) return Results.NotFound();
|
|
||||||
d.IsRevoked = true;
|
|
||||||
await db.SaveChangesAsync(ct);
|
|
||||||
return Results.Ok();
|
|
||||||
});
|
|
||||||
|
|
||||||
return app;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Guid GetUserId(HttpContext http)
|
|
||||||
{
|
|
||||||
var s = http.User.FindFirstValue(ClaimTypes.NameIdentifier);
|
|
||||||
return Guid.TryParse(s, out var g) ? g : Guid.Empty;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
|
|
||||||
namespace backend.Install;
|
|
||||||
|
|
||||||
public static class InstallEndpoints
|
|
||||||
{
|
|
||||||
public static IEndpointRouteBuilder MapInstallEndpoints(this IEndpointRouteBuilder app)
|
|
||||||
{
|
|
||||||
app.MapGet("/install.sh", (HttpContext http, InstallScriptBuilder builder) =>
|
|
||||||
{
|
|
||||||
var scheme = http.Request.Scheme;
|
|
||||||
var host = http.Request.Host.ToString();
|
|
||||||
var script = builder.Build($"{scheme}://{host}");
|
|
||||||
return Results.Text(script, "text/x-shellscript");
|
|
||||||
});
|
|
||||||
|
|
||||||
app.MapGet("/client.tar.gz", (IConfiguration cfg, IWebHostEnvironment env) =>
|
|
||||||
{
|
|
||||||
var path = cfg["Install:ClientTarballPath"]
|
|
||||||
?? Path.Combine(env.WebRootPath ?? "wwwroot", "client.tar.gz");
|
|
||||||
if (!File.Exists(path))
|
|
||||||
return Results.NotFound(new { error = "client.tar.gz not found at " + path });
|
|
||||||
return Results.File(path, "application/gzip", "client.tar.gz");
|
|
||||||
});
|
|
||||||
|
|
||||||
return app;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
namespace backend.Install;
|
|
||||||
|
|
||||||
public class InstallScriptBuilder
|
|
||||||
{
|
|
||||||
public string Build(string backendUrl)
|
|
||||||
{
|
|
||||||
return $$"""
|
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Smart Assistant installer (idempotent). Re-run to upgrade.
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
BACKEND_URL="{{backendUrl}}"
|
|
||||||
ASSISTANT_HOME="$HOME/assistant"
|
|
||||||
RESET=0
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in
|
|
||||||
--reset) RESET=1 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "Installing system deps (sudo password may be prompted)..."
|
|
||||||
sudo apt-get update -qq
|
|
||||||
sudo apt-get install -y --no-install-recommends python3-venv libportaudio2 alsa-utils
|
|
||||||
|
|
||||||
mkdir -p "$ASSISTANT_HOME/code" "$ASSISTANT_HOME/state"
|
|
||||||
|
|
||||||
if [ ! -d "$ASSISTANT_HOME/.venv" ]; then
|
|
||||||
python3 -m venv "$ASSISTANT_HOME/.venv"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Downloading client bundle from $BACKEND_URL ..."
|
|
||||||
curl -fsSL "$BACKEND_URL/client.tar.gz" | tar -xz -C "$ASSISTANT_HOME/code"
|
|
||||||
|
|
||||||
"$ASSISTANT_HOME/.venv/bin/pip" install --quiet -r "$ASSISTANT_HOME/code/requirements.txt"
|
|
||||||
"$ASSISTANT_HOME/.venv/bin/pip" install --quiet --no-deps -r "$ASSISTANT_HOME/code/requirements-nodeps.txt"
|
|
||||||
|
|
||||||
if [ "$RESET" -eq 1 ]; then
|
|
||||||
rm -f "$ASSISTANT_HOME/state/config.json"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f "$ASSISTANT_HOME/state/config.json" ]; then
|
|
||||||
echo
|
|
||||||
read -r -p "Pairing code: " CODE
|
|
||||||
"$ASSISTANT_HOME/.venv/bin/python" -m client.pair --backend "$BACKEND_URL" --code "$CODE"
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p "$HOME/.config/systemd/user"
|
|
||||||
cat > "$HOME/.config/systemd/user/assistant.service" <<UNIT
|
|
||||||
[Unit]
|
|
||||||
Description=Smart Assistant Pi client
|
|
||||||
After=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Environment=PA_ALSA_PLUGHW=1
|
|
||||||
ExecStart=$ASSISTANT_HOME/.venv/bin/python -m client.main
|
|
||||||
WorkingDirectory=$ASSISTANT_HOME/code
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=default.target
|
|
||||||
UNIT
|
|
||||||
|
|
||||||
loginctl enable-linger "$USER" 2>/dev/null || true
|
|
||||||
systemctl --user daemon-reload
|
|
||||||
systemctl --user enable --now assistant
|
|
||||||
echo "Done. systemctl --user status assistant"
|
|
||||||
""";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
[Migration("20260611181007_Initial")]
|
|
||||||
partial class Initial
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
public partial class Initial : Migration
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,267 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
[Migration("20260611182529_AddIdentity")]
|
|
||||||
partial class AddIdentity
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("RoleNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoleClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderKey")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderDisplayName")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("LoginProvider", "ProviderKey");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserLogins", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "RoleId");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Value")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "LoginProvider", "Name");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserTokens", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Auth.ApplicationUser", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("AccessFailedCount")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Email")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("EmailConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<bool>("LockoutEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LockoutEnd")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedEmail")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedUserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PasswordHash")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PhoneNumber")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("PhoneNumberConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("SecurityStamp")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("TwoFactorEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("UserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedEmail")
|
|
||||||
.HasDatabaseName("EmailIndex");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedUserName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UserNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUsers", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
public partial class AddIdentity : Migration
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetRoles",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
Name = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
NormalizedName = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
ConcurrencyStamp = table.Column<string>(type: "TEXT", nullable: true)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetRoles", x => x.Id);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetUsers",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
UserName = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
NormalizedUserName = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
Email = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
NormalizedEmail = table.Column<string>(type: "TEXT", maxLength: 256, nullable: true),
|
|
||||||
EmailConfirmed = table.Column<bool>(type: "INTEGER", nullable: false),
|
|
||||||
PasswordHash = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
SecurityStamp = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
ConcurrencyStamp = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
PhoneNumber = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
PhoneNumberConfirmed = table.Column<bool>(type: "INTEGER", nullable: false),
|
|
||||||
TwoFactorEnabled = table.Column<bool>(type: "INTEGER", nullable: false),
|
|
||||||
LockoutEnd = table.Column<DateTimeOffset>(type: "TEXT", nullable: true),
|
|
||||||
LockoutEnabled = table.Column<bool>(type: "INTEGER", nullable: false),
|
|
||||||
AccessFailedCount = table.Column<int>(type: "INTEGER", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetUsers", x => x.Id);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetRoleClaims",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<int>(type: "INTEGER", nullable: false)
|
|
||||||
.Annotation("Sqlite:Autoincrement", true),
|
|
||||||
RoleId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
ClaimType = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
ClaimValue = table.Column<string>(type: "TEXT", nullable: true)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetRoleClaims", x => x.Id);
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetRoleClaims_AspNetRoles_RoleId",
|
|
||||||
column: x => x.RoleId,
|
|
||||||
principalTable: "AspNetRoles",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetUserClaims",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<int>(type: "INTEGER", nullable: false)
|
|
||||||
.Annotation("Sqlite:Autoincrement", true),
|
|
||||||
UserId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
ClaimType = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
ClaimValue = table.Column<string>(type: "TEXT", nullable: true)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetUserClaims", x => x.Id);
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetUserClaims_AspNetUsers_UserId",
|
|
||||||
column: x => x.UserId,
|
|
||||||
principalTable: "AspNetUsers",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetUserLogins",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
LoginProvider = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
ProviderKey = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
ProviderDisplayName = table.Column<string>(type: "TEXT", nullable: true),
|
|
||||||
UserId = table.Column<Guid>(type: "TEXT", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetUserLogins", x => new { x.LoginProvider, x.ProviderKey });
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetUserLogins_AspNetUsers_UserId",
|
|
||||||
column: x => x.UserId,
|
|
||||||
principalTable: "AspNetUsers",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetUserRoles",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
UserId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
RoleId = table.Column<Guid>(type: "TEXT", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetUserRoles", x => new { x.UserId, x.RoleId });
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetUserRoles_AspNetRoles_RoleId",
|
|
||||||
column: x => x.RoleId,
|
|
||||||
principalTable: "AspNetRoles",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetUserRoles_AspNetUsers_UserId",
|
|
||||||
column: x => x.UserId,
|
|
||||||
principalTable: "AspNetUsers",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "AspNetUserTokens",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
UserId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
LoginProvider = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
Name = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
Value = table.Column<string>(type: "TEXT", nullable: true)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_AspNetUserTokens", x => new { x.UserId, x.LoginProvider, x.Name });
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_AspNetUserTokens_AspNetUsers_UserId",
|
|
||||||
column: x => x.UserId,
|
|
||||||
principalTable: "AspNetUsers",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_AspNetRoleClaims_RoleId",
|
|
||||||
table: "AspNetRoleClaims",
|
|
||||||
column: "RoleId");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "RoleNameIndex",
|
|
||||||
table: "AspNetRoles",
|
|
||||||
column: "NormalizedName",
|
|
||||||
unique: true);
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_AspNetUserClaims_UserId",
|
|
||||||
table: "AspNetUserClaims",
|
|
||||||
column: "UserId");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_AspNetUserLogins_UserId",
|
|
||||||
table: "AspNetUserLogins",
|
|
||||||
column: "UserId");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_AspNetUserRoles_RoleId",
|
|
||||||
table: "AspNetUserRoles",
|
|
||||||
column: "RoleId");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "EmailIndex",
|
|
||||||
table: "AspNetUsers",
|
|
||||||
column: "NormalizedEmail");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "UserNameIndex",
|
|
||||||
table: "AspNetUsers",
|
|
||||||
column: "NormalizedUserName",
|
|
||||||
unique: true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetRoleClaims");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetUserClaims");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetUserLogins");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetUserRoles");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetUserTokens");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetRoles");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "AspNetUsers");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,372 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
[Migration("20260611184053_AddDevicesAndSettings")]
|
|
||||||
partial class AddDevicesAndSettings
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("RoleNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoleClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderKey")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderDisplayName")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("LoginProvider", "ProviderKey");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserLogins", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "RoleId");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Value")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "LoginProvider", "Name");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserTokens", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Auth.ApplicationUser", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("AccessFailedCount")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Email")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("EmailConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<bool>("LockoutEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LockoutEnd")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedEmail")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedUserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PasswordHash")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PhoneNumber")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("PhoneNumberConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("SecurityStamp")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("TwoFactorEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("UserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedEmail")
|
|
||||||
.HasDatabaseName("EmailIndex");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedUserName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UserNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUsers", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("IsRevoked")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LastSeenAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("PairedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("TokenHash")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("OwnerUserId");
|
|
||||||
|
|
||||||
b.HasIndex("TokenHash")
|
|
||||||
.IsUnique();
|
|
||||||
|
|
||||||
b.ToTable("Devices");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("DeviceId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("EnabledToolsJson")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("IdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("Model")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("SystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Voice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("DeviceId");
|
|
||||||
|
|
||||||
b.ToTable("DeviceConfigs");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Settings.SystemSettings", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<int>("DefaultIdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultModel")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultSystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultVoice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.ToTable("SystemSettings");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Devices.Device", null)
|
|
||||||
.WithOne("Config")
|
|
||||||
.HasForeignKey("backend.Devices.DeviceConfig", "DeviceId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Navigation("Config");
|
|
||||||
});
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
public partial class AddDevicesAndSettings : Migration
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "Devices",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
OwnerUserId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
Name = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
PairedAt = table.Column<DateTimeOffset>(type: "TEXT", nullable: false),
|
|
||||||
LastSeenAt = table.Column<DateTimeOffset>(type: "TEXT", nullable: true),
|
|
||||||
TokenHash = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
IsRevoked = table.Column<bool>(type: "INTEGER", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_Devices", x => x.Id);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "SystemSettings",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Id = table.Column<int>(type: "INTEGER", nullable: false)
|
|
||||||
.Annotation("Sqlite:Autoincrement", true),
|
|
||||||
DefaultSystemPrompt = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
DefaultVoice = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
DefaultModel = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
DefaultIdleTimeoutSeconds = table.Column<int>(type: "INTEGER", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_SystemSettings", x => x.Id);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "DeviceConfigs",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
DeviceId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
SystemPrompt = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
Voice = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
Model = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
IdleTimeoutSeconds = table.Column<int>(type: "INTEGER", nullable: false),
|
|
||||||
EnabledToolsJson = table.Column<string>(type: "TEXT", nullable: false)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_DeviceConfigs", x => x.DeviceId);
|
|
||||||
table.ForeignKey(
|
|
||||||
name: "FK_DeviceConfigs_Devices_DeviceId",
|
|
||||||
column: x => x.DeviceId,
|
|
||||||
principalTable: "Devices",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_Devices_OwnerUserId",
|
|
||||||
table: "Devices",
|
|
||||||
column: "OwnerUserId");
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_Devices_TokenHash",
|
|
||||||
table: "Devices",
|
|
||||||
column: "TokenHash",
|
|
||||||
unique: true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "DeviceConfigs");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "SystemSettings");
|
|
||||||
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "Devices");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-380
@@ -1,380 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
[Migration("20260611184536_AddDeviceOwnerFkAndPinSettingsId")]
|
|
||||||
partial class AddDeviceOwnerFkAndPinSettingsId
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("RoleNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoleClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderKey")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderDisplayName")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("LoginProvider", "ProviderKey");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserLogins", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "RoleId");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Value")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "LoginProvider", "Name");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserTokens", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Auth.ApplicationUser", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("AccessFailedCount")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Email")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("EmailConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<bool>("LockoutEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LockoutEnd")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedEmail")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedUserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PasswordHash")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PhoneNumber")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("PhoneNumberConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("SecurityStamp")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("TwoFactorEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("UserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedEmail")
|
|
||||||
.HasDatabaseName("EmailIndex");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedUserName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UserNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUsers", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("IsRevoked")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LastSeenAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("PairedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("TokenHash")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("OwnerUserId");
|
|
||||||
|
|
||||||
b.HasIndex("TokenHash")
|
|
||||||
.IsUnique();
|
|
||||||
|
|
||||||
b.ToTable("Devices");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("DeviceId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("EnabledToolsJson")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("IdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("Model")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("SystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Voice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("DeviceId");
|
|
||||||
|
|
||||||
b.ToTable("DeviceConfigs");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Settings.SystemSettings", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<int>("DefaultIdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultModel")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultSystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultVoice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.ToTable("SystemSettings");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("OwnerUserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Devices.Device", null)
|
|
||||||
.WithOne("Config")
|
|
||||||
.HasForeignKey("backend.Devices.DeviceConfig", "DeviceId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Navigation("Config");
|
|
||||||
});
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
public partial class AddDeviceOwnerFkAndPinSettingsId : Migration
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.AlterColumn<int>(
|
|
||||||
name: "Id",
|
|
||||||
table: "SystemSettings",
|
|
||||||
type: "INTEGER",
|
|
||||||
nullable: false,
|
|
||||||
oldClrType: typeof(int),
|
|
||||||
oldType: "INTEGER")
|
|
||||||
.OldAnnotation("Sqlite:Autoincrement", true);
|
|
||||||
|
|
||||||
migrationBuilder.AddForeignKey(
|
|
||||||
name: "FK_Devices_AspNetUsers_OwnerUserId",
|
|
||||||
table: "Devices",
|
|
||||||
column: "OwnerUserId",
|
|
||||||
principalTable: "AspNetUsers",
|
|
||||||
principalColumn: "Id",
|
|
||||||
onDelete: ReferentialAction.Cascade);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.DropForeignKey(
|
|
||||||
name: "FK_Devices_AspNetUsers_OwnerUserId",
|
|
||||||
table: "Devices");
|
|
||||||
|
|
||||||
migrationBuilder.AlterColumn<int>(
|
|
||||||
name: "Id",
|
|
||||||
table: "SystemSettings",
|
|
||||||
type: "INTEGER",
|
|
||||||
nullable: false,
|
|
||||||
oldClrType: typeof(int),
|
|
||||||
oldType: "INTEGER")
|
|
||||||
.Annotation("Sqlite:Autoincrement", true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,405 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
[Migration("20260611185048_AddPairingCodes")]
|
|
||||||
partial class AddPairingCodes
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("RoleNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoleClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderKey")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderDisplayName")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("LoginProvider", "ProviderKey");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserLogins", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "RoleId");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Value")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "LoginProvider", "Name");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserTokens", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Auth.ApplicationUser", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("AccessFailedCount")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Email")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("EmailConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<bool>("LockoutEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LockoutEnd")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedEmail")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedUserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PasswordHash")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PhoneNumber")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("PhoneNumberConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("SecurityStamp")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("TwoFactorEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("UserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedEmail")
|
|
||||||
.HasDatabaseName("EmailIndex");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedUserName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UserNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUsers", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("IsRevoked")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LastSeenAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("PairedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("TokenHash")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("OwnerUserId");
|
|
||||||
|
|
||||||
b.HasIndex("TokenHash")
|
|
||||||
.IsUnique();
|
|
||||||
|
|
||||||
b.ToTable("Devices");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("DeviceId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("EnabledToolsJson")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("IdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("Model")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("SystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Voice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("DeviceId");
|
|
||||||
|
|
||||||
b.ToTable("DeviceConfigs");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Pairing.PairingCode", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("Code")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("ConsumedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DeviceName")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("ExpiresAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Code");
|
|
||||||
|
|
||||||
b.HasIndex("ExpiresAt");
|
|
||||||
|
|
||||||
b.ToTable("PairingCodes");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Settings.SystemSettings", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<int>("DefaultIdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultModel")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultSystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultVoice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.ToTable("SystemSettings");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("OwnerUserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Devices.Device", null)
|
|
||||||
.WithOne("Config")
|
|
||||||
.HasForeignKey("backend.Devices.DeviceConfig", "DeviceId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Navigation("Config");
|
|
||||||
});
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
public partial class AddPairingCodes : Migration
|
|
||||||
{
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.CreateTable(
|
|
||||||
name: "PairingCodes",
|
|
||||||
columns: table => new
|
|
||||||
{
|
|
||||||
Code = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
OwnerUserId = table.Column<Guid>(type: "TEXT", nullable: false),
|
|
||||||
DeviceName = table.Column<string>(type: "TEXT", nullable: false),
|
|
||||||
ExpiresAt = table.Column<DateTimeOffset>(type: "TEXT", nullable: false),
|
|
||||||
ConsumedAt = table.Column<DateTimeOffset>(type: "TEXT", nullable: true)
|
|
||||||
},
|
|
||||||
constraints: table =>
|
|
||||||
{
|
|
||||||
table.PrimaryKey("PK_PairingCodes", x => x.Code);
|
|
||||||
});
|
|
||||||
|
|
||||||
migrationBuilder.CreateIndex(
|
|
||||||
name: "IX_PairingCodes_ExpiresAt",
|
|
||||||
table: "PairingCodes",
|
|
||||||
column: "ExpiresAt");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
|
||||||
migrationBuilder.DropTable(
|
|
||||||
name: "PairingCodes");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,402 +0,0 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
|
||||||
using backend.Data;
|
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
namespace backend.Migrations
|
|
||||||
{
|
|
||||||
[DbContext(typeof(AppDbContext))]
|
|
||||||
partial class AppDbContextModelSnapshot : ModelSnapshot
|
|
||||||
{
|
|
||||||
protected override void BuildModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
#pragma warning disable 612, 618
|
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.17");
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("RoleNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetRoleClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimType")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ClaimValue")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserClaims", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderKey")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("ProviderDisplayName")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("LoginProvider", "ProviderKey");
|
|
||||||
|
|
||||||
b.HasIndex("UserId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserLogins", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("RoleId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "RoleId");
|
|
||||||
|
|
||||||
b.HasIndex("RoleId");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserRoles", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("UserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("LoginProvider")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Value")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("UserId", "LoginProvider", "Name");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUserTokens", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Auth.ApplicationUser", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("AccessFailedCount")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("ConcurrencyStamp")
|
|
||||||
.IsConcurrencyToken()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Email")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("EmailConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<bool>("LockoutEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LockoutEnd")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedEmail")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("NormalizedUserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PasswordHash")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("PhoneNumber")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("PhoneNumberConfirmed")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("SecurityStamp")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("TwoFactorEnabled")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("UserName")
|
|
||||||
.HasMaxLength(256)
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedEmail")
|
|
||||||
.HasDatabaseName("EmailIndex");
|
|
||||||
|
|
||||||
b.HasIndex("NormalizedUserName")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UserNameIndex");
|
|
||||||
|
|
||||||
b.ToTable("AspNetUsers", (string)null);
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("Id")
|
|
||||||
.ValueGeneratedOnAdd()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<bool>("IsRevoked")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("LastSeenAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Name")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("PairedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("TokenHash")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.HasIndex("OwnerUserId");
|
|
||||||
|
|
||||||
b.HasIndex("TokenHash")
|
|
||||||
.IsUnique();
|
|
||||||
|
|
||||||
b.ToTable("Devices");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.Property<Guid>("DeviceId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("EnabledToolsJson")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<int>("IdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("Model")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("SystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("Voice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("DeviceId");
|
|
||||||
|
|
||||||
b.ToTable("DeviceConfigs");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Pairing.PairingCode", b =>
|
|
||||||
{
|
|
||||||
b.Property<string>("Code")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset?>("ConsumedAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DeviceName")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<DateTimeOffset>("ExpiresAt")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<Guid>("OwnerUserId")
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Code");
|
|
||||||
|
|
||||||
b.HasIndex("ExpiresAt");
|
|
||||||
|
|
||||||
b.ToTable("PairingCodes");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Settings.SystemSettings", b =>
|
|
||||||
{
|
|
||||||
b.Property<int>("Id")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<int>("DefaultIdleTimeoutSeconds")
|
|
||||||
.HasColumnType("INTEGER");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultModel")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultSystemPrompt")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.Property<string>("DefaultVoice")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
b.HasKey("Id");
|
|
||||||
|
|
||||||
b.ToTable("SystemSettings");
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole<System.Guid>", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("RoleId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken<System.Guid>", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("UserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Auth.ApplicationUser", null)
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("OwnerUserId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.DeviceConfig", b =>
|
|
||||||
{
|
|
||||||
b.HasOne("backend.Devices.Device", null)
|
|
||||||
.WithOne("Config")
|
|
||||||
.HasForeignKey("backend.Devices.DeviceConfig", "DeviceId")
|
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
|
||||||
.IsRequired();
|
|
||||||
});
|
|
||||||
|
|
||||||
modelBuilder.Entity("backend.Devices.Device", b =>
|
|
||||||
{
|
|
||||||
b.Navigation("Config");
|
|
||||||
});
|
|
||||||
#pragma warning restore 612, 618
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
namespace backend.Pairing;
|
|
||||||
|
|
||||||
public class PairingCode
|
|
||||||
{
|
|
||||||
public string Code { get; set; } = ""; // 8 chars, primary key
|
|
||||||
public Guid OwnerUserId { get; set; }
|
|
||||||
public string DeviceName { get; set; } = "";
|
|
||||||
public DateTimeOffset ExpiresAt { get; set; }
|
|
||||||
public DateTimeOffset? ConsumedAt { get; set; }
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
using System.Security.Claims;
|
|
||||||
using Microsoft.AspNetCore.Mvc;
|
|
||||||
|
|
||||||
namespace backend.Pairing;
|
|
||||||
|
|
||||||
public record IssueCodeRequest(string Name);
|
|
||||||
public record PairRequest(string Code, string Hostname, string Client_version);
|
|
||||||
|
|
||||||
public static class PairingEndpoints
|
|
||||||
{
|
|
||||||
public static IEndpointRouteBuilder MapPairingEndpoints(this IEndpointRouteBuilder app)
|
|
||||||
{
|
|
||||||
app.MapPost("/api/pair-code", async (
|
|
||||||
[FromBody] IssueCodeRequest req,
|
|
||||||
PairingService svc,
|
|
||||||
HttpContext http,
|
|
||||||
CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var uidStr = http.User.FindFirstValue(ClaimTypes.NameIdentifier);
|
|
||||||
if (!Guid.TryParse(uidStr, out var uid)) return Results.Unauthorized();
|
|
||||||
if (string.IsNullOrWhiteSpace(req.Name))
|
|
||||||
return Results.BadRequest(new { error = "name is required" });
|
|
||||||
var pc = await svc.IssueAsync(uid, req.Name.Trim(), ct);
|
|
||||||
return Results.Ok(new { code = pc.Code, expires_at = pc.ExpiresAt });
|
|
||||||
}).RequireAuthorization();
|
|
||||||
|
|
||||||
app.MapPost("/api/pair", async (
|
|
||||||
[FromBody] PairRequest req,
|
|
||||||
PairingService svc,
|
|
||||||
HttpContext http,
|
|
||||||
CancellationToken ct) =>
|
|
||||||
{
|
|
||||||
var result = await svc.ConsumeAsync(req.Code, ct);
|
|
||||||
if (result is null) return Results.NotFound(new { error = "code invalid or expired" });
|
|
||||||
var scheme = http.Request.Scheme == "https" ? "wss" : "ws";
|
|
||||||
var host = http.Request.Host.ToString();
|
|
||||||
return Results.Ok(new
|
|
||||||
{
|
|
||||||
device_id = result.DeviceId,
|
|
||||||
device_token = result.DeviceToken,
|
|
||||||
backend_ws = $"{scheme}://{host}/device",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return app;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
using System.Security.Cryptography;
|
|
||||||
using backend.Data;
|
|
||||||
using backend.Devices;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
|
|
||||||
namespace backend.Pairing;
|
|
||||||
|
|
||||||
public record PairResult(Guid DeviceId, string DeviceToken);
|
|
||||||
|
|
||||||
public class PairingService
|
|
||||||
{
|
|
||||||
private const string Alphabet = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; // no O, 0, I, 1, L
|
|
||||||
private const int CodeLength = 8;
|
|
||||||
public static readonly TimeSpan CodeTtl = TimeSpan.FromMinutes(10);
|
|
||||||
|
|
||||||
private readonly AppDbContext? _db;
|
|
||||||
private readonly DeviceTokenService? _tokens;
|
|
||||||
|
|
||||||
public PairingService() { }
|
|
||||||
|
|
||||||
public PairingService(AppDbContext db, DeviceTokenService tokens)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_tokens = tokens;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string GenerateCode()
|
|
||||||
{
|
|
||||||
Span<byte> bytes = stackalloc byte[CodeLength];
|
|
||||||
RandomNumberGenerator.Fill(bytes);
|
|
||||||
Span<char> chars = stackalloc char[CodeLength];
|
|
||||||
for (var i = 0; i < CodeLength; i++)
|
|
||||||
chars[i] = Alphabet[bytes[i] % Alphabet.Length];
|
|
||||||
return new string(chars);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<PairingCode> IssueAsync(Guid ownerUserId, string deviceName, CancellationToken ct)
|
|
||||||
{
|
|
||||||
if (_db is null) throw new InvalidOperationException("PairingService not constructed with DB context.");
|
|
||||||
var entity = new PairingCode
|
|
||||||
{
|
|
||||||
Code = GenerateCode(),
|
|
||||||
OwnerUserId = ownerUserId,
|
|
||||||
DeviceName = deviceName,
|
|
||||||
ExpiresAt = DateTimeOffset.UtcNow.Add(CodeTtl),
|
|
||||||
};
|
|
||||||
_db.PairingCodes.Add(entity);
|
|
||||||
await _db.SaveChangesAsync(ct);
|
|
||||||
return entity;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<PairResult?> ConsumeAsync(string code, CancellationToken ct)
|
|
||||||
{
|
|
||||||
if (_db is null || _tokens is null)
|
|
||||||
throw new InvalidOperationException("PairingService not constructed with DB context + token service.");
|
|
||||||
var now = DateTimeOffset.UtcNow;
|
|
||||||
var pc = await _db.PairingCodes.FirstOrDefaultAsync(p => p.Code == code, ct);
|
|
||||||
if (pc is null || pc.ConsumedAt is not null || pc.ExpiresAt < now)
|
|
||||||
return null;
|
|
||||||
|
|
||||||
var (plaintext, hash) = _tokens.Generate();
|
|
||||||
var defaults = await _db.SystemSettings.FirstAsync(ct);
|
|
||||||
|
|
||||||
var device = new Device
|
|
||||||
{
|
|
||||||
OwnerUserId = pc.OwnerUserId,
|
|
||||||
Name = pc.DeviceName,
|
|
||||||
PairedAt = now,
|
|
||||||
TokenHash = hash,
|
|
||||||
};
|
|
||||||
device.Config = new DeviceConfig
|
|
||||||
{
|
|
||||||
DeviceId = device.Id,
|
|
||||||
SystemPrompt = defaults.DefaultSystemPrompt,
|
|
||||||
Voice = defaults.DefaultVoice,
|
|
||||||
Model = defaults.DefaultModel,
|
|
||||||
IdleTimeoutSeconds = defaults.DefaultIdleTimeoutSeconds,
|
|
||||||
};
|
|
||||||
_db.Devices.Add(device);
|
|
||||||
pc.ConsumedAt = now;
|
|
||||||
await _db.SaveChangesAsync(ct);
|
|
||||||
return new PairResult(device.Id, plaintext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
using backend.Auth;
|
|
||||||
using backend.Data;
|
|
||||||
using backend.Devices;
|
|
||||||
using backend.Install;
|
|
||||||
using backend.Pairing;
|
|
||||||
using Microsoft.AspNetCore.DataProtection;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
|
||||||
|
|
||||||
var keysDir = Environment.GetEnvironmentVariable("DataProtection__KeysDir") ?? "/keys";
|
|
||||||
if (Directory.Exists(keysDir))
|
|
||||||
{
|
|
||||||
builder.Services.AddDataProtection()
|
|
||||||
.PersistKeysToFileSystem(new DirectoryInfo(keysDir))
|
|
||||||
.SetApplicationName("smart-assistant");
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.Services.AddDbContext<AppDbContext>(opt =>
|
|
||||||
opt.UseSqlite(builder.Configuration.GetConnectionString("Default")));
|
|
||||||
|
|
||||||
builder.Services.AddAppIdentity();
|
|
||||||
builder.Services.AddAuthorization();
|
|
||||||
|
|
||||||
builder.Services.AddScoped<backend.Devices.DeviceTokenService>();
|
|
||||||
builder.Services.AddScoped<backend.Pairing.PairingService>();
|
|
||||||
builder.Services.AddSingleton<backend.Install.InstallScriptBuilder>();
|
|
||||||
|
|
||||||
var app = builder.Build();
|
|
||||||
|
|
||||||
using (var scope = app.Services.CreateScope())
|
|
||||||
{
|
|
||||||
var sp = scope.ServiceProvider;
|
|
||||||
var db = sp.GetRequiredService<AppDbContext>();
|
|
||||||
db.Database.Migrate();
|
|
||||||
await IdentitySetup.SeedRolesAsync(sp);
|
|
||||||
|
|
||||||
if (db.SystemSettings.Find(1) is null)
|
|
||||||
{
|
|
||||||
db.SystemSettings.Add(new backend.Settings.SystemSettings { Id = 1 });
|
|
||||||
await db.SaveChangesAsync();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
app.UseAuthentication();
|
|
||||||
app.UseAuthorization();
|
|
||||||
|
|
||||||
app.MapAuthEndpoints();
|
|
||||||
app.MapPairingEndpoints();
|
|
||||||
app.MapDevicesEndpoints();
|
|
||||||
app.MapInstallEndpoints();
|
|
||||||
|
|
||||||
app.MapGet("/health", () => Results.Ok(new { ok = true }));
|
|
||||||
|
|
||||||
app.Run();
|
|
||||||
|
|
||||||
public partial class Program { }
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json.schemastore.org/launchsettings.json",
|
|
||||||
"profiles": {
|
|
||||||
"http": {
|
|
||||||
"commandName": "Project",
|
|
||||||
"dotnetRunMessages": true,
|
|
||||||
"launchBrowser": true,
|
|
||||||
"applicationUrl": "http://localhost:5252",
|
|
||||||
"environmentVariables": {
|
|
||||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"https": {
|
|
||||||
"commandName": "Project",
|
|
||||||
"dotnetRunMessages": true,
|
|
||||||
"launchBrowser": true,
|
|
||||||
"applicationUrl": "https://localhost:7061;http://localhost:5252",
|
|
||||||
"environmentVariables": {
|
|
||||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace backend.Settings;
|
|
||||||
|
|
||||||
public class SystemSettings
|
|
||||||
{
|
|
||||||
public int Id { get; set; } = 1;
|
|
||||||
public string DefaultSystemPrompt { get; set; } =
|
|
||||||
"You are a helpful voice assistant. Keep responses concise.";
|
|
||||||
public string DefaultVoice { get; set; } = "alloy";
|
|
||||||
public string DefaultModel { get; set; } = "gpt-4o-realtime-preview";
|
|
||||||
public int DefaultIdleTimeoutSeconds { get; set; } = 30;
|
|
||||||
}
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"Logging": {
|
|
||||||
"LogLevel": {
|
|
||||||
"Default": "Information",
|
|
||||||
"Microsoft.AspNetCore": "Warning"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
{
|
|
||||||
"ConnectionStrings": {
|
|
||||||
"Default": "Data Source=./assistant.db"
|
|
||||||
},
|
|
||||||
"Logging": {
|
|
||||||
"LogLevel": {
|
|
||||||
"Default": "Information",
|
|
||||||
"Microsoft.AspNetCore": "Warning"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"AllowedHosts": "*"
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net9.0</TargetFramework>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="9.*" />
|
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="9.*" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.*">
|
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
|
||||||
<PrivateAssets>all</PrivateAssets>
|
|
||||||
</PackageReference>
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.*" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
Executable
+25
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/01-record-play-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs && mkdir ~/probe-cs'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs && chmod +x Probe && ./Probe'
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-2-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/02-wakeword-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-2/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-2 && mkdir ~/probe-cs-2'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-2/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-2 && chmod +x Probe && ./Probe'
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-3-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/03-full-cycle-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-3/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-3 && mkdir ~/probe-cs-3'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-3/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-3 && chmod +x Probe && ./Probe'
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-4a-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> precondition: ~/.openai_key present on Pi"
|
||||||
|
if ! sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'test -f ~/.openai_key && test -r ~/.openai_key'; then
|
||||||
|
echo "ERROR: ~/.openai_key missing or unreadable on Pi."
|
||||||
|
echo "Create it on the Pi with:"
|
||||||
|
echo " sshpass -p '$PI_PASS' ssh $PI_USER@$PI_HOST \\"
|
||||||
|
echo " 'umask 077 && printf \"%s\\n\" \"sk-PASTE-YOUR-KEY\" > ~/.openai_key'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " OK."
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/04a-realtime-oneshot-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-4a/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-4a && mkdir ~/probe-cs-4a'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-4a/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-4a && chmod +x Probe && ./Probe'
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
"""Smart Assistant client entry point. Implemented in Plan 3."""
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
raise SystemExit("client.main: placeholder; full implementation comes in Plan 3.")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
"""Smart Assistant pairing CLI. Implemented in Plan 3."""
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
raise SystemExit("client.pair: placeholder; full implementation comes in Plan 3.")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
services:
|
|
||||||
app:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
expose:
|
|
||||||
- "8080"
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_URLS=http://+:8080
|
|
||||||
- 'ConnectionStrings__Default=Data Source=/data/assistant.db'
|
|
||||||
- Install__ClientTarballPath=/app/wwwroot/client.tar.gz
|
|
||||||
- SERVICE_FQDN_APP_8080
|
|
||||||
volumes:
|
|
||||||
- assistant-data:/data
|
|
||||||
- assistant-keys:/keys
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
assistant-data:
|
|
||||||
assistant-keys:
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,570 @@
|
|||||||
|
# Record-play C# probe — implementation plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Re-implement Test 1 (record 5 s from the USB Speaker Phone → live RMS meter → write `out.wav` → play it back) in C# on the Raspberry Pi, proving .NET-driven PortAudio works on linux-arm64 with the same pass criteria the Python version met.
|
||||||
|
|
||||||
|
**Architecture:** Single-folder throwaway probe under `tests/01-record-play-cs/`. One `Program.cs` driving PortAudio via the PortAudioSharp2 NuGet binding, plus a tiny hand-rolled `WavWriter.cs` for the PCM16 RIFF file. A new `bin/probe-cs` script builds it as a linux-arm64 self-contained binary, rsyncs to the Pi, and runs it over SSH. The Pi gets no .NET install. No automated tests — verification is the same listening test the Python probe uses (`findings.md`).
|
||||||
|
|
||||||
|
**Tech Stack:** .NET 9, C#, PortAudioSharp2 (NuGet), bash, sshpass, rsync. Target host: Pi at `192.168.50.115`, user `pi`, password `assistant` (see `CLAUDE.md`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File structure
|
||||||
|
|
||||||
|
| Path | Created in task | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `tests/01-record-play-cs/Probe.csproj` | Task 1 | net9.0 console project, linux-arm64 RID, PortAudioSharp2 reference |
|
||||||
|
| `tests/01-record-play-cs/Program.cs` | Task 1 (sanity spike), expanded in Tasks 4 & 5 | Single entry point; mirrors the Python probe's structure |
|
||||||
|
| `tests/01-record-play-cs/WavWriter.cs` | Task 3 | Static `Write(path, samples, sampleRate)` — 44-byte RIFF + raw PCM16 LE |
|
||||||
|
| `bin/probe-cs` | Task 2 | `dotnet publish` → `rsync` → `ssh` flow |
|
||||||
|
|
||||||
|
`tests/01-record-play/` (Python) and the rest of the repo are untouched.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: Scaffold project + PortAudio sanity spike on the Pi
|
||||||
|
|
||||||
|
**Goal:** Stand up the .NET project and prove on the Pi that `PortAudio.Initialize()` + device enumeration work end-to-end from a self-contained linux-arm64 publish. No audio I/O yet. This de-risks the unknown the spec called out: does PortAudioSharp2's NuGet ship a `libportaudio` for linux-arm64, or does it dlopen the system one?
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/01-record-play-cs/Probe.csproj`
|
||||||
|
- Create: `tests/01-record-play-cs/Program.cs`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Find the latest stable PortAudioSharp2 version**
|
||||||
|
|
||||||
|
Open `https://www.nuget.org/packages/PortAudioSharp2` in a browser (or `curl -s https://api.nuget.org/v3-flatcontainer/portaudiosharp2/index.json | jq -r '.versions[-1]'`). Note the latest non-prerelease version (call it `<PASHARP_VER>` below). Pin that exact version in the csproj — do **not** use a floating `*`. If `curl/jq` are not available, the NuGet web page lists "Latest version" at the top right.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Write `tests/01-record-play-cs/Probe.csproj`**
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>RecordPlayProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="<PASHARP_VER>" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `<PASHARP_VER>` with the version from Step 1.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write `tests/01-record-play-cs/Program.cs` (spike version)**
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using PortAudioSharp;
|
||||||
|
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
Console.WriteLine($"PortAudio version: {PortAudio.VersionInfo.versionText}");
|
||||||
|
Console.WriteLine($"Devices ({count}):");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels} " +
|
||||||
|
$"defaultSr={info.defaultSampleRate}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Note for the engineer:** The above uses the API surface as documented in PortAudioSharp2's README at time of writing (`PortAudio.Initialize/Terminate`, `PortAudio.DeviceCount`, `PortAudio.GetDeviceInfo(int)`, `PortAudio.VersionInfo`). If a compile error reveals the surface has changed, read PortAudioSharp2's README at the version you pinned in Step 1 and adjust the calls — the intent is "init, dump device table, terminate". Do **not** change to a different binding.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Build and publish locally**
|
||||||
|
|
||||||
|
Run from the repo root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet publish tests/01-record-play-cs -c Release -r linux-arm64 --self-contained \
|
||||||
|
-o /tmp/probe-cs-out
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: build succeeds. Verify the output contains the native PortAudio binary:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ls /tmp/probe-cs-out | grep -E 'libportaudio|Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Probe` (the binary) is present. `libportaudio.so*` may or may not be present — if absent, PortAudioSharp2 will dlopen the system `libportaudio2` already installed by `bin/bootstrap` on the Pi.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Push the spike to the Pi and run it**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 'rm -rf ~/probe-cs && mkdir ~/probe-cs'
|
||||||
|
sshpass -p 'assistant' scp -r /tmp/probe-cs-out/. pi@192.168.50.115:~/probe-cs/
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 'cd ~/probe-cs && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
(We use `scp -r` instead of `rsync` because the build sandbox does not have rsync installed. The remote `rm -rf` + `mkdir` gives us the `--delete` semantics we want.)
|
||||||
|
|
||||||
|
Expected output: PortAudio version string + a device table that includes at least one entry whose name contains `USB` (the Anhui LISTENAI USB Speaker Phone — exposed as ALSA card 3 per `findings.md`). If you see `dlopen` failures for `libportaudio`, install it on the Pi: `sshpass -p 'assistant' ssh pi@192.168.50.115 'sudo apt-get install -y libportaudio2'` (the prototype `bin/bootstrap` already does this, but the Pi may have been wiped).
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/01-record-play-cs/Probe.csproj tests/01-record-play-cs/Program.cs
|
||||||
|
git commit -m "Scaffold C# record-play probe + PortAudio device-list spike"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: Wrap the deploy flow as `bin/probe-cs`
|
||||||
|
|
||||||
|
**Goal:** Capture the Task 1 build/rsync/ssh sequence as a single script so subsequent tasks just run `bin/probe-cs`.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `bin/probe-cs`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write `bin/probe-cs`**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/01-record-play-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs && mkdir ~/probe-cs'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
(Using `scp -r` rather than `rsync` because the build sandbox does not have rsync installed. The remote `rm -rf` + `mkdir` gives us the `--delete` semantics we want.)
|
||||||
|
|
||||||
|
- [ ] **Step 2: Mark it executable**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
chmod +x bin/probe-cs
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Smoke-test the script against Task 1's spike**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output: same device table as in Task 1 Step 5. If it errors, fix the script — do not work around it in later tasks.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add bin/probe-cs
|
||||||
|
git commit -m "Add bin/probe-cs: build + rsync + run the C# probe on the Pi"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: `WavWriter.cs` — hand-rolled PCM16 RIFF writer
|
||||||
|
|
||||||
|
**Goal:** A static helper that writes a mono PCM16 WAV file. No external deps.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/01-record-play-cs/WavWriter.cs`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write `tests/01-record-play-cs/WavWriter.cs`**
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
namespace RecordPlayProbe;
|
||||||
|
|
||||||
|
internal static class WavWriter
|
||||||
|
{
|
||||||
|
public static void Write(string path, ReadOnlySpan<short> samples, int sampleRate)
|
||||||
|
{
|
||||||
|
const short channels = 1;
|
||||||
|
const short bitsPerSample = 16;
|
||||||
|
const short pcmFormat = 1;
|
||||||
|
|
||||||
|
int byteRate = sampleRate * channels * bitsPerSample / 8;
|
||||||
|
short blockAlign = (short)(channels * bitsPerSample / 8);
|
||||||
|
int dataBytes = samples.Length * sizeof(short);
|
||||||
|
int riffChunkSize = 36 + dataBytes;
|
||||||
|
|
||||||
|
using var fs = new FileStream(path, FileMode.Create, FileAccess.Write);
|
||||||
|
using var bw = new BinaryWriter(fs);
|
||||||
|
|
||||||
|
// RIFF header
|
||||||
|
bw.Write(new[] { (byte)'R', (byte)'I', (byte)'F', (byte)'F' });
|
||||||
|
bw.Write(riffChunkSize);
|
||||||
|
bw.Write(new[] { (byte)'W', (byte)'A', (byte)'V', (byte)'E' });
|
||||||
|
|
||||||
|
// fmt chunk
|
||||||
|
bw.Write(new[] { (byte)'f', (byte)'m', (byte)'t', (byte)' ' });
|
||||||
|
bw.Write(16); // fmt chunk size for PCM
|
||||||
|
bw.Write(pcmFormat);
|
||||||
|
bw.Write(channels);
|
||||||
|
bw.Write(sampleRate);
|
||||||
|
bw.Write(byteRate);
|
||||||
|
bw.Write(blockAlign);
|
||||||
|
bw.Write(bitsPerSample);
|
||||||
|
|
||||||
|
// data chunk
|
||||||
|
bw.Write(new[] { (byte)'d', (byte)'a', (byte)'t', (byte)'a' });
|
||||||
|
bw.Write(dataBytes);
|
||||||
|
foreach (short s in samples)
|
||||||
|
bw.Write(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`BinaryWriter` writes integers little-endian on every supported platform — that matches the WAV spec. No byte-order conversion needed.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Build (no test step — spec says no automated tests)**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/01-record-play-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/01-record-play-cs/WavWriter.cs
|
||||||
|
git commit -m "Add WavWriter: mono PCM16 RIFF writer for the C# probe"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: Recording with live RMS meter + WAV write
|
||||||
|
|
||||||
|
**Goal:** Replace the spike `Program.cs` with the real record + meter + write flow. No playback yet — we want to confirm recording independently of playback so a regression on either is unambiguous.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/01-record-play-cs/Program.cs` (rewrite — the spike code is replaced)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Rewrite `tests/01-record-play-cs/Program.cs`**
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using PortAudioSharp;
|
||||||
|
|
||||||
|
const int SampleRate = 16_000;
|
||||||
|
const int Channels = 1;
|
||||||
|
const int DurationS = 5;
|
||||||
|
const uint BlockFrames = 1024;
|
||||||
|
const string OutWav = "out.wav";
|
||||||
|
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine(
|
||||||
|
$"Recording {DurationS}s from device {device} " +
|
||||||
|
$"('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
short[] buf = new short[SampleRate * DurationS];
|
||||||
|
int writeOffset = 0;
|
||||||
|
object gate = new();
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
int remaining = buf.Length - writeOffset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &buf[writeOffset])
|
||||||
|
{
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeOffset += take;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
|
||||||
|
var start = DateTime.UtcNow;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
int written;
|
||||||
|
lock (gate) { written = writeOffset; }
|
||||||
|
if (written >= buf.Length) break;
|
||||||
|
|
||||||
|
double level = Rms(buf, Math.Max(0, written - 1600), written);
|
||||||
|
int bars = (int)(level * 60);
|
||||||
|
double elapsed = (DateTime.UtcNow - start).TotalSeconds;
|
||||||
|
Console.Write($"\r{elapsed,4:0.0}s |{new string('#', bars).PadRight(60)}|");
|
||||||
|
Thread.Sleep(50);
|
||||||
|
}
|
||||||
|
Console.WriteLine();
|
||||||
|
stream.Stop();
|
||||||
|
|
||||||
|
WavWriter.Write(OutWav, buf, SampleRate);
|
||||||
|
Console.WriteLine($"Wrote {OutWav} ({buf.Length} frames @ {SampleRate} Hz)");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static double Rms(short[] data, int from, int to)
|
||||||
|
{
|
||||||
|
if (to <= from) return 0.0;
|
||||||
|
double sumSq = 0.0;
|
||||||
|
for (int i = from; i < to; i++)
|
||||||
|
{
|
||||||
|
double x = data[i] / 32768.0;
|
||||||
|
sumSq += x * x;
|
||||||
|
}
|
||||||
|
return Math.Sqrt(sumSq / (to - from) + 1e-12);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**RMS formula matches the Python probe** (`np.sqrt((x*x).mean() + 1e-12)`). **Tail window matches** (last 1600 samples, ~100 ms at 16 kHz). **Meter cadence matches** (50 ms sleep → ~20 Hz refresh).
|
||||||
|
|
||||||
|
The csproj also needs `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>` for the `unsafe` block in the callback. Add it:
|
||||||
|
|
||||||
|
- [ ] **Step 2: Enable unsafe blocks in `Probe.csproj`**
|
||||||
|
|
||||||
|
Add `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>` inside the existing `<PropertyGroup>`. The full PropertyGroup should now read:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>RecordPlayProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Build locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/01-record-play-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success. If you hit a `StreamParameters` / `Stream` / `Callback` API mismatch, see PortAudioSharp2's README at the version you pinned in Task 1 and adjust the type names — the structure (struct of device + channelCount + sampleFormat + latency; Stream constructor that takes input params, output params, sample rate, frames-per-buffer, flags, callback, user data) is the conventional PortAudio shape and should map closely.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run on the Pi**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected on the Pi:
|
||||||
|
- A line `Recording 5s from device <N> ('USB...')`.
|
||||||
|
- A live meter: `\r 0.0s |######...|` updating ~20×/sec for 5 seconds. The bar visibly responds to speech vs. silence.
|
||||||
|
- A final line `Wrote out.wav (80000 frames @ 16000 Hz)`.
|
||||||
|
- No sustained `[status] input overflow` messages (one at startup is acceptable).
|
||||||
|
|
||||||
|
Confirm the WAV is well-formed:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 'soxi ~/probe-cs/out.wav'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `soxi` reports 1 channel, 16000 Hz, 16-bit PCM, ~5 seconds duration. (If `soxi` isn't installed: `sudo apt-get install -y sox` on the Pi, or use `file out.wav` which will at least say `RIFF (little-endian) data, WAVE audio, Microsoft PCM, 16 bit, mono 16000 Hz`.)
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/01-record-play-cs/Probe.csproj tests/01-record-play-cs/Program.cs
|
||||||
|
git commit -m "Record loop: callback InputStream + RMS meter + WAV write"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: Playback + full pass-criteria verification
|
||||||
|
|
||||||
|
**Goal:** Add the playback step and verify all three pass criteria from the spec on real hardware.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/01-record-play-cs/Program.cs` (append playback after the WAV write)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add playback to `Program.cs`**
|
||||||
|
|
||||||
|
Insert this block immediately after the `Console.WriteLine($"Wrote {OutWav}...");` line and before the `finally` block:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
Console.WriteLine($"Playing back through device {device}");
|
||||||
|
|
||||||
|
int playOffset = 0;
|
||||||
|
object playGate = new();
|
||||||
|
var doneEvent = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
lock (playGate)
|
||||||
|
{
|
||||||
|
int remaining = buf.Length - playOffset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
int silence = (int)frameCount - take;
|
||||||
|
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &buf[playOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
playOffset += take;
|
||||||
|
}
|
||||||
|
if (silence > 0)
|
||||||
|
{
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (playOffset >= buf.Length)
|
||||||
|
{
|
||||||
|
doneEvent.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var playStream = new Stream(
|
||||||
|
null, outParams, SampleRate, BlockFrames, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
playStream.Start();
|
||||||
|
doneEvent.Wait();
|
||||||
|
// Give PortAudio a moment to drain the device buffer before Stop().
|
||||||
|
Thread.Sleep(200);
|
||||||
|
playStream.Stop();
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Build locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/01-record-play-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Run on the Pi and verify all three pass criteria**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs
|
||||||
|
```
|
||||||
|
|
||||||
|
Then **at the Pi's audio output** (the USB Speaker Phone is mic and speaker — playback comes out of the same device):
|
||||||
|
|
||||||
|
1. **Playback is recognisable as what was said into the mic.** Speak something distinct during the 5-second record window (e.g., "the quick brown fox jumps over the lazy dog") and confirm you can clearly understand the playback.
|
||||||
|
2. **The meter visibly responds to speech and to silence.** During the recording, alternate between speaking and silence; the `####` bar should clearly grow and shrink.
|
||||||
|
3. **No clipping, dropouts, or sustained PortAudio under/overrun warnings.** Scan the stderr output for `[status] input overflow` lines — a single one at start-up is acceptable, sustained ones are not. The playback should not have audible clicks, gaps, or chipmunk artefacts.
|
||||||
|
|
||||||
|
If any of the three fails: do not "fix" by tweaking thresholds or silencing logs — diagnose. Use `findings.md` ("Force PortAudio through ALSA's `plug` plugin") as the first reference; the `PA_ALSA_PLUGHW=1` env var must be in place before `PortAudio.Initialize()` runs.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/01-record-play-cs/Program.cs
|
||||||
|
git commit -m "Playback: append OutputStream playback of recorded buffer"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Note the probe outcome**
|
||||||
|
|
||||||
|
The probe is done. Append a short note to `findings.md` under a new section `## C# probe outcome (2026-06-12)` recording: (a) which PortAudioSharp2 version you pinned, (b) whether the NuGet bundled `libportaudio.so` or you used the Pi's system `libportaudio2`, (c) any API drift you had to adjust for vs. the plan, (d) whether all three pass criteria held. This belongs in `findings.md` because the next probe (Test 2 in C#) will benefit from it.
|
||||||
|
|
||||||
|
Commit:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add findings.md
|
||||||
|
git commit -m "Findings: C# record-play probe outcome"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Notes on what is intentionally **not** in this plan
|
||||||
|
|
||||||
|
- **No automated tests.** The spec is explicit; verification is the listening test on the Pi. WavWriter is verified transitively by the playback step in Task 5.
|
||||||
|
- **No queue / consumer thread.** The Python probe has the same shape (callback writes, main thread reads). The findings doc's "callback queue for input" rework belongs to the eventual main client, not this probe.
|
||||||
|
- **No cleanup on Ctrl-C mid-recording.** Probe. Crash trace is fine.
|
||||||
|
- **No retry on PortAudio init failure.** Crash and surface.
|
||||||
|
- **No abstraction over PortAudio.** No `IAudioDevice` interface, no DI. One file, one job.
|
||||||
|
|
||||||
|
If, during implementation, you feel the urge to add any of the above — stop and re-read the spec. The point of this probe is to be cheap and disposable.
|
||||||
@@ -0,0 +1,969 @@
|
|||||||
|
# Test 2 wakeword C# probe — implementation plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Re-implement Test 2 (listen for the "alexa" wakeword via openwakeword's stock model → beep on detection) in C# on the Raspberry Pi, by porting `openwakeword/model.py`'s streaming inference pipeline to C# / `Microsoft.ML.OnnxRuntime`. Pass the same hardware criteria as the Python version.
|
||||||
|
|
||||||
|
**Architecture:** Single-folder throwaway probe under `tests/02-wakeword-cs/`. `WakewordModel.cs` chains three vendored ONNX models (mel-spectrogram → Google speech embedding → alexa keyword classifier) with the same streaming buffer geometry as the Python implementation. `Program.cs` drives a PortAudio input stream into a bounded `BlockingCollection`; the main thread pulls frames, runs `WakewordModel.Predict`, and dispatches a fire-and-forget output stream for the detection beep so input ingestion never stalls. `bin/probe-cs-2` builds a self-contained linux-arm64 binary and ships it to the Pi.
|
||||||
|
|
||||||
|
**Tech Stack:** .NET 9, C#, PortAudioSharp2 (NuGet), Microsoft.ML.OnnxRuntime (NuGet), bash, sshpass. Target host: Pi at `192.168.50.115`, user `pi`, password `assistant` (see `CLAUDE.md`). The Pi gets no .NET install — the binary is self-contained.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File structure
|
||||||
|
|
||||||
|
| Path | Created in task | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `tests/02-wakeword-cs/Probe.csproj` | Task 1 | net9.0 console project; linux-arm64 RID; PortAudioSharp2 + Microsoft.ML.OnnxRuntime refs; bundles `models/*.onnx` to output. |
|
||||||
|
| `tests/02-wakeword-cs/models/melspectrogram.onnx` | Task 1 | Vendored from the Pi's openwakeword install (~14 KB). |
|
||||||
|
| `tests/02-wakeword-cs/models/embedding_model.onnx` | Task 1 | Vendored from the Pi (~10 MB), Google's speech embedding model. |
|
||||||
|
| `tests/02-wakeword-cs/models/alexa.onnx` | Task 1 | Vendored from the Pi (~1.5 MB), stock alexa classifier. |
|
||||||
|
| `tests/02-wakeword-cs/Program.cs` | Task 1 (spike), rewritten in Task 4 & extended in Task 5 | Entry point: env vars → PortAudio init → device pick → input stream → main inference loop → optional beep. |
|
||||||
|
| `tests/02-wakeword-cs/WakewordModel.cs` | Task 3 | Loads the three sessions; `Predict(short[] frame1280) → float`. |
|
||||||
|
| `bin/probe-cs-2` | Task 2 | `dotnet publish` → `scp` → `ssh -t` flow. |
|
||||||
|
|
||||||
|
`tests/02-wakeword/` (Python), `tests/01-record-play-cs/` (Test 1 C#), and `bin/probe-cs` are untouched.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: Scaffold project + vendor ONNX models + ORT sanity spike on the Pi
|
||||||
|
|
||||||
|
**Goal:** Stand up the .NET project with both NuGet refs, vendor the three openwakeword ONNX models into the repo, and prove on the Pi that `Microsoft.ML.OnnxRuntime` loads all three from a self-contained linux-arm64 publish. Capture each model's declared input / output shape (we'll use these as ground-truth assertions in Task 3).
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/02-wakeword-cs/Probe.csproj`
|
||||||
|
- Create: `tests/02-wakeword-cs/models/melspectrogram.onnx` (vendored)
|
||||||
|
- Create: `tests/02-wakeword-cs/models/embedding_model.onnx` (vendored)
|
||||||
|
- Create: `tests/02-wakeword-cs/models/alexa.onnx` (vendored)
|
||||||
|
- Create: `tests/02-wakeword-cs/Program.cs` (spike — replaced in Task 4)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Pin the Microsoft.ML.OnnxRuntime version**
|
||||||
|
|
||||||
|
Use Context7 to fetch the current Microsoft.ML.OnnxRuntime NuGet docs and identify the latest stable version (or check `https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime`):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -s https://api.nuget.org/v3-flatcontainer/microsoft.ml.onnxruntime/index.json | jq -r '.versions[]' | grep -v -E '(rc|preview|alpha|beta)' | tail -1
|
||||||
|
```
|
||||||
|
|
||||||
|
Note the exact version (call it `<ORT_VER>`). Pin this in the csproj — do NOT use a floating `*`.
|
||||||
|
|
||||||
|
Same drill for PortAudioSharp2 (already pinned at 1.0.6 in Test 1; use that exact version unless a newer non-prerelease has shipped):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -s https://api.nuget.org/v3-flatcontainer/portaudiosharp2/index.json | jq -r '.versions[-1]'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Vendor the three ONNX files from the Pi**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
mkdir -p tests/02-wakeword-cs/models
|
||||||
|
sshpass -p assistant scp \
|
||||||
|
pi@192.168.50.115:assistant/.venv/lib/python3.13/site-packages/openwakeword/resources/models/melspectrogram.onnx \
|
||||||
|
pi@192.168.50.115:assistant/.venv/lib/python3.13/site-packages/openwakeword/resources/models/embedding_model.onnx \
|
||||||
|
pi@192.168.50.115:assistant/.venv/lib/python3.13/site-packages/openwakeword/resources/models/alexa.onnx \
|
||||||
|
tests/02-wakeword-cs/models/
|
||||||
|
ls -l tests/02-wakeword-cs/models/
|
||||||
|
sha256sum tests/02-wakeword-cs/models/*.onnx
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: three files, roughly 14 KB / 10 MB / 1.5 MB. Record the SHA-256 hashes in your commit message in Step 8 — that pins the exact model files used.
|
||||||
|
|
||||||
|
If `scp` fails because the Pi's openwakeword venv lives elsewhere, run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p assistant ssh pi@192.168.50.115 'find ~ -path "*openwakeword/resources/models/*.onnx" 2>/dev/null'
|
||||||
|
```
|
||||||
|
|
||||||
|
and substitute the correct directory.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write `tests/02-wakeword-cs/Probe.csproj`**
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>WakewordProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="<ORT_VER>" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="models/*.onnx">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `<ORT_VER>` with the version from Step 1.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Write the spike `tests/02-wakeword-cs/Program.cs`**
|
||||||
|
|
||||||
|
This temporary Program.cs loads each model and prints its declared shapes — exactly what Task 3 needs to assert against. It is replaced in Task 4.
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
string[] modelFiles = { "melspectrogram.onnx", "embedding_model.onnx", "alexa.onnx" };
|
||||||
|
|
||||||
|
var opts = new SessionOptions
|
||||||
|
{
|
||||||
|
IntraOpNumThreads = 1,
|
||||||
|
InterOpNumThreads = 1,
|
||||||
|
LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR,
|
||||||
|
};
|
||||||
|
|
||||||
|
foreach (var f in modelFiles)
|
||||||
|
{
|
||||||
|
string path = Path.Combine(modelsDir, f);
|
||||||
|
Console.WriteLine($"== {f} ==");
|
||||||
|
using var session = new InferenceSession(path, opts);
|
||||||
|
foreach (var kv in session.InputMetadata)
|
||||||
|
{
|
||||||
|
var dims = string.Join(",", kv.Value.Dimensions);
|
||||||
|
Console.WriteLine($" input '{kv.Key}': shape=[{dims}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
}
|
||||||
|
foreach (var kv in session.OutputMetadata)
|
||||||
|
{
|
||||||
|
var dims = string.Join(",", kv.Value.Dimensions);
|
||||||
|
Console.WriteLine($" output '{kv.Key}': shape=[{dims}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Console.WriteLine("All three models loaded.");
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Build and publish locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet publish tests/02-wakeword-cs -c Release -r linux-arm64 --self-contained \
|
||||||
|
-o /tmp/probe-cs-2-out
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: build succeeds. Confirm the publish dir contains the binary, the three model files, and the bundled ONNX Runtime native:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ls /tmp/probe-cs-2-out | grep -E 'libonnxruntime|libportaudio|Probe'
|
||||||
|
ls /tmp/probe-cs-2-out/models/
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Probe`, `libonnxruntime.so` (or `.so.<version>`), `libportaudio.so` (or similar), plus the three `.onnx` files under `models/`. If `libonnxruntime.so` is missing, the `Microsoft.ML.OnnxRuntime` package for your version may have split the native runtime — install the matching native package (`Microsoft.ML.OnnxRuntime.Native.<rid>` if it exists for `<ORT_VER>`) or downgrade `<ORT_VER>` to a version where the linux-arm64 native is included in the main package. Do NOT proceed to Step 6 if the native is missing.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Push the spike to the Pi and run it**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p assistant ssh pi@192.168.50.115 'rm -rf ~/probe-cs-2 && mkdir ~/probe-cs-2'
|
||||||
|
sshpass -p assistant scp -r /tmp/probe-cs-2-out/. pi@192.168.50.115:~/probe-cs-2/
|
||||||
|
sshpass -p assistant ssh pi@192.168.50.115 'cd ~/probe-cs-2 && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output (numbers and names matching the openwakeword Python source from `/tmp/oww-utils.py`):
|
||||||
|
|
||||||
|
```
|
||||||
|
== melspectrogram.onnx ==
|
||||||
|
input '<name>': shape=[-1,-1] dtype=Single (1D audio, variable length)
|
||||||
|
output '<name>': shape=[1,-1,1,32] dtype=Single (mel: 32 bins, variable n_frames)
|
||||||
|
== embedding_model.onnx ==
|
||||||
|
input 'input_1': shape=[-1,76,32,1] dtype=Single (76 mel frames × 32 bins × 1 channel, variable batch)
|
||||||
|
output '<name>': shape=[-1,1,1,96] dtype=Single (96-d embedding per batch)
|
||||||
|
== alexa.onnx ==
|
||||||
|
input '<name>': shape=[-1,16,96] dtype=Single (16 embeddings × 96 dims, variable batch)
|
||||||
|
output '<name>': shape=[-1,1] dtype=Single (binary classifier score)
|
||||||
|
All three models loaded.
|
||||||
|
```
|
||||||
|
|
||||||
|
Negative dims (`-1`) mean "dynamic / batch dimension" — that's fine, we'll pass `1` at inference time.
|
||||||
|
|
||||||
|
**Save the actual printed output to your scratchpad** — the input/output names (`'input_1'` etc.) and exact dimension lists become the assertion values in Task 3 Step 4. The dimension numbers (76, 32, 16, 96) should match the Python source; if they don't, stop and investigate before proceeding.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Mark `bin/probe-cs-2` placeholder NOT created yet**
|
||||||
|
|
||||||
|
Task 2 creates the deploy script. For Task 1 we use the inline scp commands above. (Task 1 cannot depend on Task 2.)
|
||||||
|
|
||||||
|
- [ ] **Step 8: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/02-wakeword-cs/Probe.csproj tests/02-wakeword-cs/Program.cs tests/02-wakeword-cs/models/
|
||||||
|
git commit -m "Scaffold C# wakeword probe: vendored ONNX models + ORT sanity spike
|
||||||
|
|
||||||
|
Models sourced from openwakeword==0.6.0 on the Pi.
|
||||||
|
SHA-256:
|
||||||
|
melspectrogram.onnx <hash>
|
||||||
|
embedding_model.onnx <hash>
|
||||||
|
alexa.onnx <hash>"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: Wrap the deploy flow as `bin/probe-cs-2`
|
||||||
|
|
||||||
|
**Goal:** Capture the build / scp / ssh sequence as a single script so subsequent tasks just run `bin/probe-cs-2`. Mirrors `bin/probe-cs` with new paths and adds `ssh -t` so workstation Ctrl-C propagates as SIGINT to the long-running probe.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `bin/probe-cs-2`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write `bin/probe-cs-2`**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-2-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/02-wakeword-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-2/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-2 && mkdir ~/probe-cs-2'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-2/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-2 && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
The `ssh -t` (force pseudo-tty) makes Ctrl-C from the workstation propagate as SIGINT to the remote `./Probe` process — needed because Task 4's main loop runs until Ctrl-C. Test 1's `bin/probe-cs` didn't need it because the probe self-terminated after 5 s of recording + playback.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Mark it executable**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
chmod +x bin/probe-cs-2
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Smoke-test against Task 1's spike**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs-2
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: same shape-dump output as Task 1 Step 6, ending with `All three models loaded.` Ctrl-C is not needed because the spike exits on its own. If it errors, fix the script — do not work around in later tasks.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add bin/probe-cs-2
|
||||||
|
git commit -m "Add bin/probe-cs-2: build + scp + run the C# wakeword probe on the Pi"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: `WakewordModel.cs` — port openwakeword's streaming pipeline
|
||||||
|
|
||||||
|
**Goal:** A `WakewordModel` class that wraps the three ONNX sessions and exposes a single `Predict(short[] frame1280) → float` method. Internal state matches `openwakeword/AudioFeatures._streaming_features()` and `Model.predict()` from the openwakeword 0.6.0 source on the Pi (already fetched to `/tmp/model.py` and `/tmp/oww-utils.py` during plan-writing — re-fetch if needed: `sshpass -p assistant scp pi@192.168.50.115:assistant/.venv/lib/python3.13/site-packages/openwakeword/{model,utils}.py /tmp/`).
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/02-wakeword-cs/WakewordModel.cs`
|
||||||
|
- Modify: `tests/02-wakeword-cs/Program.cs` (replace spike with a load-and-print test)
|
||||||
|
|
||||||
|
### Pipeline geometry (derived from openwakeword/utils.py `AudioFeatures`)
|
||||||
|
|
||||||
|
Per call to `Predict(short[] frame1280)`:
|
||||||
|
|
||||||
|
1. **Raw-audio buffer.** Append the 1280 new samples to a ring of recent samples. Keep at least the last `1280 + 480 = 1760` samples (the +480 = `160 * 3` is the mel model's 30 ms warm-up context that openwakeword passes in via the `-n_samples-160*3:` slice in `_streaming_melspectrogram`).
|
||||||
|
2. **Mel stage.** Slice the last 1760 samples → run `melspectrogram.onnx` with input shape `(1, 1760)` dtype Single → output is `(1, n_frames, 1, 32)`; squeeze trailing dims to get `(n_frames, 32)` where `n_frames` ≈ 8 (`ceil(1760/160 - 3) = ceil(8) = 8`). Apply the openwakeword post-transform `x = x / 10 + 2` element-wise. Append the new mel frames to the mel-frame ring. Trim ring to the last 970 frames (= `10 * 97`, openwakeword's `melspectrogram_max_len`).
|
||||||
|
3. **Embedding stage.** Slice the last 76 mel frames from the ring → shape `(1, 76, 32, 1)`. Run `embedding_model.onnx` (input name `input_1`) → output shape `(1, 1, 1, 96)`; squeeze to a `float[96]` embedding. Append to the embedding ring. Trim ring to the last 120 entries (openwakeword's `feature_buffer_max_len`).
|
||||||
|
4. **Classifier stage.** Slice the last 16 embeddings from the ring → shape `(1, 16, 96)`. Run `alexa.onnx` → output shape `(1, 1)`; return `output[0, 0]` as the score.
|
||||||
|
5. **Initialization gate.** Return `0f` for the first 16 calls — until the embedding ring has 16 real entries, classifier output is meaningless. (openwakeword pre-populates with random-audio embeddings then zeros the first 5 scores; we do the simpler equivalent: skip outright until we have 16 real embeddings.)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Document the geometry in `WakewordModel.cs`**
|
||||||
|
|
||||||
|
Create `tests/02-wakeword-cs/WakewordModel.cs` and start with constants and the class skeleton:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||||
|
|
||||||
|
namespace WakewordProbe;
|
||||||
|
|
||||||
|
// Streaming wakeword inference port of openwakeword 0.6.0's predict pipeline.
|
||||||
|
// References (read alongside this file):
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_features (mel + embedding stages)
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_melspectrogram
|
||||||
|
// openwakeword/utils.py:AudioFeatures._get_embeddings
|
||||||
|
// openwakeword/model.py:Model.predict (classifier stage)
|
||||||
|
internal sealed class WakewordModel : IDisposable
|
||||||
|
{
|
||||||
|
// Audio I/O geometry — per-call input contract.
|
||||||
|
public const int FrameSamples = 1280; // 80 ms @ 16 kHz; each Predict() call.
|
||||||
|
public const int SampleRate = 16_000;
|
||||||
|
|
||||||
|
// Mel model: input is last (FrameSamples + MelContextSamples) raw samples
|
||||||
|
// -- the +480 is openwakeword's `-n_samples-160*3:` slice (3 hops of context).
|
||||||
|
public const int MelContextSamples = 480; // 160 * 3
|
||||||
|
public const int MelInputSamples = FrameSamples + MelContextSamples; // 1760
|
||||||
|
public const int MelBins = 32; // melspectrogram model output dim
|
||||||
|
public const int MelBufferMaxFrames = 970; // 10 * 97 (openwakeword `melspectrogram_max_len`)
|
||||||
|
|
||||||
|
// Embedding model: 76-mel-frame window in, 96-d embedding out.
|
||||||
|
public const int EmbeddingWindowMelFrames = 76;
|
||||||
|
public const int EmbeddingDim = 96;
|
||||||
|
public const int EmbeddingBufferMax = 120; // openwakeword `feature_buffer_max_len`
|
||||||
|
public const string EmbeddingInputName = "input_1"; // openwakeword convention; assert at startup
|
||||||
|
|
||||||
|
// Classifier: 16 embeddings in, scalar score out.
|
||||||
|
public const int ClassifierEmbeddings = 16;
|
||||||
|
|
||||||
|
// Skip the first N Predict() calls — buffer fill-up window.
|
||||||
|
public const int WarmupFrames = ClassifierEmbeddings; // 16 frames ≈ 1.28 s
|
||||||
|
|
||||||
|
private readonly InferenceSession _mel;
|
||||||
|
private readonly InferenceSession _emb;
|
||||||
|
private readonly InferenceSession _cls;
|
||||||
|
|
||||||
|
private readonly string _melInputName; // discovered at startup
|
||||||
|
private readonly string _clsInputName; // discovered at startup
|
||||||
|
|
||||||
|
private readonly short[] _rawRing = new short[MelInputSamples];
|
||||||
|
private int _rawRingFill = 0; // samples buffered (≤ MelInputSamples)
|
||||||
|
|
||||||
|
private readonly List<float[]> _melRing = new(MelBufferMaxFrames); // each entry is a length-32 mel frame
|
||||||
|
private readonly List<float[]> _embRing = new(EmbeddingBufferMax); // each entry is a length-96 embedding
|
||||||
|
|
||||||
|
private int _framesSeen = 0;
|
||||||
|
|
||||||
|
public WakewordModel(string melPath, string embeddingPath, string classifierPath)
|
||||||
|
{
|
||||||
|
var opts = new SessionOptions
|
||||||
|
{
|
||||||
|
IntraOpNumThreads = 1,
|
||||||
|
InterOpNumThreads = 1,
|
||||||
|
LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR,
|
||||||
|
};
|
||||||
|
|
||||||
|
_mel = new InferenceSession(melPath, opts);
|
||||||
|
_emb = new InferenceSession(embeddingPath, opts);
|
||||||
|
_cls = new InferenceSession(classifierPath, opts);
|
||||||
|
|
||||||
|
// Discover input names + assert shape geometry.
|
||||||
|
_melInputName = _mel.InputMetadata.Keys.Single();
|
||||||
|
|
||||||
|
AssertEmbeddingShape(_emb); // input_1: [batch, 76, 32, 1], dtype float
|
||||||
|
_clsInputName = _cls.InputMetadata.Keys.Single();
|
||||||
|
AssertClassifierShape(_cls); // [batch, 16, 96], dtype float
|
||||||
|
}
|
||||||
|
|
||||||
|
// ... methods follow in Step 2 ...
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_mel.Dispose();
|
||||||
|
_emb.Dispose();
|
||||||
|
_cls.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Implement the shape-assertion helpers**
|
||||||
|
|
||||||
|
Append inside the `WakewordModel` class:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
private static void AssertEmbeddingShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
if (!sess.InputMetadata.TryGetValue(EmbeddingInputName, out var meta))
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input named '{EmbeddingInputName}', got [{string.Join(",", sess.InputMetadata.Keys)}]");
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 76, 32, 1] — batch may be -1 (dynamic).
|
||||||
|
if (d.Length != 4 || d[1] != EmbeddingWindowMelFrames || d[2] != MelBins || d[3] != 1)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input shape [batch,{EmbeddingWindowMelFrames},{MelBins},1], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"embedding_model.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AssertClassifierShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
var inputName = sess.InputMetadata.Keys.Single();
|
||||||
|
var meta = sess.InputMetadata[inputName];
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 16, 96] — batch may be -1.
|
||||||
|
if (d.Length != 3 || d[1] != ClassifierEmbeddings || d[2] != EmbeddingDim)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"alexa.onnx: expected input shape [batch,{ClassifierEmbeddings},{EmbeddingDim}], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"alexa.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Implement `Predict`**
|
||||||
|
|
||||||
|
Append inside the `WakewordModel` class:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
public float Predict(short[] frame1280)
|
||||||
|
{
|
||||||
|
if (frame1280.Length != FrameSamples)
|
||||||
|
throw new ArgumentException($"Expected {FrameSamples} samples, got {frame1280.Length}");
|
||||||
|
|
||||||
|
// 1. Append 1280 new samples to the raw ring (shift older samples down if full).
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
int copyToFront = Math.Min(MelInputSamples - _rawRingFill, FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, _rawRingFill, copyToFront);
|
||||||
|
_rawRingFill += copyToFront;
|
||||||
|
if (copyToFront < FrameSamples)
|
||||||
|
{
|
||||||
|
// Shouldn't happen on the very first call, but defensive.
|
||||||
|
int leftover = FrameSamples - copyToFront;
|
||||||
|
Array.Copy(_rawRing, leftover, _rawRing, 0, MelInputSamples - leftover);
|
||||||
|
Array.Copy(frame1280, copyToFront, _rawRing, MelInputSamples - leftover, leftover);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Shift older samples left by FrameSamples, then append new at the tail.
|
||||||
|
Array.Copy(_rawRing, FrameSamples, _rawRing, 0, MelInputSamples - FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, MelInputSamples - FrameSamples, FrameSamples);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip everything until we have the full mel-context window primed.
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Mel stage: feed the full _rawRing as float32 (1, MelInputSamples) into mel model.
|
||||||
|
var melInputData = new float[MelInputSamples];
|
||||||
|
for (int i = 0; i < MelInputSamples; i++) melInputData[i] = _rawRing[i]; // int16 → float32, NO normalisation
|
||||||
|
var melInputTensor = new DenseTensor<float>(melInputData, new[] { 1, MelInputSamples });
|
||||||
|
using var melResults = _mel.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_melInputName, melInputTensor)
|
||||||
|
});
|
||||||
|
var melOut = melResults.First().AsTensor<float>(); // shape (1, n_frames, 1, 32)
|
||||||
|
|
||||||
|
// Apply openwakeword's `x / 10 + 2` transform and append each new frame to _melRing.
|
||||||
|
int nFrames = melOut.Dimensions[1];
|
||||||
|
for (int f = 0; f < nFrames; f++)
|
||||||
|
{
|
||||||
|
var bin = new float[MelBins];
|
||||||
|
for (int b = 0; b < MelBins; b++)
|
||||||
|
bin[b] = melOut[0, f, 0, b] / 10f + 2f;
|
||||||
|
_melRing.Add(bin);
|
||||||
|
}
|
||||||
|
if (_melRing.Count > MelBufferMaxFrames)
|
||||||
|
_melRing.RemoveRange(0, _melRing.Count - MelBufferMaxFrames);
|
||||||
|
|
||||||
|
// 3. Embedding stage: need ≥ 76 mel frames; slice the last 76 → (1, 76, 32, 1).
|
||||||
|
if (_melRing.Count < EmbeddingWindowMelFrames)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
var embInputData = new float[EmbeddingWindowMelFrames * MelBins];
|
||||||
|
int startMel = _melRing.Count - EmbeddingWindowMelFrames;
|
||||||
|
for (int f = 0; f < EmbeddingWindowMelFrames; f++)
|
||||||
|
Array.Copy(_melRing[startMel + f], 0, embInputData, f * MelBins, MelBins);
|
||||||
|
var embInputTensor = new DenseTensor<float>(embInputData, new[] { 1, EmbeddingWindowMelFrames, MelBins, 1 });
|
||||||
|
using var embResults = _emb.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(EmbeddingInputName, embInputTensor)
|
||||||
|
});
|
||||||
|
var embOut = embResults.First().AsTensor<float>(); // shape (1, 1, 1, 96)
|
||||||
|
var newEmb = new float[EmbeddingDim];
|
||||||
|
for (int i = 0; i < EmbeddingDim; i++) newEmb[i] = embOut[0, 0, 0, i];
|
||||||
|
_embRing.Add(newEmb);
|
||||||
|
if (_embRing.Count > EmbeddingBufferMax)
|
||||||
|
_embRing.RemoveRange(0, _embRing.Count - EmbeddingBufferMax);
|
||||||
|
|
||||||
|
_framesSeen++;
|
||||||
|
|
||||||
|
// 4. Warm-up + classifier stage.
|
||||||
|
if (_embRing.Count < ClassifierEmbeddings || _framesSeen <= WarmupFrames)
|
||||||
|
return 0f;
|
||||||
|
|
||||||
|
var clsInputData = new float[ClassifierEmbeddings * EmbeddingDim];
|
||||||
|
int startEmb = _embRing.Count - ClassifierEmbeddings;
|
||||||
|
for (int i = 0; i < ClassifierEmbeddings; i++)
|
||||||
|
Array.Copy(_embRing[startEmb + i], 0, clsInputData, i * EmbeddingDim, EmbeddingDim);
|
||||||
|
var clsInputTensor = new DenseTensor<float>(clsInputData, new[] { 1, ClassifierEmbeddings, EmbeddingDim });
|
||||||
|
using var clsResults = _cls.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_clsInputName, clsInputTensor)
|
||||||
|
});
|
||||||
|
var clsOut = clsResults.First().AsTensor<float>(); // shape (1, 1)
|
||||||
|
return clsOut[0, 0];
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Replace `Program.cs` spike with a load-only smoke test**
|
||||||
|
|
||||||
|
Replace `tests/02-wakeword-cs/Program.cs` with:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using WakewordProbe;
|
||||||
|
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
Console.WriteLine("Loading WakewordModel...");
|
||||||
|
var t0 = DateTime.UtcNow;
|
||||||
|
using var model = new WakewordModel(
|
||||||
|
Path.Combine(modelsDir, "melspectrogram.onnx"),
|
||||||
|
Path.Combine(modelsDir, "embedding_model.onnx"),
|
||||||
|
Path.Combine(modelsDir, "alexa.onnx"));
|
||||||
|
Console.WriteLine($"Loaded in {(DateTime.UtcNow - t0).TotalSeconds:0.00}s.");
|
||||||
|
|
||||||
|
// Smoke test: feed 50 frames of silence (1280 zero samples each).
|
||||||
|
// Expect every score to be 0f (warmup gate + no signal).
|
||||||
|
var silentFrame = new short[WakewordModel.FrameSamples];
|
||||||
|
int nonZero = 0;
|
||||||
|
for (int i = 0; i < 50; i++)
|
||||||
|
{
|
||||||
|
float s = model.Predict(silentFrame);
|
||||||
|
if (s != 0f) nonZero++;
|
||||||
|
}
|
||||||
|
Console.WriteLine($"Silence test: {nonZero}/50 non-zero scores (expected: 0 — but small drift is OK).");
|
||||||
|
Console.WriteLine("Predict pipeline ran without throwing.");
|
||||||
|
```
|
||||||
|
|
||||||
|
The "non-zero on silence" count may be small but not literally 0 — the classifier sees ~zero input but its output is rarely exactly 0f. Anything under ~5 (and well below threshold 0.5) is fine. The point of the smoke test is: did `Predict` run end-to-end without throwing on shape mismatches? If it throws, the assertion bug is in the model load or pipeline geometry.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Build locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/02-wakeword-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success. Common compile errors:
|
||||||
|
|
||||||
|
- `NamedOnnxValue` / `DenseTensor` / `InferenceSession` not found → confirm `using Microsoft.ML.OnnxRuntime;` and `using Microsoft.ML.OnnxRuntime.Tensors;` are at top.
|
||||||
|
- `'InputMetadata' has no Single()` → add `using System.Linq;` (with `ImplicitUsings` enabled this should already be there, but the spike didn't need it).
|
||||||
|
- API surface drift in `Microsoft.ML.OnnxRuntime` `<ORT_VER>` (the package occasionally renames `NamedOnnxValue.CreateFromTensor`) → consult Context7 `/microsoft/onnxruntime` docs for the pinned version. Do NOT silently switch to a different API; document the version-specific shape if it diverges.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Run on the Pi via `bin/probe-cs-2`**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs-2
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output:
|
||||||
|
|
||||||
|
```
|
||||||
|
Loading WakewordModel...
|
||||||
|
Loaded in 0.XXs.
|
||||||
|
Silence test: <small>/50 non-zero scores (expected: 0 — but small drift is OK).
|
||||||
|
Predict pipeline ran without throwing.
|
||||||
|
```
|
||||||
|
|
||||||
|
If the run aborts with an `InvalidOperationException` from one of the shape assertions, the model in the repo disagrees with what this plan expects — STOP. Either the vendored ONNX file is from a newer openwakeword version with a different shape (re-verify the SHA-256 hashes against Step 8 of Task 1), or our shape derivation from `oww-utils.py` was wrong. Investigate by re-running the Task 1 spike and comparing dimensions before adjusting `WakewordModel.cs`.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/02-wakeword-cs/WakewordModel.cs tests/02-wakeword-cs/Program.cs
|
||||||
|
git commit -m "WakewordModel: port openwakeword streaming pipeline (mel→emb→cls)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: `Program.cs` — live input stream + main inference loop (detection-only, no beep)
|
||||||
|
|
||||||
|
**Goal:** Replace the load-only smoke test with the full input stream + inference loop. Detection lines print to stdout but no beep yet — we want detection working in isolation before adding the second PortAudio stream.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/02-wakeword-cs/Program.cs` (full rewrite)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Rewrite `tests/02-wakeword-cs/Program.cs`**
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using WakewordProbe;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
const int SampleRate = WakewordModel.SampleRate;
|
||||||
|
const int Channels = 1;
|
||||||
|
const uint BlockFrames = WakewordModel.FrameSamples; // 1280
|
||||||
|
const float Threshold = 0.5f;
|
||||||
|
const double CooldownSeconds = 1.0;
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio / onnxruntime (which use getenv()) see the values.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
Environment.SetEnvironmentVariable("ORT_LOGGING_LEVEL", "3");
|
||||||
|
Libc.setenv("ORT_LOGGING_LEVEL", "3", 1);
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
WakewordModel? model = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
Console.WriteLine("Loading WakewordModel...");
|
||||||
|
var t0 = DateTime.UtcNow;
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
model = new WakewordModel(
|
||||||
|
Path.Combine(modelsDir, "melspectrogram.onnx"),
|
||||||
|
Path.Combine(modelsDir, "embedding_model.onnx"),
|
||||||
|
Path.Combine(modelsDir, "alexa.onnx"));
|
||||||
|
Console.WriteLine($"Loaded in {(DateTime.UtcNow - t0).TotalSeconds:0.00}s.");
|
||||||
|
|
||||||
|
var queue = new BlockingCollection<short[]>(boundedCapacity: 16);
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (frameCount != BlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected callback frameCount={frameCount} (want {BlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[BlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!queue.TryAdd(frame, 0))
|
||||||
|
Console.Error.WriteLine("[status] consumer behind, dropping frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
Console.WriteLine("Listening. Say 'alexa'. Ctrl-C to exit.");
|
||||||
|
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
TimeSpan lastTrigger = TimeSpan.FromSeconds(-CooldownSeconds);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
short[] frame = queue.Take(cts.Token);
|
||||||
|
float score = model.Predict(frame);
|
||||||
|
var now = sw.Elapsed;
|
||||||
|
if (score >= Threshold && (now - lastTrigger).TotalSeconds >= CooldownSeconds)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"DETECTED alexa score={score:0.000} t={now.TotalSeconds:0.0}s");
|
||||||
|
lastTrigger = now;
|
||||||
|
// Beep is added in Task 5.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
stream.Stop();
|
||||||
|
Console.WriteLine("\nbye");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
model?.Dispose();
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Key reuses from Test 1's `Program.cs` (`tests/01-record-play-cs/Program.cs`): the `using Stream = PortAudioSharp.Stream;` disambiguation, the `Libc.setenv` P/Invoke, the `FindUsbDevice` body, the `unsafe` callback `Buffer.MemoryCopy` pattern, the dual `Environment.SetEnvironmentVariable` + `Libc.setenv` calls.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Build locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/02-wakeword-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Deploy + run on the Pi**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs-2
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected on the Pi:
|
||||||
|
|
||||||
|
```
|
||||||
|
Using device <N> ('USB ...')
|
||||||
|
Loading WakewordModel...
|
||||||
|
Loaded in 0.XXs.
|
||||||
|
Listening. Say 'alexa'. Ctrl-C to exit.
|
||||||
|
```
|
||||||
|
|
||||||
|
Then say "alexa" 3-4 times at normal volume from ~1 m. Expected: one `DETECTED alexa score=0.XXX t=YYs` line per spoken wakeword, fired within ~1 s of saying it. **No beep yet** — that's Task 5.
|
||||||
|
|
||||||
|
What to check before moving on:
|
||||||
|
|
||||||
|
- Detection actually fires. If 0/4 attempts trigger, the port has a bug — re-verify shapes and the `x/10 + 2` mel transform before continuing.
|
||||||
|
- No sustained `[status] input overflow` or `[status] consumer behind` lines. Occasional ones at startup are OK.
|
||||||
|
- No ORT GPU warnings (`/sys/class/drm/card0` etc.) — `ORT_LOGGING_LEVEL=3` should silence them. If they appear, the `Libc.setenv` for ORT_LOGGING_LEVEL isn't being honoured by the onnxruntime version pinned; consult Context7 for the correct env var name for that version.
|
||||||
|
- Ctrl-C from the workstation cleanly exits the probe (prints `bye`). If Ctrl-C just hangs the SSH session, check that `bin/probe-cs-2` uses `ssh -t`.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/02-wakeword-cs/Program.cs
|
||||||
|
git commit -m "Live wakeword loop: PortAudio input + queue + threshold detection"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: Fire-and-forget beep + full hardware verification + `findings.md` update
|
||||||
|
|
||||||
|
**Goal:** Add the detection beep without blocking input ingestion, then run the full hardware test against all three pass criteria from the spec, and capture findings.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/02-wakeword-cs/Program.cs` (add beep helper + wire it into the detection branch)
|
||||||
|
- Modify: `findings.md` (append Test 2 outcome section)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add the beep buffer + dispatch helper to `Program.cs`**
|
||||||
|
|
||||||
|
At the top of `Program.cs`, after the `const double CooldownSeconds = 1.0;` line, add:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
const double BeepHz = 880.0;
|
||||||
|
const double BeepSeconds = 0.2;
|
||||||
|
```
|
||||||
|
|
||||||
|
Before the line `Console.WriteLine("Listening. Say 'alexa'. Ctrl-C to exit.");`, add the beep buffer precomputation:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
short[] beepBuffer = MakeBeep(BeepHz, BeepSeconds, SampleRate);
|
||||||
|
```
|
||||||
|
|
||||||
|
In the detection branch (inside the main loop, after `lastTrigger = now;`), add:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
FireAndForgetBeep(device, beepBuffer);
|
||||||
|
```
|
||||||
|
|
||||||
|
At the bottom of the file (after `FindUsbDevice` but before `class Libc`), add:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetBeep(int device, short[] beepBuffer)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = beepBuffer.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
int silence = (int)frameCount - take;
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &beepBuffer[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= beepBuffer.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
done.Wait();
|
||||||
|
// Brief drain pause matches Test 1's behaviour; the device buffer needs a moment after Complete.
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
});
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: `MakeBeep` and `FireAndForgetBeep` are `static` local-method-style helpers; with top-level statements they live at file scope alongside `FindUsbDevice` and `Libc`. They reference `Stream` (the `using` alias) and `PortAudio.GetDeviceInfo` — both already in scope.
|
||||||
|
|
||||||
|
The `Task.Run` is captured by the closure but not awaited; the main loop returns immediately after `stream.Start()`. The stream + event handle are kept alive by the closure until the task disposes them.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Build locally**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dotnet build tests/02-wakeword-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: success.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Run on the Pi and verify pass criterion 1 (true-positive rate)**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bin/probe-cs-2
|
||||||
|
```
|
||||||
|
|
||||||
|
Wait for `Listening...`. From ~1 m, say "alexa" at normal volume, **10 times**, pausing ≥ 2 seconds between attempts. Count printed `DETECTED ...` lines AND audible beeps.
|
||||||
|
|
||||||
|
**Pass criterion 1:** ≥ 8/10 detections, each within ~1 s of saying the word.
|
||||||
|
|
||||||
|
If you get fewer: the issue is in the inference pipeline. Common causes:
|
||||||
|
- `x/10 + 2` mel transform wasn't applied (search `WakewordModel.cs` for `/10` — if missing, that's it).
|
||||||
|
- Off-by-one in the mel window slice (`-EmbeddingWindowMelFrames` vs `-EmbeddingWindowMelFrames-1`).
|
||||||
|
- Dtype mismatch — passing int16 directly instead of converting to float32.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify pass criterion 2 (false-positive rate)**
|
||||||
|
|
||||||
|
While the probe is still running, read a newspaper or book aloud at normal volume from ~1 m for **3 continuous minutes** (anything that isn't "alexa"). Count any spurious `DETECTED ...` lines.
|
||||||
|
|
||||||
|
**Pass criterion 2:** ≤ 1 false positive per minute (i.e., ≤ 3 over the 3-minute test).
|
||||||
|
|
||||||
|
If you get more: the threshold (0.5) was right for the Python probe; if C# is over-triggering, the pipeline is producing systematically higher scores than Python — likely a normalization or transform bug (revisit Step 3 troubleshooting).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Verify pass criterion 3 (CPU budget)**
|
||||||
|
|
||||||
|
In a **second terminal**, while the probe is still running:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p assistant ssh pi@192.168.50.115 htop
|
||||||
|
```
|
||||||
|
|
||||||
|
Find the `Probe` process row. Note the `CPU%` column over ~30 seconds while the probe is doing inference (speak occasionally to keep it busy).
|
||||||
|
|
||||||
|
**Pass criterion 3:** CPU% stays under 50% of one core (the Pi 4 has 4 cores, so htop shows "100%" per core — pass = stays below 50 in that column).
|
||||||
|
|
||||||
|
If higher: confirm `WakewordModel`'s `SessionOptions` set `IntraOpNumThreads = 1` and `InterOpNumThreads = 1` (Task 3 Step 1). If they're set and it's still hot, ONNX Runtime may be ignoring the limit — set the env var via `Libc.setenv("OMP_NUM_THREADS", "1", 1)` at the top of `Program.cs` alongside the others, redeploy, recheck.
|
||||||
|
|
||||||
|
Ctrl-C the probe when done. Quit htop.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit the beep code**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add tests/02-wakeword-cs/Program.cs
|
||||||
|
git commit -m "Beep on detection: fire-and-forget OutputStream, input keeps flowing"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 7: Append Test 2 outcome section to `findings.md`**
|
||||||
|
|
||||||
|
Open `findings.md` and append, after the existing `## C# probe outcome (2026-06-12 — Test 1 ported to C#)` section, a new section in the same shape:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
## C# wakeword probe outcome (2026-06-12 — Test 2 ported to C#)
|
||||||
|
|
||||||
|
The Python Test 2 (openwakeword "alexa" listener with beep on detect) was re-implemented in C# / .NET 9, with the openwakeword `Model.predict()` streaming pipeline ported to `Microsoft.ML.OnnxRuntime`. Probe lives at `tests/02-wakeword-cs/`; deploy script `bin/probe-cs-2`. All three pass criteria from the original spec held on hardware.
|
||||||
|
|
||||||
|
### What we proved
|
||||||
|
|
||||||
|
- **Microsoft.ML.OnnxRuntime <ORT_VER>** runs on linux-arm64 from a self-contained `dotnet publish`. NuGet [bundles / does not bundle — confirm] `libonnxruntime.so` for the RID.
|
||||||
|
- The three openwakeword ONNX models (`melspectrogram.onnx`, `embedding_model.onnx`, `alexa.onnx`, vendored from openwakeword 0.6.0) load and chain correctly when fed real audio.
|
||||||
|
- **Two PortAudio streams on one USB device works**: the input stream stays open while a per-detection fire-and-forget output stream plays the beep. No errors, no input overflow during playback (fixing the bug `findings.md` § A flagged in the Python probe).
|
||||||
|
- True-positive rate: <X>/10, false-positive rate: <Y>/3 min, CPU: <Z>% of one core. (Fill in observed values.)
|
||||||
|
|
||||||
|
### Key gotchas
|
||||||
|
|
||||||
|
- (Capture anything that took > 15 minutes to debug. Likely candidates: env-var name for ORT log level on this version; whether the mel transform `x/10 + 2` actually mattered in C#; whether `IntraOpNumThreads = 1` was honoured; whether shape assertions caught anything during development.)
|
||||||
|
|
||||||
|
### Open questions for the main assistant
|
||||||
|
|
||||||
|
- **Custom wakeword model.** Stock "alexa" works as a probe but the shipped assistant needs a custom "hey assistant" (or similar) model. openwakeword has a Colab notebook for synthetic training; budget ~1 hour to train + verify on the same pipeline this probe validates.
|
||||||
|
- **Speaker-is-mic self-trigger.** Not in probe scope, but the main assistant will need detector gating during TTS playback (findings.md § B), or hardware/software AEC.
|
||||||
|
- **Inference latency.** If `WakewordModel.Predict` ever exceeds 80 ms, the queue backs up; sample real measurements from this probe before sizing the queue / consumer thread in the main assistant.
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace the bracketed placeholders (`<ORT_VER>`, `<X>`, `<Y>`, `<Z>`, the bundling note, the gotchas list) with the actual values you observed. **Do not commit with placeholders.**
|
||||||
|
|
||||||
|
- [ ] **Step 8: Commit findings**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git add findings.md
|
||||||
|
git commit -m "Findings: C# wakeword probe outcome"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Notes on what is intentionally **not** in this plan
|
||||||
|
|
||||||
|
- **No automated tests.** The spec is explicit; verification is the listening / counting test on real hardware. The silence smoke test in Task 3 is a pipeline sanity check, not a model-correctness test.
|
||||||
|
- **No numerical parity test vs Python.** If hardware passes, the port is good enough for the probe. If hardware fails, the diagnostic levers in the spec (Section "Diagnostic levers if it fails") cover the likely causes.
|
||||||
|
- **No detector gating during beep playback.** The 200 ms beep at 880 Hz is far enough from human speech to not self-trigger the "alexa" classifier; if it did, the spec's § B (gate detector during TTS) is the main-assistant solution, not the probe's.
|
||||||
|
- **No retry / restart on PortAudio init failure.** Probe — crash and surface.
|
||||||
|
- **No abstraction over PortAudio.** Same one-file Program.cs shape as Test 1.
|
||||||
|
- **No custom wakeword.** Stock alexa only. Custom is a separate task per `findings.md` § C.
|
||||||
|
- **No bin/probe-cs-2 → bin/probe-cs unification.** Decided in brainstorm — kept as separate scripts.
|
||||||
|
|
||||||
|
If during implementation you feel the urge to add any of the above, stop and re-read the spec. The point of this probe is to be cheap and disposable.
|
||||||
@@ -0,0 +1,758 @@
|
|||||||
|
# Test 3 — Full-cycle C# probe Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Build a single-binary C# probe at `tests/03-full-cycle-cs/` that runs the full assistant cycle on the Pi: idle → "alexa" → beep → record 5 s → play back → idle, with detector gating during non-idle states and a `WakewordModel.Reset()` on every IDLE re-entry.
|
||||||
|
|
||||||
|
**Architecture:** One persistent PortAudio input stream feeds 80 ms frames into a `BlockingCollection`; a single consumer thread runs a four-state machine (IDLE / BEEPING / RECORDING / PLAYBACK). Beep and playback are fire-and-forget output streams using Test 2's `Start()`-before-`Task.Run` + 2 s timeout lifetime pattern. `WakewordModel` is copied verbatim from Test 2 with a new `Reset()` method called on PLAYBACK→IDLE.
|
||||||
|
|
||||||
|
**Tech Stack:** .NET 9 (self-contained linux-arm64), PortAudioSharp2 1.0.6, Microsoft.ML.OnnxRuntime 1.26.0. Deploy via `dotnet publish` + `scp` + `ssh -t` to Pi at `pi@192.168.50.115`.
|
||||||
|
|
||||||
|
**Spec:** `docs/superpowers/specs/2026-06-12-test-3-full-cycle-csharp-design.md`. Read it before starting.
|
||||||
|
|
||||||
|
**Branch:** `fresh-start` (already current; do NOT create a new branch).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Scaffold `tests/03-full-cycle-cs/` project skeleton
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/03-full-cycle-cs/Probe.csproj`
|
||||||
|
- Create: `tests/03-full-cycle-cs/models/` (directory; populated in Task 2)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create the project directory**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir -p tests/03-full-cycle-cs/models
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Write `Probe.csproj`**
|
||||||
|
|
||||||
|
Create `tests/03-full-cycle-cs/Probe.csproj` with this exact content (identical to `tests/02-wakeword-cs/Probe.csproj` except `RootNamespace`):
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>FullCycleProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.26.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="models/*.onnx">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit the scaffold**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/03-full-cycle-cs/Probe.csproj
|
||||||
|
git commit -m "Test 3 scaffold: csproj for tests/03-full-cycle-cs"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Copy models and `WakewordModel.cs` from Test 2 (with namespace rename and `Reset()`)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/03-full-cycle-cs/WakewordModel.cs` (copied from Test 2, namespace renamed, `Reset()` added)
|
||||||
|
- Create: `tests/03-full-cycle-cs/models/melspectrogram.onnx`
|
||||||
|
- Create: `tests/03-full-cycle-cs/models/embedding_model.onnx`
|
||||||
|
- Create: `tests/03-full-cycle-cs/models/alexa.onnx`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Copy the three ONNX model files**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp tests/02-wakeword-cs/models/melspectrogram.onnx tests/03-full-cycle-cs/models/
|
||||||
|
cp tests/02-wakeword-cs/models/embedding_model.onnx tests/03-full-cycle-cs/models/
|
||||||
|
cp tests/02-wakeword-cs/models/alexa.onnx tests/03-full-cycle-cs/models/
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Copy `WakewordModel.cs`**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp tests/02-wakeword-cs/WakewordModel.cs tests/03-full-cycle-cs/WakewordModel.cs
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Rename the namespace in the copy**
|
||||||
|
|
||||||
|
Edit `tests/03-full-cycle-cs/WakewordModel.cs`: change line 4 from `namespace WakewordProbe;` to `namespace FullCycleProbe;`. Leave everything else unchanged.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Add `Reset()` method to the copy**
|
||||||
|
|
||||||
|
Edit `tests/03-full-cycle-cs/WakewordModel.cs`. Insert the following method immediately before the `public void Dispose()` method near the bottom of the file:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
public void Reset()
|
||||||
|
{
|
||||||
|
_rawRingFill = 0;
|
||||||
|
_melRing.Clear();
|
||||||
|
_embRing.Clear();
|
||||||
|
_framesSeen = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
(Note the trailing blank line. The result should be a method block sitting between `return clsOut[0, 0]; }` (end of `Predict`) and `public void Dispose()`.)
|
||||||
|
|
||||||
|
- [ ] **Step 5: Verify the project builds**
|
||||||
|
|
||||||
|
Run from repo root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet build tests/03-full-cycle-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Build succeeded` with 0 errors, 0 warnings. (At this point `Program.cs` doesn't exist yet; csproj is `<OutputType>Exe</OutputType>` but the build will produce a no-entry-point warning or error. If it errors with "CS5001: Program does not contain a static 'Main' method", that's expected at this stage — proceed to Task 3 which adds `Program.cs`.)
|
||||||
|
|
||||||
|
If the build errors for any reason other than the missing entry point, fix before continuing.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/03-full-cycle-cs/WakewordModel.cs tests/03-full-cycle-cs/models/
|
||||||
|
git commit -m "Test 3: copy WakewordModel + models from Test 2, add Reset()"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Write `Program.cs` with the full state machine
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/03-full-cycle-cs/Program.cs`
|
||||||
|
|
||||||
|
The whole file is shown below. It lifts from `tests/02-wakeword-cs/Program.cs` (env-var setup, OrtEnv init, FindUsbDevice, input stream + queue, FireAndForgetBeep, Libc, MakeBeep) and adds:
|
||||||
|
|
||||||
|
- `volatile State _state` enum-as-state-machine in the consumer loop.
|
||||||
|
- `FireAndForgetPlayback` helper (same shape as `FireAndForgetBeep` but reads from `recordBuf` and signals via `_playbackDoneFlag` instead of a `ManualResetEventSlim`).
|
||||||
|
- `model.Reset()` call on PLAYBACK→IDLE transition.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write `Program.cs` in full**
|
||||||
|
|
||||||
|
Create `tests/03-full-cycle-cs/Program.cs` with exactly this content:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using FullCycleProbe;
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
const int SampleRate = WakewordModel.SampleRate; // 16 000
|
||||||
|
const int Channels = 1;
|
||||||
|
const uint BlockFrames = WakewordModel.FrameSamples; // 1280 (80 ms)
|
||||||
|
const float Threshold = 0.5f;
|
||||||
|
const double CooldownSeconds = 1.0;
|
||||||
|
const double BeepHz = 880.0;
|
||||||
|
const double BeepSeconds = 0.2;
|
||||||
|
|
||||||
|
const int BeepingFrames = 3; // 3 * 80 ms = 240 ms (covers 200 ms beep + 40 ms drain/reverb margin)
|
||||||
|
const int RecordingFrames = 63; // 63 * 80 ms = 5.04 s (rounded up from 5 s to avoid partial-frame handling)
|
||||||
|
const int RecordBufSamples = RecordingFrames * (int)BlockFrames; // 80640
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio / onnxruntime (which use getenv()) see the values.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
Environment.SetEnvironmentVariable("ORT_LOGGING_LEVEL", "3");
|
||||||
|
Libc.setenv("ORT_LOGGING_LEVEL", "3", 1);
|
||||||
|
// ORT 1.26.0: the env var does NOT suppress early GPU-discovery warnings from device_discovery.cc.
|
||||||
|
// CreateInstanceWithOptions sets the log level at OrtEnv creation time, before EP discovery runs.
|
||||||
|
var ortEnvOpts = new EnvironmentCreationOptions { logId = "FullCycleProbe", logLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR };
|
||||||
|
OrtEnv.CreateInstanceWithOptions(ref ortEnvOpts);
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
WakewordModel? model = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
Console.WriteLine("Loading WakewordModel...");
|
||||||
|
var t0 = DateTime.UtcNow;
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
model = new WakewordModel(
|
||||||
|
Path.Combine(modelsDir, "melspectrogram.onnx"),
|
||||||
|
Path.Combine(modelsDir, "embedding_model.onnx"),
|
||||||
|
Path.Combine(modelsDir, "alexa.onnx"));
|
||||||
|
Console.WriteLine($"Loaded in {(DateTime.UtcNow - t0).TotalSeconds:0.00}s.");
|
||||||
|
|
||||||
|
var queue = new BlockingCollection<short[]>(boundedCapacity: 16);
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (frameCount != BlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected callback frameCount={frameCount} (want {BlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[BlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!queue.TryAdd(frame, 0))
|
||||||
|
Console.Error.WriteLine("[status] consumer behind, dropping frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
short[] beepBuffer = MakeBeep(BeepHz, BeepSeconds, SampleRate);
|
||||||
|
Console.WriteLine("Listening. Say 'alexa'. Ctrl-C to exit.");
|
||||||
|
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
TimeSpan lastTrigger = TimeSpan.FromSeconds(-CooldownSeconds);
|
||||||
|
|
||||||
|
// State machine (single-writer: this consumer thread).
|
||||||
|
State state = State.Idle;
|
||||||
|
int beepFrames = 0;
|
||||||
|
int recordOffset = 0;
|
||||||
|
short[] recordBuf = null!; // assigned on BEEPING→RECORDING transition
|
||||||
|
var playbackDone = new PlaybackDoneFlag();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
short[] frame = queue.Take(cts.Token);
|
||||||
|
|
||||||
|
switch (state)
|
||||||
|
{
|
||||||
|
case State.Idle:
|
||||||
|
{
|
||||||
|
float score = model.Predict(frame);
|
||||||
|
var now = sw.Elapsed;
|
||||||
|
if (score >= Threshold && (now - lastTrigger).TotalSeconds >= CooldownSeconds)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"DETECTED alexa score={score:0.000} t={now.TotalSeconds:0.0}s");
|
||||||
|
lastTrigger = now;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
FireAndForgetBeep(device, beepBuffer);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] beep stream failed to start: {ex.Message}");
|
||||||
|
// Stay in IDLE — no cycle to abort yet.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[state] BEEPING");
|
||||||
|
state = State.Beeping;
|
||||||
|
beepFrames = 0;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Beeping:
|
||||||
|
{
|
||||||
|
beepFrames++;
|
||||||
|
if (beepFrames >= BeepingFrames)
|
||||||
|
{
|
||||||
|
Console.WriteLine("[state] RECORDING");
|
||||||
|
state = State.Recording;
|
||||||
|
recordBuf = new short[RecordBufSamples];
|
||||||
|
recordOffset = 0;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Recording:
|
||||||
|
{
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
fixed (short* src = &frame[0])
|
||||||
|
fixed (short* dst = &recordBuf[recordOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
recordOffset += (int)BlockFrames;
|
||||||
|
if (recordOffset >= RecordBufSamples)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
FireAndForgetPlayback(device, recordBuf, playbackDone);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] playback stream failed to start: {ex.Message}");
|
||||||
|
// Drop the recording, reset model, return to IDLE.
|
||||||
|
model.Reset();
|
||||||
|
Console.WriteLine("[state] IDLE");
|
||||||
|
state = State.Idle;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[state] PLAYBACK");
|
||||||
|
state = State.Playback;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Playback:
|
||||||
|
{
|
||||||
|
// Discard input during playback (gate the detector).
|
||||||
|
if (playbackDone.Consume())
|
||||||
|
{
|
||||||
|
model.Reset();
|
||||||
|
Console.WriteLine("[state] IDLE");
|
||||||
|
state = State.Idle;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
stream.Stop();
|
||||||
|
Console.WriteLine("\nbye");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
model?.Dispose();
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetBeep(int device, short[] beepBuffer)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = beepBuffer.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &beepBuffer[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= beepBuffer.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
// 2 s timeout so a failed callback (which PortAudio silently swallows
|
||||||
|
// and would leave 'done' unset) eventually releases the stream + handle
|
||||||
|
// instead of leaking them for the process lifetime.
|
||||||
|
done.Wait(TimeSpan.FromSeconds(2));
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetPlayback(int device, short[] recordBuf, PlaybackDoneFlag doneFlag)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = recordBuf.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &recordBuf[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= recordBuf.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
// Same 2 s safety timeout as the beep cleanup.
|
||||||
|
done.Wait(TimeSpan.FromSeconds(2));
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
doneFlag.Set(); // tell the consumer it's safe to flip PLAYBACK→IDLE
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
enum State { Idle, Beeping, Recording, Playback }
|
||||||
|
|
||||||
|
// Volatile flag set by the playback cleanup task and consumed by the consumer thread.
|
||||||
|
// Wrapped in a tiny class so we can pass it by reference into the helper.
|
||||||
|
sealed class PlaybackDoneFlag
|
||||||
|
{
|
||||||
|
private volatile bool _set;
|
||||||
|
public void Set() => _set = true;
|
||||||
|
public bool Consume()
|
||||||
|
{
|
||||||
|
if (!_set) return false;
|
||||||
|
_set = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Verify the project builds cleanly**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet build tests/03-full-cycle-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Build succeeded` with 0 errors, 0 warnings.
|
||||||
|
|
||||||
|
If `<AllowUnsafeBlocks>` is missing from the csproj, the build will fail with `CS0227: Unsafe code may only appear if compiling with /unsafe`. Confirm Task 1's csproj has `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`.
|
||||||
|
|
||||||
|
If you see `CS0246: The type or namespace 'Stream' could not be found` or a clash with `System.IO.Stream`, confirm the `using Stream = PortAudioSharp.Stream;` line is at the top of `Program.cs`.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify `dotnet publish` produces a runnable binary**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet publish tests/03-full-cycle-cs -c Release -r linux-arm64 --self-contained -o /tmp/probe-cs-3-out
|
||||||
|
ls /tmp/probe-cs-3-out/Probe
|
||||||
|
ls /tmp/probe-cs-3-out/models/
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
- `Probe` (executable) exists.
|
||||||
|
- `models/` directory contains `melspectrogram.onnx`, `embedding_model.onnx`, `alexa.onnx`.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/03-full-cycle-cs/Program.cs
|
||||||
|
git commit -m "Test 3: full-cycle state machine (IDLE/BEEPING/RECORDING/PLAYBACK)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: Write `bin/probe-cs-3` deploy script
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `bin/probe-cs-3`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the deploy script**
|
||||||
|
|
||||||
|
Create `bin/probe-cs-3` with exactly this content (it's `bin/probe-cs-2` with three string swaps: `02-wakeword-cs` → `03-full-cycle-cs`, `probe-cs-2` → `probe-cs-3`, `/tmp/probe-cs-2-out` → `/tmp/probe-cs-3-out`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-3-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/03-full-cycle-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-3/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-3 && mkdir ~/probe-cs-3'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-3/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-3 && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Make it executable**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chmod +x bin/probe-cs-3
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add bin/probe-cs-3
|
||||||
|
git commit -m "Test 3: bin/probe-cs-3 deploy script"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: Hardware verification on the Pi
|
||||||
|
|
||||||
|
This task is the actual pass/fail gate. No automated tests; verification is the human ear + console output + `top` reading on the Pi. If anything in this task fails, do NOT mark the implementation complete — investigate and either fix in code or update the spec to reflect what actually works.
|
||||||
|
|
||||||
|
**Files:** (none — this is a runtime verification task)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Confirm the Pi is reachable**
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ping -c 2 192.168.50.115
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 2/2 replies. If unreachable, stop and ask the user.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Deploy + run**
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./bin/probe-cs-3
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected console output (abridged):
|
||||||
|
|
||||||
|
```
|
||||||
|
>> dotnet publish (linux-arm64, self-contained)
|
||||||
|
... (build output) ...
|
||||||
|
>> scp to pi@192.168.50.115:~/probe-cs-3/ (wipe-and-replace)
|
||||||
|
>> ssh + run on Pi (Ctrl-C from this terminal stops the probe)
|
||||||
|
[wakeword-model] melspectrogram.onnx
|
||||||
|
input ...
|
||||||
|
[wakeword-model] embedding_model.onnx ...
|
||||||
|
[wakeword-model] alexa.onnx ...
|
||||||
|
Loaded in 0.6s.
|
||||||
|
Using device <N> ('USB Speaker Phone...')
|
||||||
|
Listening. Say 'alexa'. Ctrl-C to exit.
|
||||||
|
```
|
||||||
|
|
||||||
|
If the publish step errors with `setenv: command not found` or similar libc-related runtime crash, confirm Task 3's `Libc.setenv` `[DllImport]` is present and the binary was self-contained-published.
|
||||||
|
|
||||||
|
If the Pi run errors with `Cannot open shared library libonnxruntime.so` or `libportaudio.so`, the self-contained publish didn't bundle the native libs — confirm `<SelfContained>true</SelfContained>` and `<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>` in Task 1's csproj.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Run 3 full cycles back-to-back**
|
||||||
|
|
||||||
|
Say "alexa" clearly at normal volume from ~1 m. Observe:
|
||||||
|
|
||||||
|
1. Beep plays within ~100 ms of saying "alexa".
|
||||||
|
2. Console prints `DETECTED alexa score=…`, then `[state] BEEPING`, then `[state] RECORDING`.
|
||||||
|
3. Say something distinctive (e.g. "hello world, this is cycle one") within the 5 s recording window.
|
||||||
|
4. Console prints `[state] PLAYBACK`.
|
||||||
|
5. Speaker plays back what you said. It should be recognisable.
|
||||||
|
6. Console prints `[state] IDLE`.
|
||||||
|
7. Pause ~1.5 s, then say "alexa" again. Verify the cycle starts over.
|
||||||
|
|
||||||
|
Repeat for cycle 2 and cycle 3. Record on paper (or transcript) which cycles succeeded.
|
||||||
|
|
||||||
|
Pass criterion 1: 3 full cycles back-to-back; each playback recognisable as what was spoken.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Measure idle CPU on the Pi**
|
||||||
|
|
||||||
|
In a second workstation terminal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 'top -b -n 3 -d 1 -p $(pgrep -f Probe)'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Probe` process appears with `%CPU` < 50 across the three samples (Test 2 measured 17–31%, Test 3 should be in the same range during IDLE).
|
||||||
|
|
||||||
|
Pass criterion 4: CPU < 50% of one core during IDLE.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Test post-playback re-detection latency**
|
||||||
|
|
||||||
|
Speak "alexa" immediately after each `[state] IDLE` log line. Note how soon detection fires.
|
||||||
|
|
||||||
|
Pass criterion 2: Next "alexa" after playback detected within ~2 s of playback ending. (Below ~1.3 s is impossible due to WarmupFrames; above ~2 s indicates a regression.)
|
||||||
|
|
||||||
|
- [ ] **Step 6: Scan for warnings**
|
||||||
|
|
||||||
|
Re-read the console output from Step 3. Confirm there are **no** lines of the form:
|
||||||
|
|
||||||
|
- `[status] input overflow`
|
||||||
|
- `[status] consumer behind, dropping frame`
|
||||||
|
- `[status] unexpected callback frameCount=...`
|
||||||
|
- `[error] beep stream failed to start: ...`
|
||||||
|
- `[error] playback stream failed to start: ...`
|
||||||
|
|
||||||
|
A single occurrence under abnormal load may be tolerable; sustained occurrences during normal cycles fail criterion 5.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Test Ctrl-C clean exit**
|
||||||
|
|
||||||
|
Press Ctrl-C in the workstation terminal that's running the probe.
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
|
||||||
|
```
|
||||||
|
^C
|
||||||
|
bye
|
||||||
|
$
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass criterion 7: process exits 0, no stack trace, terminal returns to prompt. Verify with `echo $?` → `0`.
|
||||||
|
|
||||||
|
- [ ] **Step 8: Wakeword detection rate during IDLE**
|
||||||
|
|
||||||
|
Restart the probe (`./bin/probe-cs-3`). Wait until `Listening. Say 'alexa'. Ctrl-C to exit.` appears. Say "alexa" 10 times with a ~2 s gap between each. Count how many times `DETECTED alexa score=…` appears.
|
||||||
|
|
||||||
|
Pass criterion 3: ≥ 8/10. (Test 2 demonstrated this rate; Test 3 should match.)
|
||||||
|
|
||||||
|
- [ ] **Step 9: Update `findings.md` with the outcome**
|
||||||
|
|
||||||
|
Add a new section `## Test 3 full-cycle outcome (2026-06-12)` at the bottom of `findings.md` summarising:
|
||||||
|
|
||||||
|
- Pass / fail per criterion.
|
||||||
|
- Any new gotchas discovered during implementation or hardware verification (especially any that contradicted the spec's predictions).
|
||||||
|
- CPU measurement (peak + typical), detection latency post-playback, observed pass rate.
|
||||||
|
- Reference paths (spec + plan + code + deploy script).
|
||||||
|
|
||||||
|
Use the existing Test 1 and Test 2 outcome sections in `findings.md` as the template.
|
||||||
|
|
||||||
|
- [ ] **Step 10: Commit the findings update**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add findings.md
|
||||||
|
git commit -m "Findings: Test 3 full-cycle probe outcome"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 11: Final status report**
|
||||||
|
|
||||||
|
Report to the user:
|
||||||
|
|
||||||
|
- Pass/fail per criterion (1 through 7 from spec).
|
||||||
|
- Any criteria that failed and what was discovered.
|
||||||
|
- A one-line summary of whether Test 3 unblocks the main Pi-client implementation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-review notes
|
||||||
|
|
||||||
|
- **Spec coverage:** every section of the spec maps to a task — Architecture/State machine/Components → Tasks 1–4. Why-Reset-matters → Task 2's `Reset()` + Task 3's PLAYBACK→IDLE branch. Error handling → Task 3's try/catch on the FireAndForget calls + `[status]` / `[error]` log lines. Gotchas-carried-forward → preserved verbatim in Task 3's code. Verification → Task 5.
|
||||||
|
- **Placeholder scan:** no TBD/TODO; every code step shows the actual code; every command shows expected output.
|
||||||
|
- **Type consistency:** `State` enum (`Idle`, `Beeping`, `Recording`, `Playback`) used consistently; `PlaybackDoneFlag.Set()` / `.Consume()` used consistently between `FireAndForgetPlayback` and the consumer's PLAYBACK branch; `WakewordModel.Reset()` called in two places (Recording-failure recovery and Playback→Idle), signature is parameterless in both.
|
||||||
@@ -0,0 +1,897 @@
|
|||||||
|
# Test 4a — OpenAI Realtime API one-shot voice round-trip (C#) Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Build a single-binary throwaway C# probe at `tests/04a-realtime-oneshot-cs/` that has one round-trip spoken conversation with the OpenAI Realtime API: boot → beep → user speaks one sentence → server VAD ends the turn → assistant audio reply plays back through the USB Speaker Phone → exit 0.
|
||||||
|
|
||||||
|
**Architecture:** One persistent `ClientWebSocket` to `wss://api.openai.com/v1/realtime?model=gpt-realtime`. Two `Task.Run` loops: ReceiveLoop parses server events with `JsonDocument` and writes decoded audio chunks to a `Channel<short[]>`; UpstreamSendLoop drains a separate `Channel<short[]>` of mic frames and sends them as `input_audio_buffer.append`. PortAudio input stream (24 kHz mono Int16, 1920 frames/cb) writes to the upstream channel via callback; output stream (24 kHz, 1024 frames/cb) drains the downstream channel into the speaker, zero-filling underruns. Completion (`response.done` received AND downstream channel drained AND working chunk empty) is signaled via `ManualResetEventSlim` to the main thread, which closes the WS and exits.
|
||||||
|
|
||||||
|
**Tech Stack:** .NET 9 (self-contained linux-arm64), PortAudioSharp2 1.0.6, `System.Net.WebSockets.ClientWebSocket`, `System.Text.Json`, `System.Threading.Channels`. Deploy via `dotnet publish` + `scp` + `ssh -t` to Pi at `pi@192.168.50.115`. No onnxruntime — no wakeword in 4a.
|
||||||
|
|
||||||
|
**Spec:** `docs/superpowers/specs/2026-06-12-test-4a-openai-realtime-design.md`. Read it before starting.
|
||||||
|
|
||||||
|
**Branch:** `fresh-start` (already current; do NOT create a new branch).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Scaffold `tests/04a-realtime-oneshot-cs/` project skeleton
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/04a-realtime-oneshot-cs/Probe.csproj`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create the project directory**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir -p tests/04a-realtime-oneshot-cs
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Write `Probe.csproj`**
|
||||||
|
|
||||||
|
Create `tests/04a-realtime-oneshot-cs/Probe.csproj` with this exact content. Note the differences from Test 3's csproj: `RootNamespace=RealtimeOneShotProbe`, no `Microsoft.ML.OnnxRuntime` reference (no wakeword in 4a), no `models/*.onnx` content block.
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>RealtimeOneShotProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
|
```
|
||||||
|
|
||||||
|
`ClientWebSocket` (from `System.Net.WebSockets`), `JsonDocument` / `JsonSerializer` (from `System.Text.Json`), and `Channel<T>` (from `System.Threading.Channels`) all come from the BCL — no PackageReference needed.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit the scaffold**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/04a-realtime-oneshot-cs/Probe.csproj
|
||||||
|
git commit -m "Test 4a scaffold: csproj for tests/04a-realtime-oneshot-cs"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Write `Program.cs` in full
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tests/04a-realtime-oneshot-cs/Program.cs`
|
||||||
|
|
||||||
|
The whole file is shown below. It is self-contained — no helper files, no copied modules. The structure follows Test 3's `Program.cs` shape (top-level statements; helpers, classes, and `static class Libc` at the bottom of the file) with these notable differences:
|
||||||
|
|
||||||
|
- No onnxruntime, no `WakewordModel`, no models directory.
|
||||||
|
- A `ClientWebSocket` + two background `Task.Run` loops (ReceiveLoop, UpstreamSendLoop) replace the consumer-thread state machine.
|
||||||
|
- The input stream callback's job is to push mic frames to a `Channel<short[]>` (not to call `Predict`).
|
||||||
|
- The output stream callback drains a different `Channel<short[]>`, zero-fills underruns, and signals completion via a `ManualResetEventSlim` (replacing Test 3's per-cycle `PlaybackDoneFlag`).
|
||||||
|
- A `SharedState` class holds the three `volatile bool` flags (`MicArmed`, `StopSending`, `NoMoreDeltas`) plus the output callback's per-call drain state (`CurrentChunk`, `ChunkOffset`).
|
||||||
|
- `LoadApiKey()` reads `~/.openai_key` with mode-600 validation.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write `Program.cs` in full**
|
||||||
|
|
||||||
|
Create `tests/04a-realtime-oneshot-cs/Program.cs` with exactly this content:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Threading.Channels;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio (which uses getenv()) sees the value.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
|
||||||
|
string apiKey = LoadApiKey();
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var upstreamChannel = Channel.CreateBounded<short[]>(new BoundedChannelOptions(C.UpstreamChannelCapacity)
|
||||||
|
{
|
||||||
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
|
SingleWriter = true,
|
||||||
|
SingleReader = true,
|
||||||
|
});
|
||||||
|
var downstreamChannel = Channel.CreateBounded<short[]>(new BoundedChannelOptions(C.DownstreamChannelCapacity)
|
||||||
|
{
|
||||||
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
|
SingleWriter = false,
|
||||||
|
SingleReader = true,
|
||||||
|
});
|
||||||
|
|
||||||
|
var doneFlag = new ManualResetEventSlim(false);
|
||||||
|
var state = new SharedState();
|
||||||
|
|
||||||
|
var ws = new ClientWebSocket();
|
||||||
|
ws.Options.SetRequestHeader("Authorization", $"Bearer {apiKey}");
|
||||||
|
ws.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
|
||||||
|
|
||||||
|
Stream? inputStream = null;
|
||||||
|
Stream? outputStream = null;
|
||||||
|
Task? receiveTask = null;
|
||||||
|
Task? sendTask = null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var uri = new Uri($"wss://api.openai.com/v1/realtime?model={C.ModelName}");
|
||||||
|
Console.WriteLine($"[ws] connecting to {uri}");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await ws.ConnectAsync(uri, cts.Token);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] WS connect failed: {ex.Message}");
|
||||||
|
if (ex.InnerException is HttpRequestException httpEx)
|
||||||
|
Console.Error.WriteLine($"[error] inner: HTTP {(int?)httpEx.StatusCode}");
|
||||||
|
Environment.ExitCode = 3;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[ws] connected");
|
||||||
|
|
||||||
|
receiveTask = Task.Run(() => ReceiveLoop(ws, downstreamChannel, cts, state));
|
||||||
|
|
||||||
|
await SendSessionUpdate(ws, cts.Token);
|
||||||
|
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
inputStream = OpenInputStream(device, upstreamChannel, state);
|
||||||
|
try { inputStream.Start(); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] input stream start failed: {ex.Message}");
|
||||||
|
Environment.ExitCode = 5;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
outputStream = OpenOutputStream(device, downstreamChannel, state, doneFlag);
|
||||||
|
try { outputStream.Start(); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] output stream start failed: {ex.Message}");
|
||||||
|
Environment.ExitCode = 6;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
sendTask = Task.Run(() => UpstreamSendLoop(ws, upstreamChannel, cts, state));
|
||||||
|
|
||||||
|
// Startup beep — queued onto downstreamChannel before the receive loop produces any audio.
|
||||||
|
short[] beep = MakeBeep(C.BeepHz, C.BeepSeconds, C.SampleRate);
|
||||||
|
Console.WriteLine("[beep]");
|
||||||
|
await downstreamChannel.Writer.WriteAsync(beep, cts.Token);
|
||||||
|
|
||||||
|
// Wait for beep to drain (200 ms beep + 100 ms margin) so it doesn't get captured
|
||||||
|
// by the mic and shipped to OpenAI.
|
||||||
|
Thread.Sleep((int)(C.BeepSeconds * 1000) + C.BeepDrainMs);
|
||||||
|
state.MicArmed = true;
|
||||||
|
Console.WriteLine("Speak now.");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
doneFlag.Wait(cts.Token);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
Console.WriteLine("[ws] closing");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// Cancel any background work first so receive/send loops exit cleanly.
|
||||||
|
cts.Cancel();
|
||||||
|
|
||||||
|
if (ws.State == WebSocketState.Open || ws.State == WebSocketState.CloseReceived)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
|
||||||
|
await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "done", closeTimeout.Token);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[ws] close failed: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try { inputStream?.Stop(); } catch { }
|
||||||
|
try { outputStream?.Stop(); } catch { }
|
||||||
|
try { inputStream?.Dispose(); } catch { }
|
||||||
|
try { outputStream?.Dispose(); } catch { }
|
||||||
|
try { ws.Dispose(); } catch { }
|
||||||
|
try { PortAudio.Terminate(); } catch { }
|
||||||
|
|
||||||
|
Console.WriteLine("bye");
|
||||||
|
}
|
||||||
|
|
||||||
|
static string LoadApiKey()
|
||||||
|
{
|
||||||
|
string home = Environment.GetEnvironmentVariable("HOME") ?? "/root";
|
||||||
|
string path = Path.Combine(home, ".openai_key");
|
||||||
|
if (!File.Exists(path))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key file not found at {path}");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-fatal mode-600 check.
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var mode = File.GetUnixFileMode(path);
|
||||||
|
var leaky = mode & (UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute
|
||||||
|
| UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute);
|
||||||
|
if (leaky != UnixFileMode.None)
|
||||||
|
Console.Error.WriteLine($"[warn] {path} mode includes group/other perms ({mode}); expected 600");
|
||||||
|
}
|
||||||
|
catch (PlatformNotSupportedException) { /* not Unix */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[warn] could not check {path} mode: {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
string key = File.ReadAllText(path).Trim();
|
||||||
|
if (string.IsNullOrEmpty(key))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key file empty");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
if (!key.StartsWith("sk-"))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key doesn't start with sk-");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task SendSessionUpdate(ClientWebSocket ws, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var msg = new
|
||||||
|
{
|
||||||
|
type = "session.update",
|
||||||
|
session = new
|
||||||
|
{
|
||||||
|
modalities = new[] { "audio", "text" },
|
||||||
|
instructions = C.Instructions,
|
||||||
|
voice = C.Voice,
|
||||||
|
input_audio_format = "pcm16",
|
||||||
|
output_audio_format = "pcm16",
|
||||||
|
turn_detection = new
|
||||||
|
{
|
||||||
|
type = "server_vad",
|
||||||
|
threshold = 0.5,
|
||||||
|
prefix_padding_ms = 300,
|
||||||
|
silence_duration_ms = 500,
|
||||||
|
create_response = true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(msg);
|
||||||
|
await ws.SendAsync(new ArraySegment<byte>(bytes), WebSocketMessageType.Text, endOfMessage: true, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Stream OpenInputStream(int device, Channel<short[]> upstream, SharedState state)
|
||||||
|
{
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = C.Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (!state.MicArmed || state.StopSending)
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
|
||||||
|
if (frameCount != C.MicBlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected input frameCount={frameCount} (want {C.MicBlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[C.MicBlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, C.MicBlockFrames * sizeof(short), C.MicBlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!upstream.Writer.TryWrite(frame))
|
||||||
|
Console.Error.WriteLine("[status] upstream behind, dropping mic frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Stream(inParams, null, C.SampleRate, C.MicBlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Stream OpenOutputStream(int device, Channel<short[]> downstream, SharedState state, ManualResetEventSlim doneFlag)
|
||||||
|
{
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = C.Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int fill = 0;
|
||||||
|
|
||||||
|
while (fill < frameCount)
|
||||||
|
{
|
||||||
|
// Pull next chunk if we have no working one.
|
||||||
|
if (state.CurrentChunk == null)
|
||||||
|
{
|
||||||
|
if (!downstream.Reader.TryRead(out var nextChunk))
|
||||||
|
break; // ring empty
|
||||||
|
state.CurrentChunk = nextChunk;
|
||||||
|
state.ChunkOffset = 0;
|
||||||
|
}
|
||||||
|
// Local non-null alias so the compiler doesn't warn on dereference inside `fixed`.
|
||||||
|
short[] chunk = state.CurrentChunk!;
|
||||||
|
int slack = (int)frameCount - fill;
|
||||||
|
int remaining = chunk.Length - state.ChunkOffset;
|
||||||
|
int take = Math.Min(remaining, slack);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
fixed (short* src = &chunk[state.ChunkOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst + fill, take * sizeof(short), take * sizeof(short));
|
||||||
|
}
|
||||||
|
state.ChunkOffset += take;
|
||||||
|
fill += take;
|
||||||
|
if (state.ChunkOffset >= chunk.Length)
|
||||||
|
state.CurrentChunk = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
for (int i = fill; i < (int)frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Completion = working chunk drained AND channel empty AND writer completed.
|
||||||
|
if (state.CurrentChunk == null
|
||||||
|
&& downstream.Reader.Count == 0
|
||||||
|
&& downstream.Reader.Completion.IsCompleted)
|
||||||
|
{
|
||||||
|
doneFlag.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Stream(null, outParams, C.SampleRate, C.OutBlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task UpstreamSendLoop(ClientWebSocket ws, Channel<short[]> upstream, CancellationTokenSource cts, SharedState state)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await foreach (var frame in upstream.Reader.ReadAllAsync(cts.Token))
|
||||||
|
{
|
||||||
|
if (state.StopSending) continue;
|
||||||
|
ReadOnlySpan<byte> bytes = MemoryMarshal.AsBytes(frame.AsSpan());
|
||||||
|
string base64 = Convert.ToBase64String(bytes);
|
||||||
|
var msg = new { type = "input_audio_buffer.append", audio = base64 };
|
||||||
|
byte[] json = JsonSerializer.SerializeToUtf8Bytes(msg);
|
||||||
|
await ws.SendAsync(new ArraySegment<byte>(json), WebSocketMessageType.Text, endOfMessage: true, cts.Token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* expected on Cancel */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] upstream send: {ex.Message}");
|
||||||
|
cts.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task ReceiveLoop(ClientWebSocket ws, Channel<short[]> downstream, CancellationTokenSource cts, SharedState state)
|
||||||
|
{
|
||||||
|
var transcript = new StringBuilder();
|
||||||
|
var buffer = new byte[4096];
|
||||||
|
bool firstDeltaSeen = false;
|
||||||
|
DateTime speechStoppedTime = DateTime.MinValue;
|
||||||
|
using var msgBuffer = new MemoryStream();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
msgBuffer.SetLength(0);
|
||||||
|
WebSocketReceiveResult result;
|
||||||
|
do
|
||||||
|
{
|
||||||
|
result = await ws.ReceiveAsync(new ArraySegment<byte>(buffer), cts.Token);
|
||||||
|
if (result.MessageType == WebSocketMessageType.Close) return;
|
||||||
|
msgBuffer.Write(buffer, 0, result.Count);
|
||||||
|
} while (!result.EndOfMessage);
|
||||||
|
|
||||||
|
msgBuffer.Position = 0;
|
||||||
|
using var doc = JsonDocument.Parse(msgBuffer);
|
||||||
|
string type = doc.RootElement.GetProperty("type").GetString() ?? "";
|
||||||
|
|
||||||
|
switch (type)
|
||||||
|
{
|
||||||
|
case "session.created":
|
||||||
|
{
|
||||||
|
string? sid = doc.RootElement.TryGetProperty("session", out var sess)
|
||||||
|
&& sess.TryGetProperty("id", out var idEl) ? idEl.GetString() : null;
|
||||||
|
Console.WriteLine($"[ws] session.created {sid}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "session.updated":
|
||||||
|
Console.WriteLine("[ws] session.updated — session pinned");
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.speech_started":
|
||||||
|
Console.WriteLine("[vad] speech started");
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.speech_stopped":
|
||||||
|
Console.WriteLine("[vad] speech stopped");
|
||||||
|
speechStoppedTime = DateTime.UtcNow;
|
||||||
|
state.StopSending = true;
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.committed":
|
||||||
|
Console.WriteLine("[ws] input_audio_buffer.committed");
|
||||||
|
break;
|
||||||
|
case "response.created":
|
||||||
|
Console.WriteLine("[ws] response.created");
|
||||||
|
break;
|
||||||
|
case "response.audio_transcript.delta":
|
||||||
|
if (doc.RootElement.TryGetProperty("delta", out var tEl))
|
||||||
|
transcript.Append(tEl.GetString());
|
||||||
|
break;
|
||||||
|
case "response.audio.delta":
|
||||||
|
{
|
||||||
|
string b64 = doc.RootElement.GetProperty("delta").GetString() ?? "";
|
||||||
|
byte[] audioBytes = Convert.FromBase64String(b64);
|
||||||
|
short[] samples = new short[audioBytes.Length / 2];
|
||||||
|
Buffer.BlockCopy(audioBytes, 0, samples, 0, audioBytes.Length);
|
||||||
|
if (!firstDeltaSeen)
|
||||||
|
{
|
||||||
|
firstDeltaSeen = true;
|
||||||
|
double elapsed = speechStoppedTime == DateTime.MinValue
|
||||||
|
? -1
|
||||||
|
: (DateTime.UtcNow - speechStoppedTime).TotalSeconds;
|
||||||
|
Console.WriteLine($"[audio] first delta received (t = {elapsed:0.00} s since speech_stopped)");
|
||||||
|
}
|
||||||
|
await downstream.Writer.WriteAsync(samples, cts.Token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "response.audio.done":
|
||||||
|
Console.WriteLine("[ws] response.audio.done");
|
||||||
|
break;
|
||||||
|
case "response.done":
|
||||||
|
state.NoMoreDeltas = true;
|
||||||
|
downstream.Writer.Complete();
|
||||||
|
Console.WriteLine("[ws] response.done");
|
||||||
|
Console.WriteLine($"[transcript] {transcript}");
|
||||||
|
break;
|
||||||
|
case "error":
|
||||||
|
Console.Error.WriteLine($"[error] WS error event: {doc.RootElement.GetRawText()}");
|
||||||
|
cts.Cancel();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
Console.Error.WriteLine($"[ws] ignored type={type}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* expected on Cancel */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] receive loop: {ex.Message}");
|
||||||
|
cts.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sealed class SharedState
|
||||||
|
{
|
||||||
|
public volatile bool MicArmed;
|
||||||
|
public volatile bool StopSending;
|
||||||
|
public volatile bool NoMoreDeltas;
|
||||||
|
// Output-callback-thread-local; do NOT touch from anywhere else.
|
||||||
|
public short[]? CurrentChunk;
|
||||||
|
public int ChunkOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
static class C
|
||||||
|
{
|
||||||
|
public const int SampleRate = 24_000;
|
||||||
|
public const int Channels = 1;
|
||||||
|
public const uint MicBlockFrames = 1920; // 40 ms @ 24 kHz
|
||||||
|
public const uint OutBlockFrames = 1024; // ~43 ms @ 24 kHz
|
||||||
|
public const string ModelName = "gpt-realtime";
|
||||||
|
public const string Voice = "alloy";
|
||||||
|
public const string Instructions = "Reply in one short sentence.";
|
||||||
|
public const double BeepHz = 880.0;
|
||||||
|
public const double BeepSeconds = 0.2;
|
||||||
|
public const int BeepDrainMs = 100;
|
||||||
|
public const int UpstreamChannelCapacity = 64;
|
||||||
|
public const int DownstreamChannelCapacity = 64;
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Verify the project builds cleanly**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet build tests/04a-realtime-oneshot-cs -c Release
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `Build succeeded` with 0 errors. A handful of nullable-reference warnings on `state.CurrentChunk` access inside the `unsafe` block is fine if it appears; the runtime checks above ensure non-null.
|
||||||
|
|
||||||
|
If `<AllowUnsafeBlocks>` is missing from the csproj, the build will fail with `CS0227: Unsafe code may only appear if compiling with /unsafe`. Confirm Task 1's csproj has `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`.
|
||||||
|
|
||||||
|
If you see `CS0246: The type or namespace 'Stream' could not be found` or a clash with `System.IO.Stream`, confirm the `using Stream = PortAudioSharp.Stream;` line is at the top of `Program.cs`.
|
||||||
|
|
||||||
|
If you see `error CS8803: Top-level statements must precede namespace and type declarations` or `CS8805: Program using top-level statements must be an entry point`, the file ordering is wrong. The exact required order in `Program.cs` is:
|
||||||
|
|
||||||
|
1. `using` directives
|
||||||
|
2. Top-level statements (the imperative body — `Environment.SetEnvironmentVariable`, `apiKey = LoadApiKey()`, `PortAudio.Initialize`, the `try {} finally {}` block)
|
||||||
|
3. Top-level static method declarations (`LoadApiKey`, `FindUsbDevice`, `MakeBeep`, `SendSessionUpdate`, `OpenInputStream`, `OpenOutputStream`, `UpstreamSendLoop`, `ReceiveLoop`)
|
||||||
|
4. Top-level type declarations (`sealed class SharedState`, `static class C`, `static class Libc`) — NOT wrapped in a `namespace` block.
|
||||||
|
|
||||||
|
The file above follows this order; do not reorder.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify `dotnet publish` produces a runnable binary**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet publish tests/04a-realtime-oneshot-cs -c Release -r linux-arm64 --self-contained -o /tmp/probe-cs-4a-out
|
||||||
|
ls /tmp/probe-cs-4a-out/Probe
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
- `Probe` (executable) exists.
|
||||||
|
- No `models/` directory needed.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/04a-realtime-oneshot-cs/Program.cs
|
||||||
|
git commit -m "Test 4a: WS plumbing + audio streams + server VAD round-trip"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Write `bin/probe-cs-4a` deploy script with API key precondition
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `bin/probe-cs-4a`
|
||||||
|
|
||||||
|
This is `bin/probe-cs-3` with three string swaps (`03-full-cycle-cs` → `04a-realtime-oneshot-cs`, `probe-cs-3` → `probe-cs-4a`, `/tmp/probe-cs-3-out` → `/tmp/probe-cs-4a-out`) plus ONE new precondition step: check that `~/.openai_key` exists on the Pi before publishing. This catches the silent "key missing" failure mode at the deploy step instead of 30 s into publish + scp.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the deploy script**
|
||||||
|
|
||||||
|
Create `bin/probe-cs-4a` with exactly this content:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-4a-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> precondition: ~/.openai_key present on Pi"
|
||||||
|
if ! sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'test -f ~/.openai_key && test -r ~/.openai_key'; then
|
||||||
|
echo "ERROR: ~/.openai_key missing or unreadable on Pi."
|
||||||
|
echo "Create it on the Pi with:"
|
||||||
|
echo " sshpass -p '$PI_PASS' ssh $PI_USER@$PI_HOST \\"
|
||||||
|
echo " 'umask 077 && printf \"%s\\n\" \"sk-PASTE-YOUR-KEY\" > ~/.openai_key'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " OK."
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/04a-realtime-oneshot-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-4a/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-4a && mkdir ~/probe-cs-4a'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-4a/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi (Ctrl-C from this terminal stops the probe)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-4a && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Make it executable**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chmod +x bin/probe-cs-4a
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add bin/probe-cs-4a
|
||||||
|
git commit -m "Test 4a: bin/probe-cs-4a deploy script with ~/.openai_key precondition"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: One-time Pi setup — place `~/.openai_key` on the Pi
|
||||||
|
|
||||||
|
**Files:** (none — this is a one-time remote setup task)
|
||||||
|
|
||||||
|
This task only runs once per Pi. If the file already exists from a previous deploy or another test, skip steps 1–2 and verify with step 3.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Ask the user for an API key**
|
||||||
|
|
||||||
|
Before running this task, the user must have an OpenAI API key with Realtime API access (`sk-…`). If the user hasn't given it to you yet, ask:
|
||||||
|
|
||||||
|
> "I need your OpenAI API key (`sk-…`) to write it to `~/.openai_key` on the Pi. Paste it and I'll redact it from the shell history immediately. Do not paste it into any other context."
|
||||||
|
|
||||||
|
The key the user provides MUST be treated as secret and MUST NOT appear in commit messages, file contents anywhere in the repo, console echoes, or logs. The only place it goes is the file on the Pi.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Write the key to the Pi**
|
||||||
|
|
||||||
|
Replace `sk-PASTE-YOUR-KEY-HERE` below with the actual key value. The `umask 077` ensures the file is created with mode 600 (rw for owner only). The `printf "%s\n"` appends exactly one newline.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sshpass -p 'assistant' ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
pi@192.168.50.115 \
|
||||||
|
'umask 077 && printf "%s\n" "sk-PASTE-YOUR-KEY-HERE" > ~/.openai_key'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify file exists and is mode 600**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 \
|
||||||
|
'ls -l ~/.openai_key && wc -c ~/.openai_key'
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected output (mode `-rw-------`, owner `pi`, size > 50 bytes):
|
||||||
|
|
||||||
|
```
|
||||||
|
-rw------- 1 pi pi <size> <date> /home/pi/.openai_key
|
||||||
|
<size> /home/pi/.openai_key
|
||||||
|
```
|
||||||
|
|
||||||
|
If mode is not `-rw-------`, fix with `chmod 600 ~/.openai_key`.
|
||||||
|
|
||||||
|
If size is suspiciously small (< 20 bytes), the key wasn't pasted correctly — re-do step 2.
|
||||||
|
|
||||||
|
(No commit — this step modifies state on the Pi, not the repo.)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: Hardware verification on the Pi
|
||||||
|
|
||||||
|
This task is the actual pass/fail gate. No automated tests; verification is the human ear + console output. If anything in this task fails, do NOT mark the implementation complete — investigate and either fix in code or update the spec to reflect what actually works.
|
||||||
|
|
||||||
|
**Files:** (none — this is a runtime verification task)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Confirm the Pi is reachable**
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ping -c 2 192.168.50.115
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 2/2 replies. If unreachable, stop and ask the user.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Deploy + run the probe**
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./bin/probe-cs-4a
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected console output (abridged, with elision):
|
||||||
|
|
||||||
|
```
|
||||||
|
>> precondition: ~/.openai_key present on Pi
|
||||||
|
OK.
|
||||||
|
>> dotnet publish (linux-arm64, self-contained)
|
||||||
|
... (build output) ...
|
||||||
|
>> scp to pi@192.168.50.115:~/probe-cs-4a/ (wipe-and-replace)
|
||||||
|
>> ssh + run on Pi (Ctrl-C from this terminal stops the probe)
|
||||||
|
[ws] connecting to wss://api.openai.com/v1/realtime?model=gpt-realtime
|
||||||
|
[ws] connected
|
||||||
|
[ws] session.created sess_xxxxxxxxxxxx
|
||||||
|
[ws] session.updated — session pinned
|
||||||
|
Using device <N> ('USB Speaker Phone (LISTENAI ...)')
|
||||||
|
[beep]
|
||||||
|
Speak now.
|
||||||
|
```
|
||||||
|
|
||||||
|
At this point the probe is waiting for you to speak. Say a single short question into the USB Speaker Phone — for example: **"What's the capital of France?"** — at normal volume from ~1 m.
|
||||||
|
|
||||||
|
Continue with verification in step 3.
|
||||||
|
|
||||||
|
If the publish step errors with a runtime crash invoking `Libc.setenv`, confirm Task 2's `[DllImport]` line for `setenv` is present.
|
||||||
|
|
||||||
|
If the Pi run errors with `Cannot open shared library libportaudio.so`, the self-contained publish didn't bundle the native libs — confirm `<SelfContained>true</SelfContained>` and `<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>` in Task 1's csproj.
|
||||||
|
|
||||||
|
If `[error] WS connect failed: ... HTTP 401` appears, the API key was rejected. Re-check Task 4 step 3 and confirm the key on the Pi starts with `sk-` and the user pasted the right value.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify the full conversational round-trip**
|
||||||
|
|
||||||
|
After saying your question in step 2, observe:
|
||||||
|
|
||||||
|
1. `[vad] speech started` appears in the console as you begin speaking.
|
||||||
|
2. `[vad] speech stopped` appears shortly after you stop speaking (within ~0.5–1 s, governed by `silence_duration_ms = 500` in session.update).
|
||||||
|
3. `[ws] input_audio_buffer.committed` and `[ws] response.created` follow.
|
||||||
|
4. `[audio] first delta received (t = X.XX s since speech_stopped)` appears with a small latency value (~1–2 s).
|
||||||
|
5. The assistant's spoken reply comes out of the USB Speaker Phone speaker — clearly, intelligibly, recognisable as English speech.
|
||||||
|
6. `[ws] response.audio.done` and `[ws] response.done` follow.
|
||||||
|
7. `[transcript] <the assistant's reply, as text>` appears — should match what you heard.
|
||||||
|
8. `[ws] closing` and `bye` appear; the probe exits.
|
||||||
|
|
||||||
|
**Pass criterion 1:** End-to-end round-trip works once. Probe exits 0.
|
||||||
|
**Pass criterion 2:** Response is intelligible (no clipping, no dropouts longer than one ~43 ms callback).
|
||||||
|
**Pass criterion 8:** Probe exits within ~500 ms of the last audio sample being heard.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Check latency from the log line**
|
||||||
|
|
||||||
|
Look at the `[audio] first delta received (t = X.XX s since speech_stopped)` line printed in step 3.
|
||||||
|
|
||||||
|
**Pass criterion 3:** `X.XX` is under ~2 s. Slower is acceptable if logged here; over ~5 s is a fail.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Scan for warnings**
|
||||||
|
|
||||||
|
Re-read the full console output. Confirm there are **no** lines of these forms during normal operation (between `[beep]` and `[ws] response.done`):
|
||||||
|
|
||||||
|
- `[status] upstream behind, dropping mic frame`
|
||||||
|
- `[status] input overflow`
|
||||||
|
- `[status] unexpected input frameCount=...`
|
||||||
|
- `[error] WS ...`
|
||||||
|
- `[ws] ignored type=...` (one or two of these are OK — the API may send events we don't handle; sustained occurrences during normal flow indicate a missing handler we should add)
|
||||||
|
|
||||||
|
**Pass criterion 4:** No `[status] upstream behind` warnings.
|
||||||
|
**Pass criterion 5:** No `[error] WS …` events.
|
||||||
|
**Pass criterion 6:** No PortAudio `InputOverflow` status flags.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Test Ctrl-C clean exit (separate run)**
|
||||||
|
|
||||||
|
Restart the probe:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./bin/probe-cs-4a
|
||||||
|
```
|
||||||
|
|
||||||
|
When `Speak now.` appears, press Ctrl-C in the workstation terminal **without** speaking.
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
|
||||||
|
```
|
||||||
|
^C
|
||||||
|
[ws] closing
|
||||||
|
bye
|
||||||
|
$
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify with `echo $?` — exit code 0.
|
||||||
|
|
||||||
|
**Pass criterion 7:** Ctrl-C produces a clean exit (no stack trace, terminal returns to prompt, exit code 0 when invoked during the steady listening phase).
|
||||||
|
|
||||||
|
- [ ] **Step 7: Update `findings.md` with the outcome**
|
||||||
|
|
||||||
|
Add a new section `## Test 4a outcome (2026-06-12 — OpenAI Realtime one-shot voice round-trip in C#)` at the bottom of `findings.md` summarising:
|
||||||
|
|
||||||
|
- Pass / fail per criterion (1–8).
|
||||||
|
- Latency measured (the `X.XX` value from step 4).
|
||||||
|
- Any new gotchas discovered during implementation or hardware verification — especially:
|
||||||
|
- Any OpenAI Realtime API surface differences from the spec (event types not in our switch, session.update fields rejected, model name issues).
|
||||||
|
- Any audio-format quirks (24 kHz playback dropouts, sample-rate-mismatch artifacts).
|
||||||
|
- Any concurrency surprises (channel deadlocks, send-loop wedges).
|
||||||
|
- Any errors in our LoadApiKey / mode-600 check.
|
||||||
|
- Implications for 4b (what we still need to debug there vs. what 4a proved out).
|
||||||
|
- Reference paths (spec + plan + code + deploy script).
|
||||||
|
|
||||||
|
Use the existing Test 1, 2, and 3 outcome sections in `findings.md` as the template.
|
||||||
|
|
||||||
|
- [ ] **Step 8: Commit the findings update**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add findings.md
|
||||||
|
git commit -m "Findings: Test 4a OpenAI Realtime one-shot voice round-trip outcome"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 9: Final status report**
|
||||||
|
|
||||||
|
Report to the user:
|
||||||
|
|
||||||
|
- Pass/fail per criterion (1 through 8 from spec).
|
||||||
|
- Any criteria that failed and what was discovered.
|
||||||
|
- The measured round-trip latency value.
|
||||||
|
- A one-line summary of whether Test 4a unblocks Test 4b (i.e. whether the OpenAI Realtime surface + audio plumbing is proven, so 4b can focus on wakeword + loop + 16↔24 kHz coexistence).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-review notes
|
||||||
|
|
||||||
|
- **Spec coverage:**
|
||||||
|
- Goal (single-binary one-shot probe) → Tasks 1+2+3+5.
|
||||||
|
- `~/.openai_key` precondition + file-on-Pi storage → Task 3 (script) + Task 4 (one-time setup) + Task 2 (`LoadApiKey` with mode-600 warning).
|
||||||
|
- WS connect with `Authorization` + `OpenAI-Beta` headers → Task 2 (`ws.Options.SetRequestHeader` + `ConnectAsync`).
|
||||||
|
- `session.update` payload with `server_vad` → Task 2 (`SendSessionUpdate` helper, exact JSON shape).
|
||||||
|
- Mic input stream at 24 kHz with `_micArmed` gate → Task 2 (`OpenInputStream`, `state.MicArmed` check in callback).
|
||||||
|
- UpstreamSendLoop (base64 + `input_audio_buffer.append`) → Task 2 (`UpstreamSendLoop`).
|
||||||
|
- ReceiveLoop with `JsonDocument` switch on `type` → Task 2 (`ReceiveLoop`, complete handler table).
|
||||||
|
- Output stream drain-or-zero-fill + `_noMoreDeltas` + `Reader.Completion.IsCompleted` completion → Task 2 (`OpenOutputStream` callback).
|
||||||
|
- Startup beep through `downstreamChannel` → Task 2 (lines after `outputStream.Start()`).
|
||||||
|
- `ManualResetEventSlim doneFlag` + clean shutdown via `finally` → Task 2 (try/finally block).
|
||||||
|
- Ctrl-C → `Console.CancelKeyPress` → `cts.Cancel` → `OperationCanceledException` → finally → exit 0 → Tasks 2 + 5 step 6.
|
||||||
|
- `bin/probe-cs-4a` with ssh -t + wipe-and-replace + key precondition → Task 3.
|
||||||
|
- Pass criteria 1–8 → Task 5 steps 3, 4, 5, 6.
|
||||||
|
- Findings update → Task 5 steps 7–8.
|
||||||
|
- **Placeholder scan:** no TBD/TODO; every code step shows the actual code; every command shows expected output. The single user-substitution placeholder is `sk-PASTE-YOUR-KEY-HERE` in Task 4 step 2, which is explicitly called out as a replacement target.
|
||||||
|
- **Type consistency:**
|
||||||
|
- `SharedState` fields (`MicArmed`, `StopSending`, `NoMoreDeltas`, `CurrentChunk`, `ChunkOffset`) referenced consistently from `OpenInputStream`, `OpenOutputStream`, `UpstreamSendLoop`, `ReceiveLoop`, and the top-level statements.
|
||||||
|
- `Channel<short[]>` used for both upstream and downstream; the `short[]` element type is consistent everywhere.
|
||||||
|
- `ManualResetEventSlim doneFlag` passed by reference into `OpenOutputStream` so the callback can `Set()` it; main thread awaits it via `Wait(cts.Token)`.
|
||||||
|
- `CancellationTokenSource cts` is the single source of cancellation across all three loops + the main thread.
|
||||||
|
- **Scope:** the spec is a single subsystem (one probe). One plan is the right granularity.
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Test 2 — wakeword detection in C# — kickoff prompt
|
||||||
|
|
||||||
|
Paste the block below into a fresh Claude conversation in this repo's working directory. The session will load `CLAUDE.md` automatically (which points at `findings.md`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
We just landed Test 1 of the C# Pi-client probe (record/play) on the `fresh-start` branch — proven on real hardware. Now do Test 2: re-implement the wakeword probe in C# under `tests/02-wakeword-cs/`.
|
||||||
|
|
||||||
|
**Before any code, read these in order:**
|
||||||
|
|
||||||
|
1. `CLAUDE.md` — project context + SSH access to the Pi.
|
||||||
|
2. `findings.md` — canonical "what we know works." Especially the `## C# probe outcome (2026-06-12 — Test 1 ported to C#)` section: PortAudioSharp2 1.0.6, the .NET/libc env-var split (`Environment.SetEnvironmentVariable` does NOT reach `getenv()`, must P/Invoke `setenv`), the USB Speaker Phone's 48 kHz-only native rate (needs `PA_ALSA_PLUGHW=1`), and the open questions about wakeword engines.
|
||||||
|
3. `tests/01-record-play-cs/Program.cs` and `tests/01-record-play-cs/Probe.csproj` — the patterns we'll reuse: PortAudio init + `Libc.setenv` workaround, callback InputStream, lock geometry, `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`.
|
||||||
|
4. `bin/probe-cs` — the build + scp + ssh deploy pattern. New probe will need a similar `bin/probe-cs-2` (or reuse the same script with a target arg — decide in the brainstorm).
|
||||||
|
5. `tests/02-wakeword/main.py` — the Python original. Threshold 0.5, cooldown 1 s, openwakeword's stock "alexa" model. Pass criteria are listed in its docstring.
|
||||||
|
6. The two prior spec/plan docs for shape: `docs/superpowers/specs/2026-06-12-record-play-csharp-probe-design.md` and `docs/superpowers/plans/2026-06-12-record-play-csharp-probe.md`. We'll follow the same workflow.
|
||||||
|
|
||||||
|
**Goal of Test 2 in C#:** the same pass criteria as the Python version (from its docstring):
|
||||||
|
- "alexa" at normal volume from ~1 m triggers within ~1 s, ≥8/10 attempts.
|
||||||
|
- ≤1 false positive per minute of unrelated speech.
|
||||||
|
- Process stays under 50% of one core (check with `htop` on the Pi).
|
||||||
|
|
||||||
|
Probe is throwaway under `tests/02-wakeword-cs/`. No automated tests. Verification is hardware.
|
||||||
|
|
||||||
|
**The hard question — which wakeword engine?** openwakeword has no .NET port. Three real options to brainstorm with me before committing to code:
|
||||||
|
|
||||||
|
- **(A) Port openwakeword's inference pipeline to C# using `Microsoft.ML.OnnxRuntime`.** Three ONNX files (mel extractor → Google speech embedder → keyword classifier). ~200–400 lines of glue. We own the port; no commercial lock-in; same stock "alexa" model the Python probe used. Carries forward to the eventual custom wakeword we'll train later.
|
||||||
|
- **(B) Picovoice Porcupine's .NET SDK.** First-class .NET binding, runs on linux-arm64, low CPU. Commercial / API-key gated. Free personal tier exists. Faster to integrate but locks us in.
|
||||||
|
- **(C) Vosk + keyword spotting.** Heavier model, .NET binding exists, overkill for one wakeword.
|
||||||
|
|
||||||
|
Use `superpowers:brainstorming` to walk through this with me. Lead with your recommendation and the trade-offs. Don't write spec/code until I've picked an approach.
|
||||||
|
|
||||||
|
**Specific gotchas to expect:**
|
||||||
|
|
||||||
|
- ALSA plug layer: `PA_ALSA_PLUGHW=1` via libc `setenv` (NOT just `Environment.SetEnvironmentVariable`). Pattern is already in `tests/01-record-play-cs/Program.cs`.
|
||||||
|
- onnxruntime on Pi prints `/sys/class/drm/card0` warnings while probing for GPUs. The findings doc mentions `ORT_DISABLE_GPU_DETECTION=1` (or similar) as a possible suppressor — if you want that, set it via `Libc.setenv` too.
|
||||||
|
- The USB Speaker Phone is mic AND speaker. The Python probe uses `sd.play(beep) + sd.wait()` on the same thread as the input — the findings doc flags this as a real bug (drops ~200 ms of input right after detection). Decide in the spec whether to fix it for the probe or carry over the bug for parity.
|
||||||
|
- `Stream` ambiguity with `System.IO.Stream` when `ImplicitUsings` is on — same `using Stream = PortAudioSharp.Stream;` workaround.
|
||||||
|
|
||||||
|
**Workflow:** brainstorming → spec → plan → subagent-driven implementation, same flow used for Test 1. Spec at `docs/superpowers/specs/2026-06-12-test-2-wakeword-csharp-design.md`, plan at `docs/superpowers/plans/2026-06-12-test-2-wakeword-csharp.md` (or whatever date is current).
|
||||||
|
|
||||||
|
Pi: `sshpass -p 'assistant' ssh pi@192.168.50.115`. Branch you'll work on: `fresh-start` (already current, do not create a new one unless I ask).
|
||||||
@@ -1,516 +0,0 @@
|
|||||||
# Smart Assistant — design
|
|
||||||
|
|
||||||
**Date:** 2026-06-11
|
|
||||||
**Status:** approved (brainstorming complete)
|
|
||||||
**Scope:** main build of the voice assistant, replacing the throwaway prototypes in `tests/`. Two deliverables: a Python client on the Raspberry Pi, and an ASP.NET backend (with embedded React management UI) deployed to Coolify. Brings up everything needed to wake on a wakeword, hold a multi-turn conversation through the OpenAI Realtime API, persist a text history per device, run a small built-in tool registry, and let users manage their devices through a browser.
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
The prototype phase (`docs/superpowers/specs/2026-06-11-voice-assistant-prototypes-design.md`, results in `findings.md`) proved that recording/playback and openwakeword both work on the target Pi. The main assistant is built on top of those findings.
|
|
||||||
|
|
||||||
Key carry-forwards from the prototypes:
|
|
||||||
|
|
||||||
- USB Speaker Phone (ALSA card 3, Anhui LISTENAI) is the only mic/speaker. 48 kHz native; `PA_ALSA_PLUGHW=1` set before importing `sounddevice` lets us open at any rate.
|
|
||||||
- `python-sounddevice` + `numpy` are the audio stack.
|
|
||||||
- `openwakeword` runs at ~0.6 s load, well under 50% of one core; the `alexa` stock model is reused for v1 (custom-trained wakeword is a separate follow-up task).
|
|
||||||
- `openwakeword==0.6.0` needs `--no-deps` install on Python 3.13 because its `tflite-runtime` pin has no wheel; the prototype's two-pass install pattern is kept.
|
|
||||||
- Speaker hears its own output → detector must be gated while playing audio, and uplink to OpenAI must be muted while the assistant is speaking.
|
|
||||||
- TTS/beep must not block the input stream — callback-driven InputStream + persistent OutputStream is the pattern.
|
|
||||||
|
|
||||||
## Goals
|
|
||||||
|
|
||||||
1. A fresh Raspberry Pi can be paired to a user account by running one curl-piped install command and entering a short code.
|
|
||||||
2. The Pi wakes on the wakeword, opens a Realtime conversation, supports multi-turn until the user says "bye" or goes idle, and plays optional `wake.wav` / `sleep.wav` cues.
|
|
||||||
3. A signed-in user sees only their own devices and conversations in the management UI.
|
|
||||||
4. An admin user can manage system-wide defaults and other users.
|
|
||||||
5. Per-device configuration: name, system prompt override, voice, model, idle timeout, enabled-tools toggles. Changes take effect on the next session.
|
|
||||||
6. Text-only conversation history per device, viewable in the UI.
|
|
||||||
7. Built-in C# tool registry with `end_session`, `get_current_time`, `set_volume` shipped in v1.
|
|
||||||
8. Backend deployed to Coolify as one container with one domain; deploys triggered fire-and-forget via the Coolify v4 API.
|
|
||||||
9. The install script is the update mechanism: re-running it upgrades the client in place without re-pairing.
|
|
||||||
|
|
||||||
## Non-goals (v1)
|
|
||||||
|
|
||||||
- Long-term memory across sessions.
|
|
||||||
- MCP-based external tools (only built-in C# tools).
|
|
||||||
- Audio recordings of conversations (text only).
|
|
||||||
- Live conversation tailing or audio meters in the UI.
|
|
||||||
- Per-device upload of `wake.wav`/`sleep.wav` via UI (filesystem only — v1.1).
|
|
||||||
- Custom-trained wakeword (uses the stock `alexa` model for v1).
|
|
||||||
- Barge-in (interrupting the assistant mid-reply).
|
|
||||||
- Multi-language (English only).
|
|
||||||
- Offline / local-only fallback when the network is down.
|
|
||||||
- OIDC sign-in (local ASP.NET Identity only).
|
|
||||||
- `.deb` packaging for the client.
|
|
||||||
- Load / performance testing.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Topology
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────┐ pair code / token ┌──────────────────────────────────────────┐
|
|
||||||
│ Browser (admin UI) │ ──────HTTPS (REST)────────▶ │ │
|
|
||||||
│ React SPA │ │ ASP.NET backend (Coolify container) │
|
|
||||||
└─────────────────────┘ │ │
|
|
||||||
│ ┌────────────┐ ┌────────────────────┐ │
|
|
||||||
┌─────────────────────┐ WSS audio + control │ │ REST/UI │ │ Device hub │ │
|
|
||||||
│ Pi client (Python) │ ◀───────────────────────────▶│ │ + Identity│ │ (per-device WS) │ │
|
|
||||||
│ wakeword + audio │ │ └────────────┘ └─────────┬──────────┘ │
|
|
||||||
└─────────────────────┘ │ │ │
|
|
||||||
│ ┌────────────┐ ┌────────▼──────────┐ │
|
|
||||||
│ │ Tools │ │ Realtime relay │ │
|
|
||||||
│ │ registry │◀─┤ (per-session WS │ │
|
|
||||||
│ └────────────┘ │ upstream) │ │
|
|
||||||
│ └─────────┬──────────┘ │
|
|
||||||
│ SQLite (named volume) │ │
|
|
||||||
└────────────────────────────┼────────────┘
|
|
||||||
│ WSS
|
|
||||||
▼
|
|
||||||
OpenAI Realtime API
|
|
||||||
```
|
|
||||||
|
|
||||||
One container; React SPA served by the same ASP.NET process. SQLite file lives on a Coolify named volume. The OpenAI API key is held server-side; clients never see it.
|
|
||||||
|
|
||||||
### Why backend relay (not direct Pi ↔ OpenAI)
|
|
||||||
|
|
||||||
- One API key, server-side only.
|
|
||||||
- All user-side and assistant-side transcripts land in our database directly off the upstream WS events (no separate STT pass).
|
|
||||||
- Tool definitions and changes don't require a client redeploy.
|
|
||||||
- Backend can push audio to a Pi on its own (future feature) over the same WS.
|
|
||||||
|
|
||||||
### Backend
|
|
||||||
|
|
||||||
Single ASP.NET 9 project (`backend/`) organised into modules:
|
|
||||||
|
|
||||||
```
|
|
||||||
backend/
|
|
||||||
├── Program.cs # composition root, EF migrate, route + hub mapping
|
|
||||||
├── Auth/ # ASP.NET Identity, roles (Admin, User), invite flow
|
|
||||||
├── Devices/ # device CRUD, pairing-code issuance, token mgmt
|
|
||||||
├── DeviceHub/ # WebSocket handler at /device, per-device session state
|
|
||||||
├── Realtime/ # RealtimeRelay: per-conversation upstream WS to OpenAI
|
|
||||||
├── Tools/ # ITool interface, registry, built-in tools
|
|
||||||
├── Conversations/ # persistence of turns, transcripts, tool calls
|
|
||||||
├── Install/ # serves install.sh and client.tar.gz
|
|
||||||
├── Data/ # EF Core DbContext, migrations, SQLite
|
|
||||||
└── wwwroot/ # built React SPA (Vite output) + client.tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Key services (DI):
|
|
||||||
|
|
||||||
- `DeviceRegistry` (singleton) — in-memory map `deviceId → ActiveDevice`, tracks online state and last frame timestamp.
|
|
||||||
- `RealtimeSessionFactory` (singleton) — given a `DeviceContext`, opens an upstream WSS to OpenAI and returns a `RealtimeSession` instance bound to one conversation.
|
|
||||||
- `ToolRegistry` (singleton) — static list of `ITool`s registered at startup, filterable by per-device enabled set.
|
|
||||||
- `ConversationLog` (scoped) — append-only writer for turns; flushes per event so a crash mid-conversation still preserves what was said.
|
|
||||||
- `OpenAIKeyProvider` (singleton) — reads `OPENAI_API_KEY` from env.
|
|
||||||
|
|
||||||
### Pi client
|
|
||||||
|
|
||||||
Python package (`client/`), deployed to `~/assistant/code/` on the Pi:
|
|
||||||
|
|
||||||
```
|
|
||||||
client/
|
|
||||||
├── main.py # entry: sets PA_ALSA_PLUGHW BEFORE importing sounddevice
|
|
||||||
├── audio.py # USB device discovery, persistent in/out streams
|
|
||||||
├── wakeword.py # openwakeword wrapper + gating flag
|
|
||||||
├── session.py # backend WS client: binary audio + JSON control
|
|
||||||
├── playback.py # queue-fed playback, wake/sleep wav hooks
|
|
||||||
├── state.py # state machine
|
|
||||||
├── config.py # loads ~/assistant/state/config.json
|
|
||||||
├── pair.py # one-shot: prompt for code, hit /api/pair, write config.json
|
|
||||||
└── log.py # structured logging to journald
|
|
||||||
```
|
|
||||||
|
|
||||||
State persists at `~/assistant/state/`:
|
|
||||||
- `config.json` (mode 0600) — `device_id`, `device_token`, `backend_ws`
|
|
||||||
- `wake.wav` / `sleep.wav` — optional cues, played if present
|
|
||||||
- (room for future per-device assets; never rsynced/overwritten by deploys)
|
|
||||||
|
|
||||||
`~/assistant/code/` is the only directory that the install script replaces on update; `~/assistant/state/` is untouched.
|
|
||||||
|
|
||||||
### State machine
|
|
||||||
|
|
||||||
```
|
|
||||||
wakeword fires session_started
|
|
||||||
┌─────────────────────┐ ┌──────────────────────┐
|
|
||||||
│ ▼ │ ▼
|
|
||||||
[IDLE] ─────────► [WAKE_PENDING] ─────► [LISTENING] ───────► [ASSISTANT_SPEAKING]
|
|
||||||
▲ │ ▲ │
|
|
||||||
│ │ │ assistant done │
|
|
||||||
│ └───┴──────────────────┘
|
|
||||||
│ session_ended
|
|
||||||
└─────────────────────────────── (idle timeout OR end_session tool)
|
|
||||||
plays sleep.wav, re-enables wakeword
|
|
||||||
```
|
|
||||||
|
|
||||||
- **IDLE** — wakeword detector enabled, no upstream audio. Persistent backend WS open with heartbeats.
|
|
||||||
- **WAKE_PENDING** — wakeword fired; play `wake.wav` (non-blocking); send `wake`; wait for `session_started`. Wakeword disabled.
|
|
||||||
- **LISTENING** — mic frames stream to backend; server-VAD on OpenAI decides turn boundaries; we don't compute local VAD.
|
|
||||||
- **ASSISTANT_SPEAKING** — uplink muted; inbound frames flow to playback queue.
|
|
||||||
- On `assistant_done` from backend → LISTENING again (uplink unmuted).
|
|
||||||
- On `session_ended` from backend → wait for current audio to drain, play `sleep.wav`, send `playback_done(for=sleep)` → IDLE.
|
|
||||||
|
|
||||||
### Audio plumbing
|
|
||||||
|
|
||||||
- **Wire format**: PCM16 little-endian, 24 kHz mono, exactly 1920 samples (3840 bytes, 80 ms) per binary frame. Matches OpenAI Realtime's default `pcm16`. No transcoding in the backend.
|
|
||||||
- **Capture**: single InputStream, callback-driven, blocksize 1920, sample rate 24 000 Hz. Callback only enqueues — never blocks.
|
|
||||||
- **Playback**: single OutputStream opened at startup, persistent. Playback worker pulls from an `asyncio.Queue` and writes — never `sd.play()/sd.wait()` (which would block the InputStream the way the prototype did).
|
|
||||||
- **Wakeword feed**: each captured 1920-sample frame is downsampled to 1280 samples (80 ms at 16 kHz) via `scipy.signal.resample_poly(frame, up=2, down=3)` and fed to `model.predict()`.
|
|
||||||
- `PA_ALSA_PLUGHW=1` is set at the very top of `main.py` before any `sounddevice` import.
|
|
||||||
|
|
||||||
### Threading model
|
|
||||||
|
|
||||||
- Main thread runs an asyncio loop owning the backend WS.
|
|
||||||
- PortAudio invokes the InputStream callback on its own thread; the callback writes mic frames to a bounded `queue.Queue`. On overflow, oldest frame is dropped and a warning is logged.
|
|
||||||
- A bridge thread reads `queue.Queue`, runs wakeword.predict() when state is IDLE/WAKE_PENDING, and forwards bytes to the asyncio loop via `loop.call_soon_threadsafe` for upstream sends when state is LISTENING.
|
|
||||||
- A playback worker (asyncio task) drains an `asyncio.Queue` and writes to the OutputStream in a thread executor.
|
|
||||||
|
|
||||||
### Reconnect & heartbeat
|
|
||||||
|
|
||||||
- Pi → backend WS reconnects with exponential backoff 1 → 30 s.
|
|
||||||
- While disconnected, state stays IDLE; wakewords are ignored.
|
|
||||||
- An in-flight conversation does **not** survive a network drop — backend marks conversation `EndReason="error"`.
|
|
||||||
- Client sends `{"type":"ping"}` every 15 s when IDLE; backend uses it to update `Devices.LastSeenAt` and drive the UI online indicator.
|
|
||||||
|
|
||||||
## Wire protocol (Pi ↔ backend)
|
|
||||||
|
|
||||||
One persistent WebSocket at `wss://<backend>/device`. Authenticated via `Authorization: Bearer <device-token>` on the upgrade request; connection rejected if the token is revoked or unknown.
|
|
||||||
|
|
||||||
### Binary frames
|
|
||||||
|
|
||||||
Raw PCM16 LE, 24 kHz mono, exactly 1920 samples (3840 bytes). No header. Direction inferred from sender. Backend drops binary frames received outside an active session. Wrong-sized frames are dropped silently with a log entry (protects against runaways without surfacing an error).
|
|
||||||
|
|
||||||
### Text frames (JSON envelopes)
|
|
||||||
|
|
||||||
Client → backend:
|
|
||||||
|
|
||||||
| `type` | Fields | Notes |
|
|
||||||
| ----------------- | ----------------------------------- | ----------------------------------------------------------- |
|
|
||||||
| `hello` | `client_version`, `device_id` | Sent on connect; backend replies `hello_ack` with the device's config snapshot. |
|
|
||||||
| `ping` | — | Every 15 s while IDLE. Backend replies `pong`. |
|
|
||||||
| `wake` | `at` (client ms timestamp) | Triggers backend to open upstream Realtime WS. |
|
|
||||||
| `cancel` | — | User-initiated abort (e.g. SIGINT). Backend closes session. |
|
|
||||||
| `playback_done` | `for` (`wake` \| `sleep` \| `assistant_turn`) | Lets backend know when audio finished playing on the Pi (needed to time `session_ended` after `sleep.wav`). |
|
|
||||||
| `tool_result` | `call_id`, `ok`, `result?`, `error?` | Reply to a Pi-side tool call (e.g. `set_volume`). |
|
|
||||||
|
|
||||||
Backend → client:
|
|
||||||
|
|
||||||
| `type` | Fields | Notes |
|
|
||||||
| ------------------ | ----------------------------------- | ---------------------------------------------------------- |
|
|
||||||
| `hello_ack` | `config` (voice/model/prompt/tools/idle_timeout) | Client uses this to set local state. |
|
|
||||||
| `pong` | — | |
|
|
||||||
| `config_updated` | `config` | Pushed when admin saves changes in the UI; takes effect next session. |
|
|
||||||
| `session_started` | `conversation_id` | Realtime upstream is ready; client may start uplink. |
|
|
||||||
| `assistant_done` | — | Assistant turn ended; client unmutes uplink. |
|
|
||||||
| `session_ended` | `reason` (`tool` \| `idle` \| `error`) | Client plays `sleep.wav` (if present), then `playback_done(for=sleep)`. |
|
|
||||||
| `tool_call` | `call_id`, `name`, `arguments` | Invoked for tools whose execution requires the Pi. |
|
|
||||||
| `play_audio` | `kind` (`session` \| `notification`) | Marker; subsequent binary frames are payload. Ended by `play_audio_end`. |
|
|
||||||
| `play_audio_end` | — | End-of-payload marker for `play_audio(kind=notification)`. |
|
|
||||||
| `error` | `code`, `message`, `fatal?` | Logged on client; non-fatal unless `fatal=true`. |
|
|
||||||
|
|
||||||
Uplink muting is purely a client-side decision based on state. Backend doesn't enforce it.
|
|
||||||
|
|
||||||
## Conversation lifecycle
|
|
||||||
|
|
||||||
```
|
|
||||||
PI BACKEND OPENAI REALTIME
|
|
||||||
─────────────────────────────────────────────────────────────────────────────────
|
|
||||||
state=IDLE
|
|
||||||
mic frames → wakeword.predict
|
|
||||||
wakeword fires
|
|
||||||
play wake.wav (non-blocking)
|
|
||||||
send {type:"wake"} ──►
|
|
||||||
open WSS to OpenAI ──►
|
|
||||||
send session.update ──►
|
|
||||||
(voice, instructions,
|
|
||||||
input_audio_transcription,
|
|
||||||
tools=[end_session,
|
|
||||||
get_current_time,
|
|
||||||
set_volume])
|
|
||||||
◄── session.updated
|
|
||||||
insert Conversations row
|
|
||||||
send {type:"session_started",
|
|
||||||
conversation_id} ──►
|
|
||||||
state=LISTENING
|
|
||||||
mic frames (binary) ──────────────► forward as input_audio_buffer.append ─►
|
|
||||||
◄── input_audio_buffer.speech_started
|
|
||||||
◄── input_audio_buffer.speech_stopped
|
|
||||||
◄── response.created
|
|
||||||
◄── response.audio.delta (x N)
|
|
||||||
binary frames ◄─────────────── forward to PI as binary
|
|
||||||
binary frames ──► playback queue
|
|
||||||
state=ASSISTANT_SPEAKING
|
|
||||||
(uplink muted)
|
|
||||||
◄── response.audio_transcript.done
|
|
||||||
(persist assistant Turn)
|
|
||||||
◄── response.done
|
|
||||||
send {type:"assistant_done"} ──►
|
|
||||||
state=LISTENING
|
|
||||||
◄── conversation.item.input_audio_transcription.completed
|
|
||||||
(persist user Turn)
|
|
||||||
...follow-up turn(s)...
|
|
||||||
|
|
||||||
Path A — model calls end_session
|
|
||||||
◄── response.function_call_arguments.done
|
|
||||||
(name="end_session")
|
|
||||||
◄── response.audio.delta (the "bye" reply)
|
|
||||||
forward audio to PI ──────────►
|
|
||||||
wait for response.done
|
|
||||||
◄── response.done
|
|
||||||
send {type:"assistant_done"} ──►
|
|
||||||
send {type:"session_ended",
|
|
||||||
reason="tool"} ──►
|
|
||||||
close upstream WSS
|
|
||||||
persist Conversations.EndReason
|
|
||||||
state=IDLE-PENDING
|
|
||||||
wait for current audio buffer drain
|
|
||||||
play sleep.wav
|
|
||||||
send {type:"playback_done",
|
|
||||||
for:"sleep"} ──►
|
|
||||||
state=IDLE
|
|
||||||
(wakeword re-enabled)
|
|
||||||
|
|
||||||
Path B — idle timeout
|
|
||||||
no mic activity for IdleTimeoutSeconds while LISTENING
|
|
||||||
send {type:"session_ended",
|
|
||||||
reason="idle"} ──►
|
|
||||||
close upstream WSS
|
|
||||||
(same sleep.wav + transition as Path A)
|
|
||||||
```
|
|
||||||
|
|
||||||
### `end_session` sequencing
|
|
||||||
|
|
||||||
The tool call arrives *during* the assistant's response. The relay must:
|
|
||||||
|
|
||||||
1. Forward all audio deltas until `response.done`.
|
|
||||||
2. Then send `assistant_done`, then `session_ended(reason=tool)`.
|
|
||||||
3. NOT execute the tool mid-response — `end_session` has no real side effect; the relay records "the model asked to end" and closes after the audio finishes. The tool result returned to OpenAI is `{"ok": true}` (for cleanliness; the upstream is about to close).
|
|
||||||
|
|
||||||
## Tools registry
|
|
||||||
|
|
||||||
C# interface, registered as singletons at startup:
|
|
||||||
|
|
||||||
```csharp
|
|
||||||
public interface ITool
|
|
||||||
{
|
|
||||||
string Name { get; } // OpenAI tool name
|
|
||||||
string Description { get; } // shown to the model
|
|
||||||
JsonSchema ParameterSchema { get; } // OpenAI-compatible
|
|
||||||
Task<ToolResult> ExecuteAsync(
|
|
||||||
ToolInvocation invocation,
|
|
||||||
DeviceContext device,
|
|
||||||
CancellationToken ct);
|
|
||||||
}
|
|
||||||
|
|
||||||
public record ToolInvocation(string CallId, JsonElement Arguments);
|
|
||||||
public record ToolResult(JsonElement Output); // Output is JSON sent back to model
|
|
||||||
public record DeviceContext(Guid DeviceId, Guid ConversationId, IDeviceChannel Channel);
|
|
||||||
```
|
|
||||||
|
|
||||||
`IDeviceChannel` lets a tool send `tool_call`-style messages to the Pi and await `tool_result`. Tools that don't need the Pi simply don't use it.
|
|
||||||
|
|
||||||
### v1 built-in tools
|
|
||||||
|
|
||||||
1. **`end_session`** — no parameters. Sets a "model wants to end" flag on the active `RealtimeSession`; the relay's `response.done` handler closes the upstream WS and emits `session_ended(reason=tool)`. Returns `{"ok": true}`.
|
|
||||||
|
|
||||||
2. **`get_current_time`** — no parameters. Returns `{"now": "<ISO-8601 UTC>"}` from `DateTimeOffset.UtcNow`. Smoke test for the tool plane.
|
|
||||||
|
|
||||||
3. **`set_volume`** — `level: integer 0..100`. Backend sends `tool_call(name=set_volume, arguments={level})` to the Pi; Pi runs `amixer set 'PCM' N%` (or device-specific mixer) and replies with `tool_result(call_id, ok, error?)`. Backend forwards back to OpenAI as the function call output.
|
|
||||||
|
|
||||||
The per-device enabled set is enforced by filtering the tool list in `session.update`. Disabled tools are simply not advertised to the model.
|
|
||||||
|
|
||||||
## Data model (SQLite via EF Core)
|
|
||||||
|
|
||||||
```
|
|
||||||
AspNetUsers, AspNetRoles, ... # Identity standard
|
|
||||||
Devices
|
|
||||||
Id (Guid) OwnerUserId Name PairedAt LastSeenAt TokenHash IsRevoked
|
|
||||||
DeviceConfigs # 1:1 with Devices
|
|
||||||
DeviceId SystemPrompt Voice Model IdleTimeoutSeconds EnabledToolsJson
|
|
||||||
PairingCodes
|
|
||||||
Code (8 chars) OwnerUserId ExpiresAt ConsumedAt
|
|
||||||
Conversations
|
|
||||||
Id DeviceId StartedAt EndedAt EndReason ("tool" | "idle" | "error")
|
|
||||||
Turns
|
|
||||||
Id ConversationId Index Role ("user" | "assistant" | "tool") Text ToolName ToolArgsJson ToolResultJson CreatedAt
|
|
||||||
SystemSettings # singleton row
|
|
||||||
DefaultVoice DefaultModel DefaultSystemPrompt
|
|
||||||
```
|
|
||||||
|
|
||||||
- `TokenHash` = SHA-256 of the device token; plaintext is shown once during pairing and never persisted.
|
|
||||||
- `EnabledToolsJson` is a small JSON array of tool names; no relational table needed.
|
|
||||||
- `Turns.Index` is monotonic per-conversation for ordering.
|
|
||||||
- `Database.Migrate()` runs on startup; SQLite file at `/data/assistant.db` on a Coolify named volume.
|
|
||||||
|
|
||||||
## Pairing flow
|
|
||||||
|
|
||||||
UI side:
|
|
||||||
|
|
||||||
1. User signs in, clicks **Add device**, enters a friendly name.
|
|
||||||
2. Backend generates an 8-char alphanumeric code from a 32-char alphabet (no `0/O/1/l/I`); stores `PairingCodes(Code, OwnerUserId, ExpiresAt = NOW + 10 min)`.
|
|
||||||
3. UI shows the code and the one-line installer:
|
|
||||||
```
|
|
||||||
curl -fsSL https://<backend>/install.sh | bash
|
|
||||||
```
|
|
||||||
4. Code stays visible on the device page until consumed or expired; regenerate at any time.
|
|
||||||
|
|
||||||
Pi side, first run:
|
|
||||||
|
|
||||||
1. Installer runs.
|
|
||||||
2. If `~/assistant/state/config.json` exists with a valid `device_token`, **skip pairing** and proceed to update path.
|
|
||||||
3. Otherwise prompt `Pairing code: `.
|
|
||||||
4. `POST https://<backend>/api/pair` with `{ "code", "hostname", "client_version" }`.
|
|
||||||
5. Backend validates code (exists, unexpired, unconsumed), creates `Devices(OwnerUserId = code.OwnerUserId, Name = <user-set-name>, TokenHash = sha256(newToken))`, marks code `ConsumedAt = NOW`, creates default `DeviceConfigs` row from `SystemSettings`, returns `{ "device_id", "device_token", "backend_ws" }`.
|
|
||||||
6. Installer writes `~/assistant/state/config.json` (mode 0600).
|
|
||||||
|
|
||||||
Security:
|
|
||||||
- Pairing codes are one-shot, 10-min TTL, 8 chars from a 32-char alphabet (~40 bits — fine for the TTL window).
|
|
||||||
- Device tokens are 32 bytes random, base64url-encoded, stored only as SHA-256 hash server-side.
|
|
||||||
|
|
||||||
## Install script
|
|
||||||
|
|
||||||
`install.sh` is served by the backend at `GET /install.sh` and is fully idempotent.
|
|
||||||
|
|
||||||
```
|
|
||||||
1. Resolve BACKEND_URL from the script's own served-from origin (no hardcoding).
|
|
||||||
2. apt-get install -y python3-venv libportaudio2 alsa-utils
|
|
||||||
(sudo prompt OK; passwordless-sudo per findings.md §E is the recommended setup).
|
|
||||||
3. mkdir -p ~/assistant/{code,state}
|
|
||||||
4. python3 -m venv ~/assistant/.venv (if missing)
|
|
||||||
5. curl -fsSL $BACKEND_URL/client.tar.gz | tar -xz -C ~/assistant/code
|
|
||||||
6. ~/assistant/.venv/bin/pip install -r ~/assistant/code/requirements.txt
|
|
||||||
7. ~/assistant/.venv/bin/pip install --no-deps -r ~/assistant/code/requirements-nodeps.txt
|
|
||||||
(openwakeword install split per findings.md §3).
|
|
||||||
8. If config.json missing → run `~/assistant/.venv/bin/python -m client.pair`,
|
|
||||||
which prompts for the code and writes config.json.
|
|
||||||
9. Install systemd user unit at ~/.config/systemd/user/assistant.service.
|
|
||||||
10. systemctl --user daemon-reload
|
|
||||||
11. systemctl --user enable --now assistant
|
|
||||||
12. loginctl enable-linger pi (so the unit runs without an active login).
|
|
||||||
13. Print status; if pairing happened, print "Paired to <backend> as device <name>".
|
|
||||||
```
|
|
||||||
|
|
||||||
The systemd user unit runs `~/assistant/.venv/bin/python -m client.main` with:
|
|
||||||
- `Restart=on-failure`
|
|
||||||
- `RestartSec=5`
|
|
||||||
- `Environment=PA_ALSA_PLUGHW=1`
|
|
||||||
|
|
||||||
### Updates
|
|
||||||
|
|
||||||
Re-running the same one-liner upgrades the client. Steps 5–7 fetch and reinstall the latest bundle; step 11 restarts the unit. Pairing is skipped because `config.json` already exists.
|
|
||||||
|
|
||||||
`curl ... | bash -s -- --reset` deletes `config.json` and forces a fresh pairing.
|
|
||||||
|
|
||||||
### Backend install endpoints
|
|
||||||
|
|
||||||
- `GET /install.sh` — returns the shell script with `BACKEND_URL` baked from the request's `X-Forwarded-Host` / `Host`. Public, no auth.
|
|
||||||
- `GET /client.tar.gz` — returns the current client bundle (built at deploy time into `wwwroot/`). Public, no auth — the code is not a secret; the device token gates the device WS.
|
|
||||||
- `POST /api/pair` — accepts pairing code, returns device token. No prior auth (code is the credential).
|
|
||||||
|
|
||||||
The bundle is built during the Docker image build: a stage runs `tar czf client.tar.gz -C . client/ requirements.txt requirements-nodeps.txt` and copies the result into `wwwroot/`.
|
|
||||||
|
|
||||||
## Error handling
|
|
||||||
|
|
||||||
| Scenario | Detection | Backend action | Client action |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| Pi loses network | WS close / heartbeat miss | Close upstream Realtime WS if active, log conversation `EndReason="error"`, mark device offline | Exponential backoff reconnect 1→30 s, stay IDLE, no wakeword |
|
|
||||||
| Backend restarts | Pi WS closes from server side | (n/a — process gone) | Same backoff reconnect; on reconnect send fresh `hello` |
|
|
||||||
| OpenAI WS drops mid-conversation | upstream `on_close` | Send `error(code=upstream)` + `session_ended(reason=error)`, persist partial transcript | Play `sleep.wav`, return to IDLE |
|
|
||||||
| OpenAI rate limit / 429 on connect | upstream open fails | Don't open session; send `error(code=upstream, message=...)` instead of `session_started` | Play `sleep.wav`, IDLE |
|
|
||||||
| Tool execution exception (server-side) | C# try/catch around `ITool.ExecuteAsync` | Send tool result `{"error": "..."}` back to OpenAI so the model can recover; log | (no client involvement) |
|
|
||||||
| Pi-side tool fails (e.g. `set_volume` amixer error) | `tool_result.ok=false` | Forward `{"error": "..."}` to OpenAI | Log and continue |
|
|
||||||
| Device token revoked while connected | Admin clicks revoke | Set `IsRevoked=true`, close all WS for that device with `error(code=auth_revoked, fatal=true)` | Stop the process (systemd will restart, fail to auth, sleep 60 s, retry — visible in logs) |
|
|
||||||
| SQLite write failure | EF exception | Bubble up to REST as 500; on the hub side, drop the offending log event, keep session running | (n/a) |
|
|
||||||
| Two simultaneous wakeword fires | `wake` while not IDLE | Backend ignores second `wake` (idempotent within active session) | Client also gates locally |
|
|
||||||
| Audio buffer overflow on Pi | PortAudio callback queue full | (n/a) | Drop oldest frame, log a warning |
|
|
||||||
|
|
||||||
## Management UI (v1)
|
|
||||||
|
|
||||||
React SPA, served from the backend's `wwwroot/`. Pages:
|
|
||||||
|
|
||||||
- **Sign in / sign up** (ASP.NET Identity).
|
|
||||||
- **Devices** — list of the signed-in user's devices: name, online indicator, last-seen.
|
|
||||||
- **Device detail** — rename; generate pairing code (with countdown); revoke; per-device config form: system prompt override, voice, model, idle timeout slider, enabled-tools checkboxes; conversation history viewer (paginated, text only).
|
|
||||||
- **Admin** (admin role only) — user list, invite/disable users, global model/voice/prompt defaults.
|
|
||||||
|
|
||||||
Out of v1: live conversation tail, audio meters, audio recordings, memory UI, per-device `wake.wav`/`sleep.wav` upload (filesystem only for v1; v1.1 adds upload).
|
|
||||||
|
|
||||||
## Testing strategy
|
|
||||||
|
|
||||||
- **Backend unit tests** (xUnit): `ToolRegistry` filtering by enabled set; `PairingService` (code generation, expiry, single-use); `ConversationLog` (turn ordering, persistence); auth/role enforcement on the admin endpoints.
|
|
||||||
- **Backend integration tests** (`WebApplicationFactory`): full pair flow end-to-end against in-memory SQLite; device-hub WS with a fake OpenAI upstream (test double that emits scripted Realtime events).
|
|
||||||
- **Client unit tests** (pytest): state machine transitions (pure-Python, no audio); resample correctness against golden samples; config loading. Audio I/O and wakeword are not unit-tested — already covered by the prototypes and by manual end-to-end.
|
|
||||||
- **End-to-end manual smoke test** (in repo `README.md`): pair a fresh Pi, say wakeword, ask "what time is it", say "bye", verify conversation row + sleep.wav heard. Run after any change that touches the conversation flow.
|
|
||||||
- No load / perf testing.
|
|
||||||
|
|
||||||
## Deployment (Coolify)
|
|
||||||
|
|
||||||
Single Docker image, multi-stage Dockerfile (`preparing-dotnet-react-app-for-coolify` skill is the reference for the layout):
|
|
||||||
|
|
||||||
1. **Frontend build** stage — Node, `npm ci && npm run build` against `frontend/`.
|
|
||||||
2. **Backend build** stage — .NET SDK, `dotnet publish backend/ -c Release -o /publish`.
|
|
||||||
3. **Client bundle** stage — `tar czf /publish/wwwroot/client.tar.gz -C . client/ requirements.txt requirements-nodeps.txt`.
|
|
||||||
4. **Final** stage — `mcr.microsoft.com/dotnet/aspnet:9.0`, copies `/publish`, copies SPA `dist/` into `wwwroot/`, sets `ASPNETCORE_URLS=http://+:8080`, exposes 8080.
|
|
||||||
|
|
||||||
Coolify app (provisioned via `deploying-to-coolify-via-api`):
|
|
||||||
|
|
||||||
- Source: this repo on Gitea.
|
|
||||||
- Expose 8080 (Traefik fronts it).
|
|
||||||
- `SERVICE_FQDN_BACKEND` → backend domain.
|
|
||||||
- Env vars:
|
|
||||||
- `OPENAI_API_KEY` — secret.
|
|
||||||
- `ConnectionStrings__Default=Data Source=/data/assistant.db`
|
|
||||||
- `ASPNETCORE_URLS=http://+:8080`
|
|
||||||
- Named volumes:
|
|
||||||
- `/data` — SQLite file.
|
|
||||||
- `/keys` — ASP.NET data-protection keys (cookies survive container restarts).
|
|
||||||
- Deploys triggered fire-and-forget via the Coolify v4 API (per global CLAUDE.md rule).
|
|
||||||
|
|
||||||
## Repo layout (final)
|
|
||||||
|
|
||||||
```
|
|
||||||
Assistant/
|
|
||||||
├── CLAUDE.md
|
|
||||||
├── README.md # smoke-test instructions
|
|
||||||
├── findings.md # prototype-phase findings (kept)
|
|
||||||
├── tests/ # prototype throwaways (kept for reference)
|
|
||||||
│ ├── 01-record-play/
|
|
||||||
│ └── 02-wakeword/
|
|
||||||
├── docs/superpowers/
|
|
||||||
│ ├── specs/
|
|
||||||
│ │ ├── 2026-06-11-voice-assistant-prototypes-design.md
|
|
||||||
│ │ └── 2026-06-11-smart-assistant-design.md # this design
|
|
||||||
│ └── plans/
|
|
||||||
│ └── (one or more implementation plans, written next)
|
|
||||||
├── backend/ # ASP.NET solution
|
|
||||||
│ ├── backend.csproj
|
|
||||||
│ ├── Program.cs
|
|
||||||
│ ├── Auth/ Devices/ DeviceHub/ Realtime/ Tools/ Conversations/ Install/ Data/
|
|
||||||
│ └── wwwroot/ # filled at build: SPA dist + client.tar.gz
|
|
||||||
├── frontend/ # React + Vite source
|
|
||||||
│ ├── package.json
|
|
||||||
│ └── src/
|
|
||||||
├── client/ # Python package
|
|
||||||
│ ├── requirements.txt # sounddevice, numpy, scipy, onnxruntime, websockets, ...
|
|
||||||
│ ├── requirements-nodeps.txt # openwakeword
|
|
||||||
│ ├── main.py audio.py wakeword.py session.py playback.py state.py config.py pair.py log.py
|
|
||||||
│ └── systemd/assistant.service
|
|
||||||
├── Dockerfile
|
|
||||||
└── deploy.json # Coolify config (project/server uuids, domain)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Open items deferred to implementation plans
|
|
||||||
|
|
||||||
- Exact OpenAI Realtime model + voice default (`SystemSettings.DefaultModel`/`DefaultVoice`) — pick at implementation time from currently-available choices.
|
|
||||||
- Whether wake.wav / sleep.wav playback uses the persistent OutputStream (preferred — same path as session audio) or a separate one-shot — settled in the client plan.
|
|
||||||
- ASP.NET data-protection key seeding (initial bootstrap on first run).
|
|
||||||
- React UI component library — Tailwind only, or Tailwind + a small headless component lib (Radix / shadcn).
|
|
||||||
- Exact mixer control name for `set_volume` on the LISTENAI device — confirm with `amixer scontrols` on the Pi during implementation.
|
|
||||||
|
|
||||||
## Reference paths
|
|
||||||
|
|
||||||
- Prototype spec: `docs/superpowers/specs/2026-06-11-voice-assistant-prototypes-design.md`
|
|
||||||
- Prototype findings: `findings.md`
|
|
||||||
- Pi access: `pi@192.168.50.115`, password `assistant` (see `CLAUDE.md`)
|
|
||||||
- Coolify skills: `deploying-to-coolify-via-api`, `preparing-dotnet-react-app-for-coolify`
|
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
# Record-play C# probe — design
|
||||||
|
|
||||||
|
Date: 2026-06-12
|
||||||
|
Branch: `fresh-start`
|
||||||
|
Status: approved (pending written-spec review)
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Prove that .NET 9 on the Pi can drive PortAudio with the same behaviour we already validated from Python's `sounddevice` in `tests/01-record-play/main.py`. This is a throwaway probe. Once it passes the same criteria the Python test passed, its job is done — it does not become the foundation of the main C# Pi client. Any abstractions for the real client will be designed later, after both audio probes (and the wakeword probe) are green.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- No reusable audio abstraction. One-file program; if it grows, that's a smell, not a feature.
|
||||||
|
- No automated tests. Verification is the same listening test the Python probe uses.
|
||||||
|
- No changes to `bin/bootstrap`. The Pi gets a self-contained binary; .NET is never installed on the Pi.
|
||||||
|
- No coupling to the Python prototypes. They keep working, untouched.
|
||||||
|
- No build of the main C# Pi client. That's a separate spec.
|
||||||
|
|
||||||
|
## Functional spec
|
||||||
|
|
||||||
|
Identical to the Python Test 1 (`tests/01-record-play/main.py:4-7`):
|
||||||
|
|
||||||
|
- Record 5 s of mono PCM16 audio at 16 kHz from the USB Speaker Phone on the Pi.
|
||||||
|
- Render a live RMS level meter to the terminal (`\r{elapsed}s |####...|`, refresh ~20 Hz).
|
||||||
|
- Write the recording to `out.wav` in the working directory.
|
||||||
|
- Play the recording back through the same USB device.
|
||||||
|
|
||||||
|
### Pass criteria
|
||||||
|
|
||||||
|
Same as the original spec carried in `findings.md`:
|
||||||
|
|
||||||
|
1. Playback is recognisable as what was said into the mic.
|
||||||
|
2. The meter visibly responds to speech and to silence.
|
||||||
|
3. No clipping, dropouts, or sustained PortAudio under/overrun warnings.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
tests/01-record-play-cs/
|
||||||
|
Probe.csproj # net9.0, RuntimeIdentifier=linux-arm64, PortAudioSharp2 NuGet ref
|
||||||
|
Program.cs # entry point: env var → init → device pick → record → write → play
|
||||||
|
WavWriter.cs # static helper, PCM16 mono RIFF/fmt/data header + raw bytes
|
||||||
|
|
||||||
|
bin/probe-cs # build (locally) → rsync (to Pi) → run (over SSH)
|
||||||
|
```
|
||||||
|
|
||||||
|
The Python prototypes under `tests/01-record-play/` and `tests/02-wakeword/` are untouched.
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
### `Program.cs`
|
||||||
|
|
||||||
|
Single entry point with the full record-play loop. Mirrors the Python structure for diffability.
|
||||||
|
|
||||||
|
- Set `PA_ALSA_PLUGHW=1` via `Environment.SetEnvironmentVariable` **before** `PortAudio.Initialize()` — same ordering rule documented in `findings.md` (the env var must be in place before the PortAudio host-API table is built).
|
||||||
|
- Initialise PortAudio.
|
||||||
|
- Enumerate devices; pick the first one whose name (case-insensitively) contains `usb` and whose `maxInputChannels >= 1`. If none found, print the device table to stderr and exit 1. Mirrors `find_usb_device` in the Python probe.
|
||||||
|
- Open a callback-driven input stream: 16 kHz, 1 channel, Int16 format, 1024-frame block size. The callback copies into a preallocated `Int16[5 * 16000]` at a running write offset. The PortAudio status flag is checked on every callback; on overflow, write `[status] input overflow` to stderr and continue (matches Python `[status] {status}` behaviour).
|
||||||
|
- Main thread render loop: sleep 50 ms, snapshot the last 1600 samples written, compute RMS as in Python (`sqrt(mean((x/32768)^2) + 1e-12)`), print `\r{elapsed:0.1}s |####...|` where `####...` has 60 chars and the bar length is `int(rms * 60)`.
|
||||||
|
- After the buffer is full, close the input stream. Call `WavWriter.Write("out.wav", buf, 16000)`. Open an output stream on the same device (16 kHz, 1 channel, Int16, 1024-frame block) that pulls from the same buffer in a callback, with a completion signal. Wait for completion, close, terminate PortAudio.
|
||||||
|
|
||||||
|
### `WavWriter.cs`
|
||||||
|
|
||||||
|
```
|
||||||
|
static void Write(string path, ReadOnlySpan<short> samples, int sampleRate)
|
||||||
|
```
|
||||||
|
|
||||||
|
Writes a standard 44-byte RIFF header (RIFF/fmt /data chunks, PCM format = 1, channels = 1, bits = 16, byte rate / block align computed from `sampleRate`) followed by `samples` as little-endian Int16. ~30 lines. No streaming, no chunking — the full buffer fits in memory (160 KB at the spec rate × duration).
|
||||||
|
|
||||||
|
## Data flow
|
||||||
|
|
||||||
|
```
|
||||||
|
PortAudio input callback (RT thread)
|
||||||
|
└─ memcpy into Int16[80000] at writeOffset; advance writeOffset
|
||||||
|
(no allocation in callback)
|
||||||
|
|
||||||
|
Main thread (every 50 ms)
|
||||||
|
├─ snapshot last 1600 samples of buf up to writeOffset
|
||||||
|
├─ RMS → 60-char bar → \r-update on stdout
|
||||||
|
└─ exit when writeOffset == buf.Length
|
||||||
|
|
||||||
|
Then (still main thread):
|
||||||
|
├─ WavWriter.Write("out.wav", buf, 16000)
|
||||||
|
├─ open OUTPUT stream on same device
|
||||||
|
├─ output callback drains buf into PortAudio
|
||||||
|
└─ wait for completion, terminate
|
||||||
|
```
|
||||||
|
|
||||||
|
No queue, no extra threads. The callback writes; the main thread reads a tail snapshot for the meter only. Mirrors what `sounddevice` does in the Python version.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
- USB device not found → device table to stderr, exit 1.
|
||||||
|
- `PortAudio.Initialize()`, `PortAudio.OpenStream()`, `PortAudio.StartStream()` → on `PaError != NoError`, write the error name to stderr and exit 1.
|
||||||
|
- Input overflow status → `[status] input overflow` to stderr, keep recording. Sustained overflow violates pass criterion 3 and gets noticed by the human listener.
|
||||||
|
- File write failure → uncaught `IOException` (this is a probe; crash trace is fine).
|
||||||
|
|
||||||
|
Anything else (PortAudio buffer underrun on playback, codec errors, device disappears mid-stream) → uncaught throw. Again — probe, not production.
|
||||||
|
|
||||||
|
## Deploy & run (`bin/probe-cs`)
|
||||||
|
|
||||||
|
A new shell script alongside `bin/deploy`. It does:
|
||||||
|
|
||||||
|
1. `dotnet publish tests/01-record-play-cs -c Release -r linux-arm64 --self-contained -o /tmp/probe-cs-out` on this sandbox.
|
||||||
|
2. `rsync -az --delete /tmp/probe-cs-out/ pi@192.168.50.115:~/probe-cs/` (sshpass with the Pi password, as in `bin/deploy`).
|
||||||
|
3. `ssh pi@192.168.50.115 'cd ~/probe-cs && ./Probe'` — the meter and the playback are interactive on the Pi's audio output; we observe via SSH.
|
||||||
|
|
||||||
|
Cleanup (when the probe is retired): `ssh pi@... 'rm -rf ~/probe-cs'` plus `git rm -r tests/01-record-play-cs bin/probe-cs`.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
The pass criteria above. There are no automated tests. Run `bin/probe-cs`, listen to the playback, watch the meter, confirm no warnings.
|
||||||
|
|
||||||
|
## Open assumption (verify at implementation time)
|
||||||
|
|
||||||
|
PortAudioSharp2's NuGet package on linux-arm64 either bundles its own `libportaudio.so` or `dlopen`s the system one. The Pi already has `libportaudio2` from `bin/bootstrap`, so either way the binary should resolve PortAudio. Confirm via the PortAudioSharp2 README / Context7 lookup at implementation, not now — does not affect the design.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Python probe: `tests/01-record-play/main.py`
|
||||||
|
- Prototype findings: `findings.md` (esp. "Force PortAudio through ALSA's `plug` plugin", "USB device discovery", "Beep / TTS blocks the input stream")
|
||||||
|
- Pi access: `pi@192.168.50.115`, password `assistant` (see `CLAUDE.md`)
|
||||||
@@ -0,0 +1,263 @@
|
|||||||
|
# Test 2 wakeword C# probe — design
|
||||||
|
|
||||||
|
Date: 2026-06-12
|
||||||
|
Branch: `fresh-start`
|
||||||
|
Status: approved (pending written-spec review)
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Prove that openwakeword's stock "alexa" detector can be driven from .NET on the Pi, by porting `openwakeword/model.py`'s `predict()` pipeline (three ONNX models chained: mel-spectrogram → speech embedding → keyword classifier) to C# via `Microsoft.ML.OnnxRuntime`. End-to-end behaviour replicates the Python probe at `tests/02-wakeword/main.py`.
|
||||||
|
|
||||||
|
This is a throwaway probe. Once it passes the criteria the Python probe passed, its job is done — it does not become the foundation of the main C# Pi client. The `WakewordModel.cs` class it produces is the natural lift-and-shift candidate for the eventual client, but that's a future decision, not a constraint on the probe.
|
||||||
|
|
||||||
|
Test 1's record-play C# port (`tests/01-record-play-cs/`) is the immediate ancestor. Its patterns — `Libc.setenv` for env knobs the C runtime reads, callback-driven `PortAudioSharp.Stream`, `<AllowUnsafeBlocks>`, `using Stream = PortAudioSharp.Stream;` — carry over verbatim. See `findings.md` § "C# probe outcome (2026-06-12 — Test 1 ported to C#)".
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- No numerical-parity check against the Python implementation. We trust the port if hardware passes.
|
||||||
|
- No custom wakeword training. Stock "alexa" model only. Custom wakeword is a separate task (see `findings.md` § C).
|
||||||
|
- No fix for "speaker is the mic" self-triggering. Main-assistant concern (`findings.md` § B), out of probe scope.
|
||||||
|
- No reusable audio abstraction. One-folder probe; if it grows beyond ~400 lines that's a smell.
|
||||||
|
- No automated tests. Verification is the same listening test the Python probe uses.
|
||||||
|
- No changes to `bin/bootstrap`. The Pi gets a self-contained binary; .NET is never installed on the Pi.
|
||||||
|
- No coupling to the Python prototypes. They keep working, untouched.
|
||||||
|
|
||||||
|
## Functional spec
|
||||||
|
|
||||||
|
Identical to the Python Test 2 (`tests/02-wakeword/main.py:4-7`):
|
||||||
|
|
||||||
|
- Open a 16 kHz mono PCM16 input stream on the USB Speaker Phone.
|
||||||
|
- Run each 80 ms (1280-sample) frame through the openwakeword "alexa" pipeline.
|
||||||
|
- When the classifier score crosses `THRESHOLD = 0.5` and at least `COOLDOWN_S = 1.0` seconds have passed since the last trigger, print `DETECTED alexa score={s:0.000} t={elapsed:0.1}s` and play a 200 ms 880 Hz beep through the same USB device.
|
||||||
|
- Loop until Ctrl-C.
|
||||||
|
|
||||||
|
### Pass criteria
|
||||||
|
|
||||||
|
Same as the original Python spec:
|
||||||
|
|
||||||
|
1. "alexa" at normal volume from ~1 m triggers within ~1 s in ≥ 8/10 attempts.
|
||||||
|
2. ≤ 1 false positive per minute of unrelated speech (~3 minutes of reading aloud).
|
||||||
|
3. Process stays under 50% of one core, observed via `htop` on the Pi.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
tests/02-wakeword-cs/
|
||||||
|
Probe.csproj # net9.0, RuntimeIdentifier=linux-arm64, PortAudioSharp2 + Microsoft.ML.OnnxRuntime
|
||||||
|
Program.cs # entry point: env vars → init → device pick → input stream → main loop → beep dispatch
|
||||||
|
WakewordModel.cs # IDisposable; loads three ONNX sessions; Predict(short[] frame1280) → float score
|
||||||
|
models/
|
||||||
|
melspectrogram.onnx # ~14 KB, openwakeword resource
|
||||||
|
embedding_model.onnx # ~10 MB, Google speech embedding model (openwakeword resource)
|
||||||
|
alexa.onnx # ~1.5 MB, stock alexa classifier (openwakeword resource)
|
||||||
|
|
||||||
|
bin/probe-cs-2 # build (locally) → scp (to Pi) → run (over SSH); copy of bin/probe-cs with new paths
|
||||||
|
```
|
||||||
|
|
||||||
|
The Python prototypes under `tests/01-record-play/` and `tests/02-wakeword/` and the Test 1 C# probe under `tests/01-record-play-cs/` are untouched.
|
||||||
|
|
||||||
|
### Sourcing the ONNX files for the repo
|
||||||
|
|
||||||
|
The three `.onnx` files are sourced from the Pi's existing openwakeword install at implementation time:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p assistant scp \
|
||||||
|
pi@192.168.50.115:.local/lib/python3.13/site-packages/openwakeword/resources/models/{melspectrogram,embedding_model,alexa}.onnx \
|
||||||
|
tests/02-wakeword-cs/models/
|
||||||
|
```
|
||||||
|
|
||||||
|
Their SHA-256 hashes are recorded in the implementation plan for reproducibility. The same files openwakeword's Python `download_models()` helper fetches at first run — we vendor them rather than introducing a startup network dependency. ~11 MB total in git is acceptable for a probe.
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
### `WakewordModel.cs` — the substantive port
|
||||||
|
|
||||||
|
Public surface:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
class WakewordModel : IDisposable {
|
||||||
|
WakewordModel(string melPath, string embeddingPath, string classifierPath);
|
||||||
|
float Predict(short[] frame1280); // returns score in [0, 1]
|
||||||
|
void Dispose();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Constructor loads three `Microsoft.ML.OnnxRuntime.InferenceSession` instances (one per ONNX file). Each session is configured with `SessionOptions { IntraOpNumThreads = 1, InterOpNumThreads = 1, LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR }` to keep CPU bounded and the console quiet.
|
||||||
|
|
||||||
|
`Predict` is called once per 80 ms input frame (1280 PCM16 samples at 16 kHz) and returns the most recent classifier score.
|
||||||
|
|
||||||
|
**Pipeline (matches `openwakeword/model.py`'s `Model.predict()`):**
|
||||||
|
|
||||||
|
1. **Audio buffer.** Append the incoming 1280 samples to a raw-audio ring of recent samples.
|
||||||
|
2. **Mel stage.** Run `melspectrogram.onnx` over the latest window of audio. Append output mel frames to a mel-frame ring.
|
||||||
|
3. **Embedding stage.** When the mel ring has accumulated enough fresh mel frames to fill the embedding model's input window (76 frames per openwakeword's source — but verify), slice the window, run `embedding_model.onnx`. Append the resulting embedding to an embedding ring.
|
||||||
|
4. **Classifier stage.** Slice the most recent N embeddings (N = whatever shape `alexa.onnx` declares on its input). Run the classifier. Return the score.
|
||||||
|
|
||||||
|
Per 80 ms input, the classifier runs exactly once. Between calls, the three ring buffers hold all cross-call state.
|
||||||
|
|
||||||
|
**Ground truth — do not guess tensor shapes.** The exact buffer sizes (raw-audio window length, mel-frame count for embedder input, embedding count for classifier input) MUST be derived during implementation from two sources:
|
||||||
|
|
||||||
|
- The Python reference: `openwakeword/model.py` on the Pi at `~/.local/lib/python3.13/site-packages/openwakeword/model.py`. Read it side-by-side with the C# code. Numbers quoted in this spec are from memory and authoritative only after verification.
|
||||||
|
- The ONNX files themselves: `InferenceSession.InputMetadata` exposes the declared input shapes. The constructor MUST print the declared input + output shapes of all three models at startup the first time the probe runs, and SHOULD assert against them to fail fast on geometry mismatch.
|
||||||
|
|
||||||
|
**Threading.** `Predict` is called only from the main thread (per the data-flow section below). No internal locking. `Dispose` cleans up all three sessions.
|
||||||
|
|
||||||
|
**Why three sessions instead of one combined graph.** openwakeword ships them as three separate files because the embedding model is Google's pretrained TF Hub model. Combining them offline is possible but a deliberate optimisation, not needed for the probe.
|
||||||
|
|
||||||
|
### `Program.cs` — entry point and audio I/O
|
||||||
|
|
||||||
|
Single top-level-statements file. Reuses every Test 1 pattern:
|
||||||
|
|
||||||
|
- Set `PA_ALSA_PLUGHW=1` via `Libc.setenv("PA_ALSA_PLUGHW", "1", 1)` + `Environment.SetEnvironmentVariable` (the dual-call workaround documented in `findings.md`).
|
||||||
|
- Set `ORT_LOGGING_LEVEL=3` (error-only) via `Libc.setenv` to suppress onnxruntime's `/sys/class/drm/card0` GPU-detection warnings on the Pi.
|
||||||
|
- `PortAudio.Initialize()`.
|
||||||
|
- `FindUsbDevice()` — same body as Test 1; first device whose name contains `usb` (case-insensitive) and has `maxInputChannels >= 1`. On failure: print device table to stderr, exit 1.
|
||||||
|
- `var model = new WakewordModel(...)` — paths resolved relative to `AppContext.BaseDirectory + "models/"`. Print load time (mirrors Python's `"Model loaded in {t}s"`).
|
||||||
|
- Pre-compute the 200 ms 880 Hz sine-wave beep as a `short[3200]` once at startup. Reused for every detection.
|
||||||
|
- Open input stream: callback-driven, 16 kHz, 1 channel, Int16, 1280-frame block size. Callback copies into a fresh `short[1280]` and pushes into a `BlockingCollection<short[]>(boundedCapacity: 16)`. On `StreamCallbackFlags.InputOverflow`: log `[status] input overflow` to stderr. On `TryAdd` failure (queue full): log `[status] consumer behind` and drop the frame.
|
||||||
|
- `stream.Start()`. Print `Listening on device {i} ({name}). Say 'alexa'. Ctrl-C to exit.`
|
||||||
|
- Main loop:
|
||||||
|
|
||||||
|
```
|
||||||
|
while (!cts.IsCancellationRequested) {
|
||||||
|
var frame = queue.Take(cts.Token);
|
||||||
|
float score = model.Predict(frame);
|
||||||
|
var now = sw.Elapsed;
|
||||||
|
if (score >= 0.5f && (now - lastTrigger).TotalSeconds >= 1.0) {
|
||||||
|
Console.WriteLine($"DETECTED alexa score={score:0.000} t={now.TotalSeconds:0.0}s");
|
||||||
|
lastTrigger = now;
|
||||||
|
FireAndForgetBeep(device, beepBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- Ctrl-C via `Console.CancelKeyPress` → `cts.Cancel()` → break out of `Take`, stop input stream, dispose `WakewordModel`, `PortAudio.Terminate()`. Graceful.
|
||||||
|
|
||||||
|
### `FireAndForgetBeep(int device, short[] beepBuffer)` — the bug fix
|
||||||
|
|
||||||
|
The Python probe blocks the input thread for ~200 ms on each beep (`findings.md` § A). The C# probe fixes this:
|
||||||
|
|
||||||
|
- On detection, open a new output-only `Stream` instance (16 kHz, 1 channel, Int16, 1024-frame block). Callback drains `beepBuffer` at a write offset; on exhaustion sets a `ManualResetEventSlim` and returns `StreamCallbackResult.Complete`.
|
||||||
|
- A small `Task.Run` waits the event, then `Stream.Stop()` + `Stream.Dispose()`s the stream. Main thread does not wait.
|
||||||
|
- The 1.0 s cooldown is well past the 200 ms playback duration, so the previous beep stream has always finished disposing before the next one is opened. If for any reason it hasn't, opening a fresh one is fine — they're independent.
|
||||||
|
|
||||||
|
This is the first time we run two PortAudio streams concurrently on the same USB device. Validating that this works is part of what the probe surfaces.
|
||||||
|
|
||||||
|
## Data flow
|
||||||
|
|
||||||
|
```
|
||||||
|
PortAudio input callback (RT thread)
|
||||||
|
└─ alloc short[1280], memcpy from PortAudio buffer
|
||||||
|
└─ queue.TryAdd(frame, 0ms)
|
||||||
|
├─ success → return Continue
|
||||||
|
└─ failure → log "[status] consumer behind", drop frame, return Continue
|
||||||
|
|
||||||
|
Main thread
|
||||||
|
└─ for each frame from queue:
|
||||||
|
├─ WakewordModel.Predict(frame) # runs all three ONNX models
|
||||||
|
├─ if score >= 0.5 AND cooldown ok:
|
||||||
|
│ ├─ print DETECTED line
|
||||||
|
│ └─ FireAndForgetBeep(device, beepBuffer):
|
||||||
|
│ ├─ create ManualResetEventSlim 'done'
|
||||||
|
│ ├─ create output Stream (callback drains beepBuffer, Sets 'done' on exhaustion)
|
||||||
|
│ ├─ Task.Run(() => { done.Wait(); stream.Stop(); stream.Dispose(); })
|
||||||
|
│ └─ stream.Start(); return immediately
|
||||||
|
└─ loop
|
||||||
|
|
||||||
|
Output callback (separate RT thread, one per beep)
|
||||||
|
└─ drain beepBuffer → return Complete on exhaustion
|
||||||
|
└─ Sets 'done' event → cleanup Task wakes and disposes the stream
|
||||||
|
```
|
||||||
|
|
||||||
|
Three concerns isolated to three threads: PortAudio's input callback (allocate + enqueue only), the main thread (all inference + dispatch), per-beep PortAudio output callbacks. No locks in the steady state — the queue and the per-beep events are the synchronisation surface.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
- USB device not found → device table to stderr, exit 1. (Same as Test 1.)
|
||||||
|
- `PortAudio.Initialize`, `Stream` ctor, `Stream.Start` → `PaError != NoError` throws `PortAudioException` from PortAudioSharp; uncaught, crashes with a useful trace. (Same as Test 1.)
|
||||||
|
- `WakewordModel` constructor:
|
||||||
|
- Missing ONNX file at given path → throw `FileNotFoundException` with the expected path.
|
||||||
|
- Loaded model's `InputMetadata` shape disagrees with what the pipeline expects → throw `InvalidOperationException` with which model and what shape was declared vs expected.
|
||||||
|
- Input overflow flag → log `[status] input overflow` to stderr, keep going. Same as Test 1 and the Python probe.
|
||||||
|
- Queue full at enqueue time → log `[status] consumer behind`, drop frame, keep going. Sustained occurrence is a probe finding worth documenting in `findings.md` post-run.
|
||||||
|
- Anything else (PortAudio mid-stream device disappearance, onnxruntime inference errors, CoreCLR issues) → uncaught throw. Probe, not production.
|
||||||
|
|
||||||
|
## Deploy & run (`bin/probe-cs-2`)
|
||||||
|
|
||||||
|
A copy of `bin/probe-cs` with three changes: project path, remote dir name, run path. Body:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PI_HOST="${PI_HOST:-192.168.50.115}"
|
||||||
|
PI_USER="${PI_USER:-pi}"
|
||||||
|
PI_PASS="${PI_PASS:-assistant}"
|
||||||
|
PUBLISH_DIR="${PUBLISH_DIR:-/tmp/probe-cs-2-out}"
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
echo ">> dotnet publish (linux-arm64, self-contained)"
|
||||||
|
dotnet publish tests/02-wakeword-cs \
|
||||||
|
-c Release -r linux-arm64 --self-contained \
|
||||||
|
-o "$PUBLISH_DIR"
|
||||||
|
|
||||||
|
echo ">> scp to $PI_USER@$PI_HOST:~/probe-cs-2/ (wipe-and-replace)"
|
||||||
|
sshpass -p "$PI_PASS" ssh -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'rm -rf ~/probe-cs-2 && mkdir ~/probe-cs-2'
|
||||||
|
sshpass -p "$PI_PASS" scp -r "$PUBLISH_DIR/." \
|
||||||
|
"$PI_USER@$PI_HOST:~/probe-cs-2/"
|
||||||
|
|
||||||
|
echo ">> ssh + run on Pi"
|
||||||
|
sshpass -p "$PI_PASS" ssh -t -o StrictHostKeyChecking=accept-new \
|
||||||
|
"$PI_USER@$PI_HOST" 'cd ~/probe-cs-2 && chmod +x Probe && ./Probe'
|
||||||
|
```
|
||||||
|
|
||||||
|
Note `ssh -t` (pseudo-terminal) so Ctrl-C from the workstation propagates as SIGINT to the remote process for the `Console.CancelKeyPress` graceful-shutdown path. Test 1 didn't need this because the probe self-terminated.
|
||||||
|
|
||||||
|
Cleanup (when the probe is retired): `ssh pi@... 'rm -rf ~/probe-cs-2'` plus `git rm -r tests/02-wakeword-cs bin/probe-cs-2`.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
The pass criteria above. There are no automated tests.
|
||||||
|
|
||||||
|
**Hardware test protocol.**
|
||||||
|
|
||||||
|
1. `bin/probe-cs-2` from the workstation. Wait for `Listening...` prompt.
|
||||||
|
2. Position yourself ~1 m from the USB Speaker Phone.
|
||||||
|
3. Say "alexa" at normal volume, 10 times, pausing ≥ 2 s between attempts. Count detections (printed line + beep).
|
||||||
|
4. Read three minutes of arbitrary text aloud (newspaper or book — anything that isn't "alexa"). Count false-positive beeps.
|
||||||
|
5. From a second terminal: `sshpass -p assistant ssh pi@192.168.50.115 htop`. Read the `Probe` process CPU% column.
|
||||||
|
|
||||||
|
Pass = ≥ 8/10 detections, ≤ 1 false positive per minute, < 50% of one core. Ctrl-C to exit.
|
||||||
|
|
||||||
|
### Diagnostic levers if it fails
|
||||||
|
|
||||||
|
- **Detections < 8/10 but scores log near threshold.** Likely buffer geometry off-by-one in the embedding stage; re-verify `WakewordModel.cs` against `openwakeword/model.py` side-by-side.
|
||||||
|
- **No detections, scores ≈ 0.** Tensor-shape or dtype mismatch upstream of the classifier — the startup assertion should have caught it; if it didn't, the assertion is incomplete. Fix the assertion first, then re-check.
|
||||||
|
- **Excess false positives.** Threshold too low for this model + room. The Python probe passed at 0.5; if C# scores are systematically higher, the bug is in the port. If lower, also a port bug. Either way, do not blindly retune the threshold.
|
||||||
|
- **CPU > 50%.** Confirm `IntraOpNumThreads = 1` was set on all sessions. Inspect with `htop` whether the spike is on input frames or on a sustained background — the latter suggests something is spinning.
|
||||||
|
- **Input-overflow / consumer-behind warnings.** Inference latency spike. Log per-frame `WakewordModel.Predict` duration; spikes above 80 ms mean inference cannot keep up.
|
||||||
|
|
||||||
|
## Open assumption (verify at implementation time)
|
||||||
|
|
||||||
|
`Microsoft.ML.OnnxRuntime` NuGet package on linux-arm64 ships its own native `libonnxruntime.so` for the RID, so the self-contained publish includes it. Verify by listing the publish dir after `dotnet publish`. If for some reason it doesn't (older versions of the package split the native runtime separately), fall back to `Microsoft.ML.OnnxRuntime.Managed` + an explicit native-runtime package for the RID, or fetch the prebuilt binary from the ONNX Runtime GitHub releases. Confirmed at implementation, not now — does not affect the design.
|
||||||
|
|
||||||
|
## Post-run: `findings.md` update
|
||||||
|
|
||||||
|
After hardware passes, append a `## C# wakeword probe outcome (2026-06-12 — Test 2 ported to C#)` section to `findings.md` in the same shape as the Test 1 section:
|
||||||
|
|
||||||
|
- What we proved (the three pass criteria, plus "two PortAudio streams on one device works", plus "ONNX runtime on Pi performant enough").
|
||||||
|
- Key gotchas hit during the port (anything not already captured by the Test 1 section).
|
||||||
|
- Open questions deferred to the main assistant build (custom wakeword training, AEC for speaker-is-mic, queue depth in real conditions).
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Python probe: `tests/02-wakeword/main.py`
|
||||||
|
- Test 1 C# probe (patterns reused): `tests/01-record-play-cs/Program.cs`, `tests/01-record-play-cs/Probe.csproj`, `bin/probe-cs`
|
||||||
|
- openwakeword inference reference (ground truth for buffering geometry): `~/.local/lib/python3.13/site-packages/openwakeword/model.py` on the Pi
|
||||||
|
- Prototype findings: `findings.md` — especially "C# probe outcome (2026-06-12 — Test 1 ported to C#)", "Force PortAudio through ALSA's `plug` plugin", "Beep / TTS blocks the input stream", "Cosmetic / known-warnings" (ORT GPU detection)
|
||||||
|
- Microsoft.ML.OnnxRuntime — fetch current package docs via Context7 at implementation time (`/microsoft/onnxruntime`).
|
||||||
|
- Test 1 C# probe spec / plan (precedent for this spec's shape): `docs/superpowers/specs/2026-06-12-record-play-csharp-probe-design.md`, `docs/superpowers/plans/2026-06-12-record-play-csharp-probe.md`
|
||||||
|
- Pi access: `pi@192.168.50.115`, password `assistant` (see `CLAUDE.md`)
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
# Test 3 — full-cycle C# probe (wakeword + record + playback)
|
||||||
|
|
||||||
|
Combine the C# Test 1 (record + play) and Test 2 (wakeword + beep) into a single state machine that exercises the full assistant conversation cycle on the Pi:
|
||||||
|
|
||||||
|
> Idle, listen for "alexa" → beep → record 5 s → play the recording back through the same USB Speaker Phone → idle, listen for "alexa" again. Repeat indefinitely until Ctrl-C.
|
||||||
|
|
||||||
|
This is the last throwaway probe before the main Pi-client code starts. Its job is to surface anything the prior two probes couldn't, in particular **whether the wakeword detector and the recording/playback path coexist correctly on one USB device with one persistent input stream**.
|
||||||
|
|
||||||
|
Read `findings.md` (§ "C# probe outcome", § "C# wakeword probe outcome") before reading this. Everything below assumes those two outcomes as ground truth.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A single-binary C# probe that:
|
||||||
|
|
||||||
|
1. Opens one persistent PortAudio input stream and one `WakewordModel` for the lifetime of the process.
|
||||||
|
2. Runs a four-state machine on each incoming 80 ms audio frame: **IDLE → BEEPING → RECORDING → PLAYBACK → IDLE**.
|
||||||
|
3. Plays a 200 ms beep on detection (fire-and-forget output stream).
|
||||||
|
4. Records 5 s of audio into an in-memory `short[]` buffer.
|
||||||
|
5. Plays that buffer back through the same USB device (fire-and-forget output stream).
|
||||||
|
6. Resets the wakeword model on every IDLE re-entry so the next "alexa" is detected quickly (target ≤ ~2 s after playback ends; see "Why detector gating works"), not garbage from stale ring buffers.
|
||||||
|
7. Exits cleanly on Ctrl-C from the workstation (SIGINT propagated via `ssh -t`).
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- No automated tests. Verification is hardware-only (run on Pi, judge by ear + console output).
|
||||||
|
- No software AEC. The detector is gated during PLAYBACK (state-machine discards frames); that is sufficient to prevent the playback re-triggering the wakeword. The Anhui LISTENAI USB device may have hardware AEC; not in scope to probe.
|
||||||
|
- No custom wakeword. Stock "alexa" classifier carried forward from Test 2.
|
||||||
|
- No WAV output. Test 1 wrote `out.wav`; Test 3 keeps the recording in memory. If a future debugging session needs raw captures, add an opt-in flag then.
|
||||||
|
- No `WakewordModel` extraction into a shared library. The file is copied verbatim from Test 2 into Test 3, and `Reset()` is added only to the Test 3 copy. Test 2's copy is frozen.
|
||||||
|
- No barge-in (user interrupting playback by speaking). The detector is fully gated during PLAYBACK; there is no way for the user to retrigger mid-playback.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────────────────────────┐
|
||||||
|
│ PortAudio input stream (persistent) │
|
||||||
|
│ 16 kHz mono Int16, 1280 frames/cb │
|
||||||
|
│ callback memcpy → BlockingCollection │
|
||||||
|
└────────────────┬─────────────────────────┘
|
||||||
|
│ short[1280] frames
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────────────┐
|
||||||
|
│ Consumer thread │
|
||||||
|
│ while (!cts.IsCancellationRequested) { │
|
||||||
|
│ frame = queue.Take(cts.Token); │
|
||||||
|
│ switch (_state) { │
|
||||||
|
│ case IDLE → model.Predict(...) │
|
||||||
|
│ case BEEPING → discard + count │
|
||||||
|
│ case RECORDING→ memcpy → recordBuf │
|
||||||
|
│ case PLAYBACK → discard │
|
||||||
|
│ } │
|
||||||
|
│ } │
|
||||||
|
└────────────────┬─────────────────────────┘
|
||||||
|
│ on demand
|
||||||
|
┌──────────────┴──────────────┐
|
||||||
|
▼ ▼
|
||||||
|
FireAndForgetBeep FireAndForgetPlayback
|
||||||
|
(200 ms 880 Hz tone) (5.04 s of recordBuf)
|
||||||
|
Output stream pattern from Test 2 (Start under
|
||||||
|
try/catch BEFORE Task.Run + 2 s timeout on done.Wait)
|
||||||
|
```
|
||||||
|
|
||||||
|
One process. One persistent input stream. Per-cycle output streams spawned fire-and-forget. State machine lives entirely in the consumer thread, which is the single writer to `_state`, so `_state` only needs `volatile` — no locks.
|
||||||
|
|
||||||
|
## State machine
|
||||||
|
|
||||||
|
Frame size is 1280 samples = 80 ms at 16 kHz. All counters below are in frames.
|
||||||
|
|
||||||
|
| State | Per-frame action | Transition |
|
||||||
|
|-|-|-|
|
||||||
|
| **IDLE** | `score = model.Predict(frame)` | `score ≥ 0.5` AND cooldown elapsed → log `DETECTED`, call `FireAndForgetBeep(...)`, set `beepFrames = 0`, → **BEEPING** |
|
||||||
|
| **BEEPING** | discard frame; `beepFrames++` | after **3 frames** (240 ms) → allocate `recordBuf = new short[63 * 1280]`, `recordOffset = 0`, → **RECORDING** |
|
||||||
|
| **RECORDING** | `Buffer.MemoryCopy` frame into `recordBuf` at `recordOffset`; `recordOffset += 1280` | after **63 frames** (5.04 s, full buffer) → call `FireAndForgetPlayback(recordBuf, …)`, → **PLAYBACK** |
|
||||||
|
| **PLAYBACK** | discard frame; check `volatile bool _playbackDoneFlag` | `_playbackDoneFlag == true` → reset flag, call `model.Reset()`, → **IDLE** |
|
||||||
|
|
||||||
|
Rationale for the constants:
|
||||||
|
|
||||||
|
- **BEEPING = 3 frames (240 ms).** Beep duration is 200 ms; 3 × 80 ms = 240 ms is the next frame boundary at-or-above 200 ms, leaving a 40 ms margin to absorb device buffering, speaker reverb, and PortAudio's output latency so the recording doesn't open with the tail of the beep. If verification shows audible beep tail still leaking into the recording, raise to 4 frames (320 ms) — easy tweak.
|
||||||
|
- **RECORDING = 63 frames (5.04 s).** 5 s = 62.5 frames; rounding up to 63 avoids partial-frame handling. The extra 40 ms of recording is inaudibly different from 5 s.
|
||||||
|
- **Cooldown** applies only to the IDLE→BEEPING transition. The state machine itself blocks re-detection in the other states. 1 s cooldown carries forward from Test 2.
|
||||||
|
|
||||||
|
`_playbackDoneFlag` is set by the playback output stream's audio callback when its read offset reaches the end of `recordBuf`. The consumer checks the flag on every PLAYBACK frame (and only on PLAYBACK frames). Between the flag firing and the consumer seeing it, the consumer continues to drain input frames into the discard branch — that's what keeps the input stream from backing up.
|
||||||
|
|
||||||
|
## Why detector gating works (and why Reset matters)
|
||||||
|
|
||||||
|
The USB Speaker Phone is the same physical device for input and output. During PLAYBACK, the recorded audio comes out the speaker and re-enters the input stream. If `WakewordModel.Predict` were running across all states, any "alexa" buried in the recording would re-trigger detection, kicking the state machine back into BEEPING mid-playback — an infinite loop.
|
||||||
|
|
||||||
|
The state-machine gating (PLAYBACK discards every frame, doesn't call `Predict`) prevents the re-trigger directly. That much is straightforward.
|
||||||
|
|
||||||
|
The subtler issue is what happens to `WakewordModel`'s internal ring buffers (`_rawRing`, `_melRing`, `_embRing`) during BEEPING + RECORDING + PLAYBACK. With ~10 s of skipped `Predict` calls (240 ms BEEPING + 5.04 s RECORDING + ~5 s PLAYBACK), the rings hold audio from ~10 s ago — pre-detection. When we return to IDLE and resume `Predict`, the next call computes mel features from a window that straddles 10-second-old audio at the bottom and fresh post-playback audio at the top. The first one or two predictions on re-entry would be on this garbage window.
|
||||||
|
|
||||||
|
The `WarmupFrames = 16` guard already in `WakewordModel.cs` exists to suppress exactly this kind of cold-start junk on initial process boot. It does NOT re-arm after a pause. So without intervention, the first ~1.3 s after returning to IDLE would be a false-positive vulnerability window.
|
||||||
|
|
||||||
|
**Fix:** Add a `WakewordModel.Reset()` method that clears `_rawRingFill = 0`, `_melRing.Clear()`, `_embRing.Clear()`, `_framesSeen = 0`. The consumer calls it on the PLAYBACK→IDLE transition. The existing `WarmupFrames` guard then suppresses the next 16 `Predict` calls (~1.3 s of audio) until the rings refill with fresh post-playback audio. After that, detection runs as it did at startup.
|
||||||
|
|
||||||
|
This means the next "alexa" after playback can only be detected on a frame at least 1.3 s after the IDLE re-entry, plus whatever time the user takes to start saying "alexa". The verification budget for "next detection within ~1 s of playback" needs to acknowledge this: the practical lower bound is ~1.3 s (warmup) + ~0.5 s (user reaction + saying the word). See "Pass criteria" below for the adjusted threshold.
|
||||||
|
|
||||||
|
## Components and files
|
||||||
|
|
||||||
|
```
|
||||||
|
tests/03-full-cycle-cs/
|
||||||
|
├── Probe.csproj
|
||||||
|
│ <TargetFramework>net9.0</TargetFramework>
|
||||||
|
│ <RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
│ <SelfContained>true</SelfContained>
|
||||||
|
│ <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
│ <InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
│ PackageReference PortAudioSharp2 1.0.6
|
||||||
|
│ PackageReference Microsoft.ML.OnnxRuntime 1.26.0
|
||||||
|
│ Content Include="models/*.onnx" CopyToOutputDirectory=PreserveNewest
|
||||||
|
│ RootNamespace=FullCycleProbe, AssemblyName=Probe
|
||||||
|
│
|
||||||
|
├── Program.cs ~200 lines, top-level statements:
|
||||||
|
│ 1. Environment.SetEnvironmentVariable + Libc.setenv for
|
||||||
|
│ PA_ALSA_PLUGHW=1 and ORT_LOGGING_LEVEL=3
|
||||||
|
│ 2. OrtEnv.CreateInstanceWithOptions with
|
||||||
|
│ logLevel = ORT_LOGGING_LEVEL_ERROR (BEFORE any InferenceSession)
|
||||||
|
│ 3. PortAudio.Initialize, FindUsbDevice (lifted from Test 2)
|
||||||
|
│ 4. WakewordModel ctor with models/{melspectrogram,embedding_model,alexa}.onnx
|
||||||
|
│ 5. BlockingCollection<short[]> queue (bounded 16)
|
||||||
|
│ 6. CancellationTokenSource + Console.CancelKeyPress handler
|
||||||
|
│ 7. Input StreamParameters + callback (memcpy → queue.TryAdd)
|
||||||
|
│ 8. Stream construction + Start
|
||||||
|
│ 9. Consumer loop with state-machine switch (sketched above)
|
||||||
|
│ 10. finally: stream.Stop, model.Dispose, PortAudio.Terminate
|
||||||
|
│ Helpers: FireAndForgetBeep (from Test 2), FireAndForgetPlayback (new,
|
||||||
|
│ same shape but reads recordBuf and sets _playbackDoneFlag),
|
||||||
|
│ MakeBeep, FindUsbDevice, static class Libc.setenv
|
||||||
|
│
|
||||||
|
├── WakewordModel.cs Copied verbatim from tests/02-wakeword-cs/.
|
||||||
|
│ Add Reset() method (≈5 lines):
|
||||||
|
│ public void Reset() {
|
||||||
|
│ _rawRingFill = 0;
|
||||||
|
│ _melRing.Clear();
|
||||||
|
│ _embRing.Clear();
|
||||||
|
│ _framesSeen = 0;
|
||||||
|
│ }
|
||||||
|
│ Namespace changes from WakewordProbe → FullCycleProbe.
|
||||||
|
│
|
||||||
|
└── models/ Copied from tests/02-wakeword-cs/models/.
|
||||||
|
melspectrogram.onnx, embedding_model.onnx, alexa.onnx
|
||||||
|
SHA-256 unchanged from the vendored set in commit 849ad47.
|
||||||
|
|
||||||
|
bin/probe-cs-3 Copy of bin/probe-cs-2 with two string swaps:
|
||||||
|
tests/02-wakeword-cs → tests/03-full-cycle-cs
|
||||||
|
~/probe-cs-2 → ~/probe-cs-3
|
||||||
|
PUBLISH_DIR /tmp/probe-cs-2-out → /tmp/probe-cs-3-out
|
||||||
|
ssh -t is retained (Ctrl-C propagation).
|
||||||
|
chmod +x at install time.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Concurrency model
|
||||||
|
|
||||||
|
Three threads of execution in steady state:
|
||||||
|
|
||||||
|
1. **PortAudio input audio thread** (managed by the library). Runs the input callback every 80 ms. Allocates a `short[1280]`, memcpy's the input buffer, calls `queue.TryAdd(frame, 0)`. On `TryAdd` failure (queue full), logs `[status] consumer behind, dropping frame`. On `InputOverflow` status flag, logs `[status] input overflow`. Never blocks.
|
||||||
|
2. **Consumer thread** (`Task.Run` or just main thread after `stream.Start`). The state-machine loop above. Single writer of `_state`. Calls `model.Predict` in IDLE; calls `model.Reset` on PLAYBACK→IDLE.
|
||||||
|
3. **PortAudio output audio threads** (one per fire-and-forget beep/playback). Each output stream has its own callback that writes from a managed buffer (beep buffer or recordBuf). Sets `done.Set()` (beep) or `_playbackDoneFlag = true` (playback) when its source buffer is fully consumed.
|
||||||
|
|
||||||
|
Shared mutable state:
|
||||||
|
|
||||||
|
- `volatile State _state` — single writer (consumer), multiple readers (consumer only, actually). Volatile is enough.
|
||||||
|
- `volatile bool _playbackDoneFlag` — set by playback audio thread, read+reset by consumer.
|
||||||
|
- `BlockingCollection<short[]> queue` — its own concurrency.
|
||||||
|
|
||||||
|
No locks. No `Interlocked`. The state machine is intentionally a single-writer design so we don't have to reason about race conditions.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
- **Input overflow** (PortAudio callback status flag): log `[status] input overflow`, continue. No corrective action — the buffer dropped audio at the kernel level, nothing the consumer can do.
|
||||||
|
- **Consumer-behind** (`queue.TryAdd` returns false because the bounded queue is full): log `[status] consumer behind, dropping frame`, continue. In steady state this should never fire; if it does during verification it's a signal that one of the state branches is too slow (likely `Predict` exceeding 80 ms, which Test 2 confirmed doesn't happen on the Pi).
|
||||||
|
- **Output stream `Start()` throws**: dispose the stream, log `[error] output stream failed to start: <ex>`, snap `_state` back to IDLE, call `model.Reset()`. The current cycle is lost; the probe survives. (Test 2 surfaced this as the exact failure mode the fire-and-forget pattern guards against.)
|
||||||
|
- **`WakewordModel` exceptions** during `Predict` or `Reset`: let propagate. The `finally` block disposes the model and terminates PortAudio. Probe exits non-zero. This is fail-fast on the assumption that any model exception is an invariant violation worth investigating.
|
||||||
|
- **Ctrl-C from workstation**: `Console.CancelKeyPress` calls `cts.Cancel()`. The consumer's `queue.Take(cts.Token)` throws `OperationCanceledException`, the outer `try` swallows it, the `finally` block runs `stream.Stop` + `model.Dispose` + `PortAudio.Terminate`. Process exits 0. `ssh -t` in `bin/probe-cs-3` ensures the SIGINT reaches the remote process.
|
||||||
|
|
||||||
|
## Gotchas carried forward (from findings.md)
|
||||||
|
|
||||||
|
1. **Libc.setenv for env vars libc reads via getenv().** `Environment.SetEnvironmentVariable` does NOT propagate to libc on Linux. Apply `Libc.setenv` for both `PA_ALSA_PLUGHW` (PortAudio) and `ORT_LOGGING_LEVEL` (onnxruntime) alongside the managed call.
|
||||||
|
2. **OrtEnv.CreateInstanceWithOptions BEFORE first InferenceSession.** ORT 1.26.0's GPU-detection logger fires from C++ before managed code runs, so `Libc.setenv("ORT_LOGGING_LEVEL", ...)` alone doesn't suppress the `/sys/class/drm/card0` warnings. Construct `OrtEnv` with `logLevel = ORT_LOGGING_LEVEL_ERROR` *before* `WakewordModel`'s constructor runs.
|
||||||
|
3. **`using Stream = PortAudioSharp.Stream;`** at the top of `Program.cs` to disambiguate from `System.IO.Stream` that `ImplicitUsings` pulls in.
|
||||||
|
4. **`<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`** in `Probe.csproj` for the callback's `Buffer.MemoryCopy` over `short*`.
|
||||||
|
5. **Fire-and-forget output stream lifetime.** `stream.Start()` runs synchronously under try/catch *before* `Task.Run` launches; on `Start()` failure, dispose + rethrow without ever creating the cleanup task. The cleanup task waits on `done.Wait(TimeSpan.FromSeconds(2))` — 2 s timeout so a faulting callback that silently fails (PortAudio swallows callback exceptions) can't hang cleanup. Applied to both `FireAndForgetBeep` AND `FireAndForgetPlayback`.
|
||||||
|
6. **Vendored alexa classifier has static batch dim `[1, 16, 96]`.** `WakewordModel`'s shape assertion intentionally skips dim 0 so both `1` (static) and `-1` (dynamic) pass. Preserve.
|
||||||
|
7. **Mel-spectrogram output shape `(1, 1, n_frames, 32)`.** `n_frames` lives at `Dimensions[2]`, not `[1]`. Preserve in the copied `WakewordModel.cs`.
|
||||||
|
8. **`ssh -t` for Ctrl-C propagation.** `bin/probe-cs-3` forces a pseudo-TTY so workstation SIGINT reaches the remote `./Probe` and triggers `Console.CancelKeyPress`.
|
||||||
|
|
||||||
|
## Verification (hardware only, no automated tests)
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/probe-cs-3
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected console flow (abridged):
|
||||||
|
|
||||||
|
```
|
||||||
|
[wakeword-model] melspectrogram.onnx
|
||||||
|
input 'input': shape=[1,-1] dtype=Single
|
||||||
|
output 'output': shape=[1,1,-1,32] dtype=Single
|
||||||
|
[wakeword-model] embedding_model.onnx
|
||||||
|
...
|
||||||
|
[wakeword-model] alexa.onnx
|
||||||
|
input 'onnx::Unsqueeze_0': shape=[1,16,96] dtype=Single
|
||||||
|
output ...
|
||||||
|
Loaded in 0.6s.
|
||||||
|
Using device 3 ('USB Speaker Phone (LISTENAI ...)')
|
||||||
|
Listening. Say 'alexa'. Ctrl-C to exit.
|
||||||
|
DETECTED alexa score=0.872 t=4.3s
|
||||||
|
[state] BEEPING
|
||||||
|
[state] RECORDING
|
||||||
|
[state] PLAYBACK
|
||||||
|
[state] IDLE
|
||||||
|
DETECTED alexa score=0.811 t=15.2s
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
(Exact `[wakeword-model]` shape output reflects whatever the ONNX models actually report; the example above is illustrative.)
|
||||||
|
|
||||||
|
### Pass criteria
|
||||||
|
|
||||||
|
All must hold during a single run on the Pi:
|
||||||
|
|
||||||
|
1. **3 full cycles back-to-back** without restarting the probe. Each playback is clearly recognisable as what was spoken into the mic in the preceding 5 s.
|
||||||
|
2. **Next "alexa" after playback** is detected within ~2 s of playback ending. (The ~1.3 s `WarmupFrames` re-arm after `model.Reset()` is a hard floor; budget another ~0.5 s for user reaction + saying the word. If detection consistently takes longer than ~2 s, that's a regression vs. Test 2's IDLE-state detection latency and worth investigating.)
|
||||||
|
3. **Wakeword detection rate during IDLE** ≥ 8/10 attempts (carries forward from Test 2's measured rate).
|
||||||
|
4. **CPU during IDLE steady state** < 50% of one core, measured by `top` on the Pi. (Test 2 measured 17–31% on the same workload; Test 3 adds only a state-machine switch — no extra CPU in IDLE.)
|
||||||
|
5. **No `[status] input overflow` or `[status] consumer behind` warnings** during any phase of normal operation.
|
||||||
|
6. **No stuck states.** The probe never sits in BEEPING, RECORDING, or PLAYBACK for longer than its bounded duration. (BEEPING ≤ 240 ms, RECORDING ≤ 5.04 s, PLAYBACK ≤ ~5.5 s including device drain.)
|
||||||
|
7. **Workstation Ctrl-C** triggers a clean exit: `\nbye` line, process exit 0, no stack trace, terminal returns to prompt.
|
||||||
|
|
||||||
|
### What success teaches us for the main assistant
|
||||||
|
|
||||||
|
- The "persistent input stream + state-machine consumer + per-cycle output streams" shape works on this hardware. The main assistant can build directly on it.
|
||||||
|
- Detector gating during playback (without AEC) is sufficient to prevent self-trigger, at least with stock "alexa" and the recorded user audio. Real TTS may need re-evaluation; this probe doesn't answer that.
|
||||||
|
- `WakewordModel.Reset()` belongs in the production model interface — the main assistant will need it for the same reasons.
|
||||||
|
|
||||||
|
### What failure would tell us
|
||||||
|
|
||||||
|
- If detection rate drops below 8/10 in IDLE: the state-machine overhead is interfering. Investigate (unlikely — the switch is O(constant)).
|
||||||
|
- If `[status] consumer behind` warnings appear: one of the consumer branches is too slow. Most likely `Predict` exceeded 80 ms (Test 2 didn't measure this directly). Add per-`Predict` timing and re-run.
|
||||||
|
- If playback re-triggers detection (probe loops in BEEPING/RECORDING/PLAYBACK indefinitely): `Reset` is wrong, or `_playbackDoneFlag` plumbing has a race. Investigate via logging which state the trigger occurred in.
|
||||||
|
- If `Ctrl-C` produces a stack trace: `OperationCanceledException` isn't being caught at the right level. Fix.
|
||||||
|
|
||||||
|
## Reference paths
|
||||||
|
|
||||||
|
- Spec (this doc): `docs/superpowers/specs/2026-06-12-test-3-full-cycle-csharp-design.md`
|
||||||
|
- Plan (next): `docs/superpowers/plans/2026-06-12-test-3-full-cycle-csharp.md`
|
||||||
|
- Code (to be written): `tests/03-full-cycle-cs/`
|
||||||
|
- Deploy (to be written): `bin/probe-cs-3`
|
||||||
|
- Test 1 reference: `tests/01-record-play-cs/Program.cs`, `tests/01-record-play-cs/WavWriter.cs`, `bin/probe-cs`
|
||||||
|
- Test 2 reference: `tests/02-wakeword-cs/Program.cs`, `tests/02-wakeword-cs/WakewordModel.cs`, `bin/probe-cs-2`
|
||||||
|
- Pi access: `pi@192.168.50.115`, password `assistant` (see `CLAUDE.md`)
|
||||||
|
- Canonical findings: `findings.md` § "C# probe outcome (2026-06-12 — Test 1 ported to C#)" and § "C# wakeword probe outcome (2026-06-12 — Test 2 ported to C#)"
|
||||||
@@ -0,0 +1,474 @@
|
|||||||
|
# Test 4a — OpenAI Realtime API one-shot voice round-trip (C#)
|
||||||
|
|
||||||
|
A throwaway C# probe that has a single round-trip spoken conversation with the OpenAI Realtime API:
|
||||||
|
|
||||||
|
> Boot → beep → user speaks one sentence → server VAD ends the turn → assistant audio reply plays back through the same USB Speaker Phone → exit 0.
|
||||||
|
|
||||||
|
No wakeword. No loop. No state machine. One process, runs once, dies.
|
||||||
|
|
||||||
|
Test 4a's job is to debug the OpenAI Realtime API surface (auth, WS handshake, `session.update` payload, event JSON shapes, base64 audio decode, server VAD endpointing, streaming-length output-stream cleanup) **in isolation** from the harder concurrency story of Test 4b (wakeword + loop + 16↔24 kHz coexistence).
|
||||||
|
|
||||||
|
Read `findings.md` (§ "C# probe outcome", § "C# wakeword probe outcome", § "C# full-cycle outcome") before reading this. Everything below assumes those outcomes as ground truth. The patterns lifted directly from Test 3 are: `Libc.setenv`, the `using Stream = PortAudioSharp.Stream;` import disambiguation, `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`, the fire-and-forget output stream lifetime (Start under try/catch BEFORE the cleanup task launches), `FindUsbDevice`, the `MakeBeep` helper, `bin/probe-cs-3`'s wipe-and-replace publish shape with `ssh -t`.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A single-binary C# probe that:
|
||||||
|
|
||||||
|
1. Reads the OpenAI API key from `~/.openai_key` on the Pi (mode 600).
|
||||||
|
2. Connects to `wss://api.openai.com/v1/realtime?model=gpt-realtime` over a WebSocket with `Authorization: Bearer …` + `OpenAI-Beta: realtime=v1` headers.
|
||||||
|
3. Sends one `session.update` event pinning audio formats, voice, instructions, and `turn_detection: server_vad`.
|
||||||
|
4. Plays a startup beep through the USB Speaker Phone.
|
||||||
|
5. Opens a 24 kHz mono Int16 input stream on the same device; streams mic audio to OpenAI as `input_audio_buffer.append` events.
|
||||||
|
6. Waits for the server VAD to fire `input_audio_buffer.speech_stopped` (server then auto-commits the buffer and creates the response).
|
||||||
|
7. Stops sending mic audio at that point. Receives `response.audio.delta` chunks, base64-decodes them, plays them through a 24 kHz output stream on the same device.
|
||||||
|
8. On `response.done` AND the output buffer drained, closes the WebSocket cleanly, terminates PortAudio, exits 0.
|
||||||
|
9. Exits cleanly on Ctrl-C from the workstation (SIGINT propagated via `ssh -t`) at any stage.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **No wakeword.** No `WakewordModel.cs`. No 16 kHz audio path anywhere in 4a. (Test 4b layers that on top of this proven base.)
|
||||||
|
- **No state machine, no loop.** Probe runs once and exits.
|
||||||
|
- **No barge-in.** Once `speech_stopped` fires, mic frames are dropped on the floor. The user cannot interrupt the assistant in 4a.
|
||||||
|
- **No client-side resampling.** Mic = 24 kHz, output = 24 kHz, OpenAI = 24 kHz pcm16. PortAudio plug-resamples 48↔24 at ALSA on the USB device.
|
||||||
|
- **No WAV capture.** Logging is stdout/stderr only.
|
||||||
|
- **No retry on WS errors, no reconnect, no health check.** Fail loud, exit non-zero, log enough to diagnose.
|
||||||
|
- **No automated tests.** Hardware verification only.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────────────────────┐
|
||||||
|
│ ClientWebSocket │
|
||||||
|
│ wss://api.openai.com/v1/realtime? │
|
||||||
|
│ model=gpt-realtime │
|
||||||
|
│ Headers: Authorization: Bearer … │
|
||||||
|
│ OpenAI-Beta: realtime=v1 │
|
||||||
|
└────┬──────────────────────────┬──────┘
|
||||||
|
│ SendAsync │ ReceiveAsync
|
||||||
|
▼ ▼
|
||||||
|
┌──────────────────────────┐ ┌──────────────────────────────┐
|
||||||
|
│ UpstreamSendLoop │ │ ReceiveLoop (Task.Run) │
|
||||||
|
│ (Task.Run) │ │ while (!cts.Cancel) { │
|
||||||
|
│ drain upstreamChannel: │ │ msg = ReceiveAsync(…) │
|
||||||
|
│ pull short[1920] │ │ doc = JsonDocument.Parse │
|
||||||
|
│ → base64 │ │ switch (doc["type"]) │
|
||||||
|
│ → input_audio_ │ │ session.created → log │
|
||||||
|
│ buffer.append │ │ session.updated → log │
|
||||||
|
│ → ws.SendAsync │ │ speech_started → log │
|
||||||
|
│ exits when │ │ speech_stopped → │
|
||||||
|
│ _stopSending && empty│ │ _stopSending = true │
|
||||||
|
└──────────┬───────────────┘ │ response.audio.delta: │
|
||||||
|
│ (FullMode.Wait) │ base64 → short[] │
|
||||||
|
│ │ → downstreamChannel │
|
||||||
|
▲ │ .Writer.WriteAsync │
|
||||||
|
│ short[1920] │ audio_transcript.delta:│
|
||||||
|
│ (40 ms @ 24 kHz) │ append to log buf │
|
||||||
|
┌──────────┴───────────────┐ │ response.done → │
|
||||||
|
│ PortAudio input stream │ │ _noMoreDeltas = true │
|
||||||
|
│ 24 kHz mono Int16, │ │ downstreamChannel │
|
||||||
|
│ 1920 frames/cb │ │ .Writer.Complete │
|
||||||
|
│ callback: │ │ error → log + Cancel │
|
||||||
|
│ if !_micArmed: discard│ └────────────┬─────────────────┘
|
||||||
|
│ elif _stopSending: │ │
|
||||||
|
│ discard │ ▼
|
||||||
|
│ else: │ ┌──────────────────────────────┐
|
||||||
|
│ copy → short[] │ │ Channel<short[]> │
|
||||||
|
│ TryWrite to │ │ (bounded cap 64, │
|
||||||
|
│ upstreamChannel │ │ FullMode.Wait) │
|
||||||
|
└──────────────────────────┘ └────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ PortAudio output stream │
|
||||||
|
│ 24 kHz mono Int16, │
|
||||||
|
│ 1024 frames/cb │
|
||||||
|
│ callback: │
|
||||||
|
│ fill = 0 │
|
||||||
|
│ while fill < frameCount: │
|
||||||
|
│ if currentChunk == null: │
|
||||||
|
│ try pull short[] or break │
|
||||||
|
│ take = min(remaining, slack)│
|
||||||
|
│ memcpy chunk → output │
|
||||||
|
│ zero-fill remaining │
|
||||||
|
│ if currentChunk == null │
|
||||||
|
│ && Reader.Count == 0 │
|
||||||
|
│ && Completion.IsCompleted: │
|
||||||
|
│ doneFlag.Set() │
|
||||||
|
│ return Complete │
|
||||||
|
│ return Continue │
|
||||||
|
└────────────┬─────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
main thread: doneFlag.Wait
|
||||||
|
→ ws.CloseAsync(NormalClosure)
|
||||||
|
→ input/output stream.Stop
|
||||||
|
→ PortAudio.Terminate
|
||||||
|
→ exit 0
|
||||||
|
```
|
||||||
|
|
||||||
|
One process. Three managed threads (main + UpstreamSendLoop + ReceiveLoop) plus PortAudio's input and output audio threads (managed by the library).
|
||||||
|
|
||||||
|
**Shared mutable state — the full list, small on purpose:**
|
||||||
|
|
||||||
|
- `Channel<short[]> upstreamChannel`, bounded 64, `FullMode.Wait`, single-writer (mic callback), single-reader (UpstreamSendLoop). Holds mic frames waiting to be base64'd and sent.
|
||||||
|
- `Channel<short[]> downstreamChannel`, bounded 64, `FullMode.Wait`, **multi-writer** (main thread writes the startup beep; ReceiveLoop writes response audio chunks; temporally separated but Channel<T>'s `SingleWriter` flag is a contract not a runtime check, so keep it `false`), single-reader (output callback). Holds decoded audio chunks waiting to be played. Its `Writer.Complete()` (called by ReceiveLoop on `response.done`) is the WS-stream-end signal for the output callback.
|
||||||
|
- `volatile bool _micArmed`: starts `false`; set `true` after the startup beep has been queued and drained. Mic frames before this point are discarded so the beep doesn't get captured and shipped to OpenAI.
|
||||||
|
- `volatile bool _stopSending`: set by ReceiveLoop on `input_audio_buffer.speech_stopped`. Read by both the mic callback (which then discards) and UpstreamSendLoop (which then drains its queue and exits).
|
||||||
|
- `volatile bool _noMoreDeltas`: set by ReceiveLoop on `response.done`. Not strictly required because `downstreamChannel.Writer.Complete()` carries the same signal — but kept as a redundant fast path that the output callback can check without touching the channel.
|
||||||
|
- `ManualResetEventSlim doneFlag`: set by the output callback when it has drained the channel after completion. Awaited by main.
|
||||||
|
- `CancellationTokenSource cts`: wired to `Console.CancelKeyPress` + any WS error. Cancels both Task.Run loops; main's `doneFlag.Wait(cts.Token)` honours it.
|
||||||
|
|
||||||
|
No locks. No `Interlocked`. The bounded `Channel<T>` handles all the cross-thread audio handoff with the right backpressure semantics.
|
||||||
|
|
||||||
|
## WebSocket lifecycle
|
||||||
|
|
||||||
|
Six discrete phases, all in `Program.cs` top-level statements + the two `Task.Run` loops:
|
||||||
|
|
||||||
|
1. **Load API key.** `LoadApiKey()` helper reads `~/.openai_key`, trims whitespace. Mode 600 verified — non-fatal warning if looser. Fatal error (exit 2) if the file is missing, empty, or doesn't start with `sk-`.
|
||||||
|
2. **Construct `ClientWebSocket`** with two request headers: `Authorization: Bearer <key>` and `OpenAI-Beta: realtime=v1`. `ConnectAsync` to `wss://api.openai.com/v1/realtime?model=gpt-realtime` with `cts.Token`. On failure (DNS, TLS, 401, 403, 429, 5xx), log the exception message + any HTTP status carried on `WebSocketException.WebSocketErrorCode` / inner `HttpRequestException.StatusCode`; exit 3.
|
||||||
|
3. **Launch ReceiveLoop** (`Task.Run`) before sending anything, so we don't miss `session.created`.
|
||||||
|
4. **Send `session.update`.** First client message; pins the session config (see "Client → server events" below). Don't strictly block on `session.updated` — the receive loop logs it asynchronously — but the spec expects it within ~200 ms.
|
||||||
|
5. **Open + start PortAudio streams.** Input stream first (24 kHz mono Int16, 1920 frames/cb), then output stream (24 kHz mono Int16, 1024 frames/cb). Both opened on `FindUsbDevice()`. Launch UpstreamSendLoop (`Task.Run`).
|
||||||
|
6. **Startup beep + arm mic.** Queue a 200 ms 880 Hz beep (`MakeBeep` lifted from Test 3, regenerated at 24 kHz) onto `downstreamChannel`. Sleep ~300 ms (beep + drain margin). Set `_micArmed = true`. Print `Speak now.` to stdout.
|
||||||
|
7. **Wait for `doneFlag`** on the main thread (with `cts.Token` for Ctrl-C). On wake-up: `ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "done", …)` with a 2 s timeout token. Stop input stream, stop output stream, terminate PortAudio, exit 0.
|
||||||
|
|
||||||
|
## Client → server event payloads
|
||||||
|
|
||||||
|
Three send-once messages on the main thread, plus a stream of `input_audio_buffer.append` from UpstreamSendLoop. All written with `JsonSerializer.Serialize` over `Dictionary<string, object?>` (lightest thing that doesn't require typed records for a probe).
|
||||||
|
|
||||||
|
```json
|
||||||
|
// (1) Pinned session config — first thing sent after WS connect.
|
||||||
|
{
|
||||||
|
"type": "session.update",
|
||||||
|
"session": {
|
||||||
|
"modalities": ["audio", "text"],
|
||||||
|
"instructions": "Reply in one short sentence.",
|
||||||
|
"voice": "alloy",
|
||||||
|
"input_audio_format": "pcm16",
|
||||||
|
"output_audio_format": "pcm16",
|
||||||
|
"turn_detection": {
|
||||||
|
"type": "server_vad",
|
||||||
|
"threshold": 0.5,
|
||||||
|
"prefix_padding_ms": 300,
|
||||||
|
"silence_duration_ms": 500,
|
||||||
|
"create_response": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) Per mic frame (~40 ms cadence), from UpstreamSendLoop:
|
||||||
|
{ "type": "input_audio_buffer.append", "audio": "<base64-pcm16-24k>" }
|
||||||
|
```
|
||||||
|
|
||||||
|
No `conversation.item.create` and no `response.create` from the client — server VAD's `create_response: true` makes the server commit and create the response automatically on `speech_stopped`.
|
||||||
|
|
||||||
|
`instructions: "Reply in one short sentence."` keeps the response audio bounded (~2-4 s of speech) so a probe run is fast to verify by ear and cheap on quota.
|
||||||
|
|
||||||
|
`voice: "alloy"` is the long-standing default voice and is documented as universally available for raw WS sessions.
|
||||||
|
|
||||||
|
## Server → client event handlers
|
||||||
|
|
||||||
|
ReceiveLoop uses a 4 KB receive buffer + standard `ReceiveAsync` loop with `EndOfMessage` accumulation into a `MemoryStream`. On message end, `JsonDocument.Parse`. Switch on the top-level `type` property's string value:
|
||||||
|
|
||||||
|
| `type` | Action |
|
||||||
|
|-|-|
|
||||||
|
| `session.created` | log `[ws] session.created <session_id>` |
|
||||||
|
| `session.updated` | log `[ws] session.updated — session pinned` |
|
||||||
|
| `input_audio_buffer.speech_started` | log `[vad] speech started` |
|
||||||
|
| `input_audio_buffer.speech_stopped` | log `[vad] speech stopped`; set `_stopSending = true` |
|
||||||
|
| `input_audio_buffer.committed` | log only — server confirms it sealed the buffer |
|
||||||
|
| `response.created` | log only |
|
||||||
|
| `response.audio_transcript.delta` | append `delta` to an accumulating `StringBuilder` |
|
||||||
|
| `response.audio.delta` | base64-decode `delta` → bytes → reinterpret as `short[]` → `await downstreamChannel.Writer.WriteAsync(samples, cts.Token)`. The `await` is the backpressure path: if the channel is full, this blocks the receive loop, which blocks the WS receive buffer — correct behavior for a slow speaker. |
|
||||||
|
| `response.audio.done` | log only |
|
||||||
|
| `response.done` | set `_noMoreDeltas = true`; `downstreamChannel.Writer.Complete()`; print the accumulated transcript |
|
||||||
|
| `error` | log the full event JSON; `cts.Cancel()` |
|
||||||
|
| anything else | log `[ws] ignored type=<…>` once (or at debug verbosity), continue |
|
||||||
|
|
||||||
|
## Audio buffering — input
|
||||||
|
|
||||||
|
- Mic callback runs every 40 ms (1920 samples @ 24 kHz mono Int16, matches `framesPerBuffer = 1920`).
|
||||||
|
- Callback allocates `short[1920]`, `Buffer.MemoryCopy`s the input pointer, `upstreamChannel.Writer.TryWrite(frame)`.
|
||||||
|
- On TryWrite failure (channel full = UpstreamSendLoop is behind): log `[status] upstream behind, dropping mic frame`, continue.
|
||||||
|
- Discard frames when `_micArmed == false` (during startup beep) or `_stopSending == true` (after `speech_stopped`).
|
||||||
|
- On the `InputOverflow` status flag from PortAudio: log `[status] input overflow`, continue.
|
||||||
|
|
||||||
|
UpstreamSendLoop pseudocode:
|
||||||
|
|
||||||
|
```
|
||||||
|
await ReceiveLoop signals session.updated (or just optimistically loop from start)
|
||||||
|
while (!cts.IsCancellationRequested):
|
||||||
|
if _stopSending && upstreamChannel.Reader.Count == 0: break
|
||||||
|
frame = await upstreamChannel.Reader.ReadAsync(cts.Token)
|
||||||
|
if _stopSending: continue // drop frames captured after server VAD cutoff
|
||||||
|
bytes = MemoryMarshal.AsBytes(frame.AsSpan())
|
||||||
|
base64 = Convert.ToBase64String(bytes)
|
||||||
|
json = JsonSerializer.SerializeToUtf8Bytes(new {
|
||||||
|
type = "input_audio_buffer.append",
|
||||||
|
audio = base64
|
||||||
|
})
|
||||||
|
await ws.SendAsync(json, WebSocketMessageType.Text, endOfMessage: true, cts.Token)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Audio buffering — output
|
||||||
|
|
||||||
|
`downstreamChannel = Channel.CreateBounded<short[]>(new BoundedChannelOptions(64) { FullMode = BoundedChannelFullMode.Wait, SingleWriter = false, SingleReader = true })`. `SingleWriter = false` because the main thread queues the startup beep AND ReceiveLoop queues response audio chunks (temporally separated, but the flag is a hard contract).
|
||||||
|
|
||||||
|
Capacity rationale: 64 chunks × ~40-80 ms per `response.audio.delta` ≈ 2.5-5 s of buffered audio. Comfortable headroom against network jitter without holding silly amounts of memory.
|
||||||
|
|
||||||
|
ReceiveLoop writes `short[]` chunks of varying length (one per `response.audio.delta` event). On `response.done`, calls `downstreamChannel.Writer.Complete()`.
|
||||||
|
|
||||||
|
Output callback's job is to fill `frameCount` shorts every callback (1024 frames = ~43 ms @ 24 kHz). It maintains a `short[]? _currentChunk` + `int _chunkOffset` across calls. Pseudocode:
|
||||||
|
|
||||||
|
```
|
||||||
|
int fill = 0;
|
||||||
|
while (fill < frameCount) {
|
||||||
|
if (_currentChunk == null) {
|
||||||
|
if (!downstreamChannel.Reader.TryRead(out _currentChunk)) break; // drained
|
||||||
|
_chunkOffset = 0;
|
||||||
|
}
|
||||||
|
int slack = frameCount - fill;
|
||||||
|
int remaining = _currentChunk.Length - _chunkOffset;
|
||||||
|
int take = Math.Min(remaining, slack);
|
||||||
|
unsafe {
|
||||||
|
fixed (short* src = &_currentChunk[_chunkOffset])
|
||||||
|
Buffer.MemoryCopy(src, output + fill, take * 2, take * 2);
|
||||||
|
}
|
||||||
|
_chunkOffset += take;
|
||||||
|
fill += take;
|
||||||
|
if (_chunkOffset >= _currentChunk.Length) _currentChunk = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Zero-fill the unfilled tail (underrun mid-response, or post-completion drain).
|
||||||
|
for (int i = fill; i < frameCount; i++) output[i] = 0;
|
||||||
|
|
||||||
|
// Completion check: drained AND writer completed AND no working chunk left.
|
||||||
|
if (_currentChunk == null
|
||||||
|
&& downstreamChannel.Reader.Count == 0
|
||||||
|
&& downstreamChannel.Reader.Completion.IsCompleted)
|
||||||
|
{
|
||||||
|
doneFlag.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Underrun behavior: zero-fill, return Continue.** If `response.done` hasn't arrived yet and the next `response.audio.delta` is just in flight, we play silence for one callback (~43 ms) instead of starving — the stream stays armed, no click on resume.
|
||||||
|
|
||||||
|
**This is the streaming-length cleanup discipline** that replaces Test 3's "scale `done.Wait` timeout to the buffer length" trick. Test 3 knew the playback wall-clock up front (`recordBuf.Length / SampleRate`); Test 4a does not, because the assistant audio length is unknown a priori. So the signaling moves from "wait this many seconds" to "wait until (writer-completed AND channel-empty AND working-chunk-empty)." The output callback is the single oracle for this triple condition.
|
||||||
|
|
||||||
|
**Startup beep wrinkle:** the beep is queued onto `downstreamChannel` BEFORE the receive loop starts producing. The output callback drains it just like any other audio. The completion check is safe because `Writer.Complete()` is only ever called on `response.done` — the beep getting drained does NOT cause an early completion firing.
|
||||||
|
|
||||||
|
## Components and files
|
||||||
|
|
||||||
|
```
|
||||||
|
tests/04a-realtime-oneshot-cs/
|
||||||
|
├── Probe.csproj
|
||||||
|
│ <TargetFramework>net9.0</TargetFramework>
|
||||||
|
│ <RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
│ <SelfContained>true</SelfContained>
|
||||||
|
│ <PublishSingleFile>false</PublishSingleFile>
|
||||||
|
│ <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
│ <InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
│ <Nullable>enable</Nullable>
|
||||||
|
│ <ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
│ RootNamespace=RealtimeOneShotProbe, AssemblyName=Probe
|
||||||
|
│ PackageReference PortAudioSharp2 Version="1.0.6"
|
||||||
|
│ (no Microsoft.ML.OnnxRuntime — no wakeword in 4a)
|
||||||
|
│ System.Net.WebSockets + System.Text.Json come from the BCL; no package.
|
||||||
|
│
|
||||||
|
├── Program.cs ~300 lines, top-level statements:
|
||||||
|
│ 1. Libc.setenv("PA_ALSA_PLUGHW", "1", 1) + Environment mirror
|
||||||
|
│ (no ORT env vars — no onnxruntime in 4a)
|
||||||
|
│ 2. PortAudio.Initialize
|
||||||
|
│ 3. apiKey = LoadApiKey()
|
||||||
|
│ 4. var cts = new CancellationTokenSource();
|
||||||
|
│ Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
│ 5. var ws = new ClientWebSocket();
|
||||||
|
│ ws.Options.SetRequestHeader("Authorization", $"Bearer {apiKey}");
|
||||||
|
│ ws.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
|
||||||
|
│ await ws.ConnectAsync(uri, cts.Token);
|
||||||
|
│ 6. Channel<short[]> upstreamChannel, downstreamChannel (both bounded 64)
|
||||||
|
│ ManualResetEventSlim doneFlag = new(false);
|
||||||
|
│ 7. Task receiveTask = Task.Run(() => ReceiveLoop(ws, downstreamChannel,
|
||||||
|
│ cts, _stopSending, _noMoreDeltas, transcriptBuilder));
|
||||||
|
│ 8. Send session.update (JsonSerializer.SerializeToUtf8Bytes over a dict)
|
||||||
|
│ 9. int device = FindUsbDevice();
|
||||||
|
│ 10. Open + start input stream (24 kHz, 1920 frames/cb)
|
||||||
|
│ 11. Open + start output stream (24 kHz, 1024 frames/cb)
|
||||||
|
│ 12. Task sendTask = Task.Run(() => UpstreamSendLoop(ws, upstreamChannel,
|
||||||
|
│ cts, _stopSending));
|
||||||
|
│ 13. Queue MakeBeep(880, 0.2, 24000) onto downstreamChannel; sleep 300 ms;
|
||||||
|
│ _micArmed = true; Console.WriteLine("Speak now.");
|
||||||
|
│ 14. try { doneFlag.Wait(cts.Token); }
|
||||||
|
│ catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
│ 15. finally:
|
||||||
|
│ ws.CloseAsync(NormalClosure, "done", linkedCtsWith2sTimeout.Token)
|
||||||
|
│ inputStream.Stop; outputStream.Stop;
|
||||||
|
│ inputStream.Dispose; outputStream.Dispose;
|
||||||
|
│ PortAudio.Terminate;
|
||||||
|
│ Helpers (all in Program.cs):
|
||||||
|
│ - LoadApiKey() — reads ~/.openai_key, perms-warns, sanity
|
||||||
|
│ - MicCallback — discards unless _micArmed && !_stopSending
|
||||||
|
│ - OutputCallback — drain-or-zero-fill loop above
|
||||||
|
│ - MakeBeep(hz, sec, rate) — sin wave Int16 (lifted from Test 3, retuned 24 kHz)
|
||||||
|
│ - UpstreamSendLoop(...)
|
||||||
|
│ - ReceiveLoop(...) — JsonDocument switch on "type"
|
||||||
|
│ - FindUsbDevice() — lifted from Test 3 verbatim
|
||||||
|
│ - static class Libc { setenv }
|
||||||
|
│
|
||||||
|
└── (no models/ dir, no WakewordModel.cs)
|
||||||
|
|
||||||
|
bin/probe-cs-4a Copy of bin/probe-cs-3 with substitutions:
|
||||||
|
tests/03-full-cycle-cs → tests/04a-realtime-oneshot-cs
|
||||||
|
~/probe-cs-3 → ~/probe-cs-4a
|
||||||
|
PUBLISH_DIR /tmp/probe-cs-3-out → /tmp/probe-cs-4a-out
|
||||||
|
ssh -t retained (Ctrl-C propagation).
|
||||||
|
Wipe-and-replace publish dir retained (partial-deploy
|
||||||
|
footgun from Test 3).
|
||||||
|
Adds ONE new precondition step before publishing:
|
||||||
|
sshpass -p "$PI_PASS" ssh "$PI_USER@$PI_HOST" \
|
||||||
|
'test -f ~/.openai_key && test -r ~/.openai_key' \
|
||||||
|
|| { echo "Create ~/.openai_key on the Pi (chmod 600),
|
||||||
|
paste your sk-... key into it, then re-run."; \
|
||||||
|
exit 1; }
|
||||||
|
This catches the silent "key missing" failure mode at
|
||||||
|
the deploy step rather than 30 s into the publish + scp.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Concurrency model
|
||||||
|
|
||||||
|
Five concurrent threads of execution in steady state:
|
||||||
|
|
||||||
|
1. **Main thread.** Setup, session.update send, beep + arm, `doneFlag.Wait`, teardown via `finally`.
|
||||||
|
2. **ReceiveLoop** (`Task.Run`). Pulls WS messages, dispatches on `type`. Single writer of `downstreamChannel`, `_stopSending`, `_noMoreDeltas`, transcript builder. Calls `cts.Cancel()` on error.
|
||||||
|
3. **UpstreamSendLoop** (`Task.Run`). Pulls mic frames from `upstreamChannel`, JSON-serializes `input_audio_buffer.append`, calls `ws.SendAsync`. Single writer of the WS send stream.
|
||||||
|
4. **PortAudio input audio thread** (managed by the library). Runs the mic callback every 40 ms. Allocates `short[1920]`, memcpy's the input, `upstreamChannel.Writer.TryWrite(frame)`. Never blocks.
|
||||||
|
5. **PortAudio output audio thread** (managed by the library). Runs the output callback every 43 ms. Drains `downstreamChannel` into the output buffer; zero-fills underruns; sets `doneFlag` on completion. Never blocks.
|
||||||
|
|
||||||
|
Note that `ws.SendAsync` and `ws.ReceiveAsync` can both be active concurrently — `ClientWebSocket` supports one concurrent send + one concurrent receive (but NOT multiple concurrent sends or multiple concurrent receives). Our design uses exactly one of each, so this is safe.
|
||||||
|
|
||||||
|
## Error handling
|
||||||
|
|
||||||
|
| Failure mode | Behavior | Exit |
|
||||||
|
|-|-|-|
|
||||||
|
| `~/.openai_key` missing on Pi | `bin/probe-cs-4a` precondition check exits before publishing. | 1 |
|
||||||
|
| `~/.openai_key` exists but empty / not `sk-`-shaped | `LoadApiKey()` logs `[error] API key file empty` / `[error] API key doesn't start with sk-`, exits. | 2 |
|
||||||
|
| `ws.ConnectAsync` throws (DNS, TLS, 401, 403, 429, 500, network down) | Log `[error] WS connect failed: {ex.Message}` plus inner `HttpRequestException.StatusCode` if present. | 3 |
|
||||||
|
| Server `error` event mid-session | ReceiveLoop logs the full event JSON, `cts.Cancel()`. Main wakes from `doneFlag.Wait(cts.Token)` with `OperationCanceledException`; `finally` runs CloseAsync (no-throw with timeout) + stream.Stop + PortAudio.Terminate. | 4 |
|
||||||
|
| WS drops mid-conversation (`WebSocketException` in ReceiveLoop) | Same as above — log + cancel + cleanup-via-finally. | 4 |
|
||||||
|
| PortAudio input stream `Start()` throws | Log + Dispose stream + cts.Cancel. | 5 |
|
||||||
|
| PortAudio output stream `Start()` throws | Log + Dispose stream + cts.Cancel. | 6 |
|
||||||
|
| Server takes forever and never sends `response.done` | No timeout in 4a. User's recourse is Ctrl-C. (If verification surfaces this, add a configurable timeout in 4b — out of scope for 4a.) | 0 (via Ctrl-C) |
|
||||||
|
| Mic callback can't `TryWrite` to `upstreamChannel` (UpstreamSendLoop falling behind) | Log `[status] upstream behind, dropping mic frame`, continue. (Same shape as Test 3's "consumer behind" log.) | 0 |
|
||||||
|
| Workstation Ctrl-C | `Console.CancelKeyPress` → `e.Cancel = true; cts.Cancel()`. Main wakes from `doneFlag.Wait(cts.Token)` with `OperationCanceledException`. `finally` runs full cleanup path. | 0 |
|
||||||
|
|
||||||
|
The `finally` block at the bottom of `Program.cs`'s outer `try` runs **once** and is the single point where WS is closed, streams are stopped, and PortAudio is terminated. Cleanup is idempotent (`stream.Stop` is a no-op if already stopped; `PortAudio.Terminate` is fine to call once even if some path didn't run).
|
||||||
|
|
||||||
|
## Gotchas carried forward (from findings.md)
|
||||||
|
|
||||||
|
1. **`Libc.setenv("PA_ALSA_PLUGHW", "1", 1)`** before any PortAudio init. `Environment.SetEnvironmentVariable` alone does NOT propagate to libc on Linux. Apply alongside the managed call.
|
||||||
|
2. **No OrtEnv work in 4a** — no `Microsoft.ML.OnnxRuntime` dependency. (4b reintroduces it.)
|
||||||
|
3. **`using Stream = PortAudioSharp.Stream;`** at the top of `Program.cs` to disambiguate from `System.IO.Stream` that `ImplicitUsings` pulls in.
|
||||||
|
4. **`<AllowUnsafeBlocks>true</AllowUnsafeBlocks>`** in `Probe.csproj` for the input/output callbacks' `Buffer.MemoryCopy` over `short*`.
|
||||||
|
5. **Fire-and-forget output stream lifetime.** `stream.Start()` runs synchronously under try/catch BEFORE any auxiliary cleanup task launches; on `Start()` failure, dispose + rethrow without ever leaking the stream. Test 4a uses a *persistent* output stream (opened once, drained for both the beep and the response audio, stopped on shutdown), so there's no per-cycle cleanup task — the `Start()` failure path is the only one that matters.
|
||||||
|
6. **Streaming-length cleanup discipline.** Test 3's "scale `done.Wait(timeout)` to recordBuf length" trick does NOT apply because the assistant audio length is unknown a priori. Replaced by the output callback's `_currentChunk == null && Reader.Count == 0 && Reader.Completion.IsCompleted` triple check. The `ManualResetEventSlim doneFlag` carries the signal to the main thread.
|
||||||
|
7. **`ssh -t` in `bin/probe-cs-4a`** for Ctrl-C propagation from workstation SIGINT.
|
||||||
|
8. **Always wipe-and-replace the whole publish directory.** A partial `scp Probe` after a rebuild leaves the old `.dll` on the target, which the new launcher loads. `bin/probe-cs-4a` does `ssh "rm -rf ~/probe-cs-4a && mkdir ~/probe-cs-4a"` BEFORE the `scp -r` (Test 3 pattern).
|
||||||
|
9. **Startup beep must not be captured by the mic and shipped to OpenAI.** The `_micArmed` flag stays `false` until ~300 ms after the beep is queued to `downstreamChannel` (200 ms beep + 100 ms drain margin). The mic callback discards frames while `!_micArmed`.
|
||||||
|
|
||||||
|
New for 4a (none carried forward from prior tests — these are surfaced here for the first time):
|
||||||
|
|
||||||
|
10. **`ClientWebSocket` concurrency rule:** one concurrent Send + one concurrent Receive is supported; multiple concurrent Sends (or Receives) is not. Our design has exactly one SendLoop and one ReceiveLoop, so this is safe by construction — but if a future change adds a second sender (e.g. a ping task), it must serialize with the existing one.
|
||||||
|
11. **Channel completion semantics:** `Writer.Complete()` makes `Reader.Completion.IsCompleted` true ONLY after all written items are consumed. The output callback's triple check accounts for this correctly. Don't change to a different completion sentinel without re-verifying the corner case.
|
||||||
|
12. **API key file ownership:** the precondition check in `bin/probe-cs-4a` verifies *presence* and *readability* by `pi`, not strict mode 600. The probe itself logs a non-fatal warning if the mode is looser than 600; this lets a slightly looser local setup proceed.
|
||||||
|
|
||||||
|
## Verification (hardware only, no automated tests)
|
||||||
|
|
||||||
|
One-time setup on the Pi (workstation tells you to do this; `bin/probe-cs-4a` precondition-checks it before publishing):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sshpass -p 'assistant' ssh pi@192.168.50.115 \
|
||||||
|
'umask 077 && printf "%s\n" "sk-PASTE-YOUR-KEY-HERE" > ~/.openai_key'
|
||||||
|
```
|
||||||
|
|
||||||
|
Run from workstation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./bin/probe-cs-4a
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected console flow (abridged):
|
||||||
|
|
||||||
|
```
|
||||||
|
>> precondition: ~/.openai_key present on Pi. OK.
|
||||||
|
>> dotnet publish (linux-arm64, self-contained)
|
||||||
|
>> scp to pi@192.168.50.115:~/probe-cs-4a/ (wipe-and-replace)
|
||||||
|
>> ssh + run on Pi (Ctrl-C from this terminal stops the probe)
|
||||||
|
[ws] connecting to wss://api.openai.com/v1/realtime?model=gpt-realtime
|
||||||
|
[ws] connected
|
||||||
|
[ws] session.created sess_xxxxxxxxxxxx
|
||||||
|
[ws] session.updated — session pinned
|
||||||
|
Using device 3 ('USB Speaker Phone (LISTENAI ...)')
|
||||||
|
[beep]
|
||||||
|
Speak now.
|
||||||
|
[vad] speech started
|
||||||
|
[vad] speech stopped
|
||||||
|
[ws] response.created
|
||||||
|
[audio] first delta received (t = … s since speech_stopped)
|
||||||
|
[audio] playback drained
|
||||||
|
[transcript] Hello! How can I help you today?
|
||||||
|
[ws] closing
|
||||||
|
bye
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pass criteria
|
||||||
|
|
||||||
|
All must hold during a single run on the Pi:
|
||||||
|
|
||||||
|
1. **End-to-end round-trip works once.** Probe boots, beeps, you speak a short sentence, the assistant's spoken reply plays back recognisably through the same USB Speaker Phone, the probe exits 0.
|
||||||
|
2. **Response is intelligible.** The assistant's audio is clear speech, no clipping, no dropouts longer than a single 43 ms callback (one zero-fill is fine and expected on the first chunk arrival).
|
||||||
|
3. **Round-trip latency.** Time from `[vad] speech stopped` log to `[audio] first delta received` log (i.e. first response audio byte *arriving* at the probe — actual speaker emission adds a fixed ~43 ms output-callback period on top) is **under ~2 s**. Slower is acceptable if explicitly logged; **over ~5 s is a fail.**
|
||||||
|
4. **No `[status] upstream behind` warnings** during normal operation.
|
||||||
|
5. **No `[error] WS …` events** during normal operation.
|
||||||
|
6. **No PortAudio `InputOverflow` status flags** during normal operation.
|
||||||
|
7. **Workstation Ctrl-C** at any point produces a clean exit: WS closes with normal closure, no stack trace, terminal returns to prompt. Process exit code 0 on Ctrl-C during steady listen; non-zero on Ctrl-C during a failure path is fine.
|
||||||
|
8. **`response.done`-followed-cleanup signaling works.** Probe exits within ~500 ms of the last audio sample being heard. No long hang.
|
||||||
|
|
||||||
|
### What success teaches us for 4b
|
||||||
|
|
||||||
|
- Auth + WS handshake + headers + URL are correct.
|
||||||
|
- `session.update` with `server_vad` works; the server fires `speech_started` / `speech_stopped` correctly on real Pi mic input.
|
||||||
|
- Base64 decode of `response.audio.delta` produces playable PCM16 at 24 kHz on the USB Speaker Phone (PortAudio plug-resamples 48↔24 at ALSA without trouble).
|
||||||
|
- The streaming-length cleanup discipline (`_noMoreDeltas` + `Channel.Completion.IsCompleted` + working-chunk check) is correct.
|
||||||
|
- The `Channel<short[]>` upstream/downstream pattern handles backpressure cleanly.
|
||||||
|
- `bin/probe-cs-4a`'s `~/.openai_key` precondition pattern is the right key-management story (it'll lift verbatim into 4b and the main assistant).
|
||||||
|
|
||||||
|
### What failure would tell us
|
||||||
|
|
||||||
|
- **401 / 403 on connect:** API key is wrong or the file's been mangled.
|
||||||
|
- **WS connects but `session.updated` never arrives:** the session config is malformed for the current Realtime API surface — `modalities`, `turn_detection`, or `voice` is using an outdated field name or value.
|
||||||
|
- **No `speech_stopped` event ever fires:** `turn_detection` config is wrong for the current API (field renamed?), or the mic is silent (gain too low, wrong device, ALSA plug-rate-conversion broken at 24 kHz). Cross-check by adding a periodic RMS log in the mic callback.
|
||||||
|
- **`[status] upstream behind` repeats:** UpstreamSendLoop is slower than the mic callback. Most likely `ws.SendAsync` is blocking on TCP. Cheap remediation: bump channel capacity. Real remediation: investigate the WS backpressure.
|
||||||
|
- **First delta takes >5 s:** server-side issue OR our `instructions` is producing a long thinking pause. Tighten instructions or report and move on.
|
||||||
|
- **Garbled / chipmunk-speed playback:** sample-rate mismatch. Verify the output stream actually opened at 24 kHz (PortAudio might have silently substituted a default).
|
||||||
|
- **Click at the start of playback:** zero-fill underrun ran before the first delta. Acceptable for a probe; if it bothers you, add a small pre-roll buffer.
|
||||||
|
- **Probe hangs on exit:** `Writer.Complete()` was never called (receive loop missed `response.done`), or the output callback's triple check is wrong. Investigate via logging.
|
||||||
|
|
||||||
|
## Reference paths
|
||||||
|
|
||||||
|
- Spec (this doc): `docs/superpowers/specs/2026-06-12-test-4a-openai-realtime-design.md`
|
||||||
|
- Plan (next, to write): `docs/superpowers/plans/2026-06-12-test-4a-openai-realtime.md`
|
||||||
|
- Code (to be written): `tests/04a-realtime-oneshot-cs/`
|
||||||
|
- Deploy (to be written): `bin/probe-cs-4a`
|
||||||
|
- Test 3 reference: `tests/03-full-cycle-cs/Program.cs`, `bin/probe-cs-3`, `docs/superpowers/specs/2026-06-12-test-3-full-cycle-csharp-design.md`
|
||||||
|
- Pi access: `pi@192.168.50.115`, password `assistant` (see `CLAUDE.md`)
|
||||||
|
- API key file location: `/home/pi/.openai_key` on the Pi, mode 600, single line of `sk-…`
|
||||||
|
- Canonical findings: `findings.md` § "C# probe outcome" through § "C# full-cycle outcome"
|
||||||
|
- OpenAI Realtime API surface as of this brainstorm:
|
||||||
|
- WebSocket URL: `wss://api.openai.com/v1/realtime?model=gpt-realtime`
|
||||||
|
- Headers: `Authorization: Bearer <key>`, `OpenAI-Beta: realtime=v1`
|
||||||
|
- Audio formats: `pcm16` (default 24 kHz Int16), `g711_ulaw`, `g711_alaw`
|
||||||
|
- Session config keys used: `modalities`, `instructions`, `voice`, `input_audio_format`, `output_audio_format`, `turn_detection`
|
||||||
|
- `turn_detection.server_vad` subkeys used: `threshold`, `prefix_padding_ms`, `silence_duration_ms`, `create_response`
|
||||||
|
- Server events consumed: `session.created`, `session.updated`, `input_audio_buffer.speech_started`, `input_audio_buffer.speech_stopped`, `input_audio_buffer.committed`, `response.created`, `response.audio_transcript.delta`, `response.audio.delta`, `response.audio.done`, `response.done`, `error`
|
||||||
+183
@@ -130,6 +130,189 @@ Not in scope for prototypes. The main assistant needs a user systemd service so
|
|||||||
5. Local-only fallback when the network is down — required behaviour or out of scope?
|
5. Local-only fallback when the network is down — required behaviour or out of scope?
|
||||||
6. Multi-language? Or English only for v1?
|
6. Multi-language? Or English only for v1?
|
||||||
|
|
||||||
|
## C# probe outcome (2026-06-12 — Test 1 ported to C#)
|
||||||
|
|
||||||
|
The Python Test 1 (record 5 s → write WAV → play back) was re-implemented in C# / .NET 9 to validate that the Pi client can be written in the same language as the backend. The full probe lives at `tests/01-record-play-cs/` with the deploy script at `bin/probe-cs`. All three pass criteria from the original spec held on hardware.
|
||||||
|
|
||||||
|
### What we proved
|
||||||
|
|
||||||
|
- **PortAudioSharp2 1.0.6** is a usable .NET binding for PortAudio. Conventional API surface (`PortAudio.Initialize`, `Stream` ctor taking input/output `StreamParameters`, callback delegate with `ref StreamCallbackTimeInfo`) — matched our assumptions, no adjustments needed.
|
||||||
|
- The NuGet package **bundles `libportaudio.so` for linux-arm64** in its runtime folder, so a self-contained publish doesn't need to rely on the Pi's system `libportaudio2`. (Both are present; the bundled one wins.)
|
||||||
|
- A single `dotnet publish -c Release -r linux-arm64 --self-contained` produces a ~70 MB directory that runs as-is on the Pi. **No .NET install on the Pi.** Deploy is `dotnet publish → scp -r → ssh ./Probe`, wrapped in `bin/probe-cs`.
|
||||||
|
- The `PA_ALSA_PLUGHW=1` trick from the original findings is still required in C# — but with a real twist (see next section).
|
||||||
|
- Recording, RMS metering, WAV writing, and playback all behave identically to the Python prototype. Pass criteria 1 (recognisable playback), 2 (meter responsive), 3 (no over/underflow) all met.
|
||||||
|
|
||||||
|
### Key gotcha — .NET env vars do NOT propagate to libc `getenv()` on Linux
|
||||||
|
|
||||||
|
`Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1")` updates .NET's process-table mirror but does **not** call libc `setenv`. libportaudio reads its config via `getenv("PA_ALSA_PLUGHW")` and sees nothing. ALSA then opens the USB Speaker Phone as `hw:3,0` (native 48 kHz only), and the 16 kHz stream fails with `paInvalidSampleRate`.
|
||||||
|
|
||||||
|
Python's `os.environ[x] = y` calls `setenv` synchronously, which is why the Python probe never tripped on this.
|
||||||
|
|
||||||
|
**Fix:** P/Invoke libc `setenv` directly, alongside the managed call:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
static extern int setenv(string name, string value, int overwrite);
|
||||||
|
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1"); // for .NET-side readers
|
||||||
|
setenv("PA_ALSA_PLUGHW", "1", 1); // for the C runtime view PortAudio reads
|
||||||
|
```
|
||||||
|
|
||||||
|
**Implication for future C# probes and the eventual Pi client:** any env knob a C library reads (PortAudio, onnxruntime, any P/Invoked native) must be set via libc `setenv`, not just `Environment.SetEnvironmentVariable`. Bake a small `Libc.setenv` helper into the eventual client.
|
||||||
|
|
||||||
|
### Small .NET ergonomics
|
||||||
|
|
||||||
|
- `<AllowUnsafeBlocks>true</AllowUnsafeBlocks>` is required in the csproj for the callback's `Buffer.MemoryCopy` over `short*`.
|
||||||
|
- `ImplicitUsings` pulls in `System.IO.Stream`, which collides with `PortAudioSharp.Stream`. Disambiguate with `using Stream = PortAudioSharp.Stream;`.
|
||||||
|
- Top-level statements don't auto-pick up the project's own namespace. Add `using RecordPlayProbe;` (or whatever the project's namespace is) so the entry-point file can reach internal helpers.
|
||||||
|
|
||||||
|
### Open questions deferred to Test 2
|
||||||
|
|
||||||
|
- **No equivalent to openwakeword exists in .NET.** The choices are (a) port the openwakeword inference pipeline to C# using `Microsoft.ML.OnnxRuntime` (mel extractor → Google speech embedder → keyword classifier, three ONNX files), (b) use Picovoice Porcupine's .NET SDK (commercial, key-gated), (c) Vosk + keyword spotting (heavier). Decision deferred to the Test 2 brainstorm.
|
||||||
|
- The `Libc.setenv` workaround will likely be needed again for any onnxruntime env knob on the Pi.
|
||||||
|
|
||||||
|
### Reference paths (C# probe)
|
||||||
|
|
||||||
|
- Spec: `docs/superpowers/specs/2026-06-12-record-play-csharp-probe-design.md`
|
||||||
|
- Plan: `docs/superpowers/plans/2026-06-12-record-play-csharp-probe.md`
|
||||||
|
- Code: `tests/01-record-play-cs/`
|
||||||
|
- Deploy: `bin/probe-cs`
|
||||||
|
|
||||||
|
## C# wakeword probe outcome (2026-06-12 — Test 2 ported to C#)
|
||||||
|
|
||||||
|
The Python Test 2 (openwakeword "alexa" listener with beep on detect) was re-implemented in C# / .NET 9. openwakeword's streaming inference pipeline — three ONNX models chained: mel-spectrogram → Google speech embedding → "alexa" classifier — was ported to `Microsoft.ML.OnnxRuntime`. Probe lives at `tests/02-wakeword-cs/`; deploy script `bin/probe-cs-2`. All three pass criteria from the original spec held on hardware.
|
||||||
|
|
||||||
|
### What we proved
|
||||||
|
|
||||||
|
- **Microsoft.ML.OnnxRuntime 1.26.0** runs on linux-arm64 from a self-contained `dotnet publish`. The NuGet package **bundles** `libonnxruntime.so` and `libonnxruntime_providers_shared.so` for the RID — no separate native package, no fallback needed.
|
||||||
|
- The three openwakeword ONNX models (`melspectrogram.onnx`, `embedding_model.onnx`, `alexa.onnx`, vendored from openwakeword 0.6.0 on the Pi, SHA-256-pinned in commit `849ad47`) load and chain correctly when fed real audio. Predict() loop returned plausible scores from the first run on hardware.
|
||||||
|
- **Two PortAudio streams on one USB device works.** The persistent input stream stays open while a per-detection fire-and-forget output stream plays the beep. `stream.Start()` on the second stream didn't throw, the input callback kept ticking during the 200 ms beep — no input overflow during playback. This fixes the bug `findings.md` § A flagged in the Python probe (where `sd.wait()` stalled input for the beep duration).
|
||||||
|
- Hardware test on the Pi: detection fires reliably on "alexa" at normal volume from ~1 m, audible beep on each. CPU peak **31%** of one core, typical **17%** — well under the 50% budget.
|
||||||
|
|
||||||
|
### Key gotchas
|
||||||
|
|
||||||
|
- **The mel-spectrogram model's output shape was wrong in our initial port.** The plan's verbatim code assumed `(1, n_frames, 1, 32)` based on a memory-derived shape. The actual shape is `(1, 1, n_frames, 32)` — `n_frames` lives at `Dimensions[2]`, not `Dimensions[1]`. Caught during Task 3 implementation by the InputMetadata-vs-derived-shape comparison done in Task 1's sanity spike. Fix is documented in `WakewordModel.cs`. The lesson: always print the actual model metadata on first contact with a vendored ONNX file; do not infer shapes from upstream code without verification.
|
||||||
|
- **ORT 1.26.0's GPU-detection logger initializes BEFORE managed code can set env vars.** `Libc.setenv("ORT_LOGGING_LEVEL", "3", 1)` alone does NOT suppress the `Failed to detect devices under "/sys/class/drm/card0"` warnings — by the time .NET code runs `setenv`, ORT's C++ `device_discovery` already logged. Workaround: explicit `OrtEnv.CreateInstanceWithOptions` with `logLevel = ORT_LOGGING_LEVEL_ERROR` BEFORE constructing any `InferenceSession`. The `Libc.setenv` call stays in place for consistency with the PortAudio pattern but does no work for ORT itself on this version. Bake the `OrtEnv` initialization into the eventual client.
|
||||||
|
- **Fire-and-forget output streams need careful lifetime management.** Initial implementation had `Task.Run` waiting on a `ManualResetEventSlim` BEFORE `stream.Start()` ran. If `Start()` had thrown (the exact failure mode the dual-stream test exists to surface), the cleanup task would have blocked forever on an event that would never fire — silently leaking the stream and hanging the diagnostic. Fix: run `Start()` synchronously under try/catch first, dispose + rethrow on failure, only then launch the cleanup task. Also added a 2 s timeout on `done.Wait()` — PortAudio silently swallows callback exceptions, so without the timeout a faulting callback would hang cleanup the same way.
|
||||||
|
- **Vendored alexa classifier has a static batch dim.** `alexa.onnx` declares input shape `[1, 16, 96]` (not `[-1, 16, 96]` like a typical dynamic-batch model). Other two models use dynamic batch. The shape-assertion code intentionally ignores dim 0 so static-`1` and dynamic-`-1` both pass; the input tensor is created with batch=1 either way.
|
||||||
|
- **Workstation Ctrl-C needs `ssh -t`.** `bin/probe-cs-2` uses `ssh -t` (force pseudo-TTY) so workstation Ctrl-C propagates as SIGINT to the remote `Probe`, triggering the `Console.CancelKeyPress` graceful-shutdown path. Test 1 didn't need this because the probe self-terminated after 5 s; Test 2 listens until interrupted.
|
||||||
|
- **Inference latency was not measured.** Frame size is 1280 samples (80 ms); if `WakewordModel.Predict` ever exceeds 80 ms, the bounded `BlockingCollection` (capacity 16) will start dropping frames with `[status] consumer behind` warnings. No such warnings were observed during the hardware test, but we did not instrument `Predict` timing. Add this when promoting to the main assistant.
|
||||||
|
|
||||||
|
### Open questions for the main assistant
|
||||||
|
|
||||||
|
- **Custom wakeword model.** Stock "alexa" works as a probe, but the shipped assistant needs a custom phrase (e.g. "hey assistant"). openwakeword has a Colab notebook for synthetic training; budget ~1 hour to produce a model that this same C# pipeline can load (it's a drop-in `.onnx` replacement for `alexa.onnx`).
|
||||||
|
- **Speaker-is-mic self-trigger.** The 880 Hz 200 ms beep used here is far enough from speech-band that it doesn't re-trigger the classifier. Real TTS playback will. Spec § B recommends detector gating during playback as the v1 fix; revisit AEC only if barge-in becomes a requirement.
|
||||||
|
- **Inference timing instrumentation.** Add per-`Predict` latency logging in the main assistant so consumer-behind warnings are diagnosable instead of opaque.
|
||||||
|
|
||||||
|
### Reference paths (C# wakeword probe)
|
||||||
|
|
||||||
|
- Spec: `docs/superpowers/specs/2026-06-12-test-2-wakeword-csharp-design.md`
|
||||||
|
- Plan: `docs/superpowers/plans/2026-06-12-test-2-wakeword-csharp.md`
|
||||||
|
- Code: `tests/02-wakeword-cs/`
|
||||||
|
- Deploy: `bin/probe-cs-2`
|
||||||
|
|
||||||
|
## C# full-cycle outcome (2026-06-12 — Test 3)
|
||||||
|
|
||||||
|
Test 3 stitched the C# Test 1 (record + play) and Test 2 (wakeword + beep) into one state machine: idle → "alexa" → beep → record 5 s → playback → idle. Probe lives at `tests/03-full-cycle-cs/`; deploy script `bin/probe-cs-3`. All seven spec pass criteria held on hardware on the first run-after-fix (see "Key gotchas" below).
|
||||||
|
|
||||||
|
### What we proved
|
||||||
|
|
||||||
|
- **The single-input-stream + state-machine + per-cycle-output-streams shape works on this hardware.** One PortAudio input stream open for the process lifetime feeds a `BlockingCollection<short[]>` (bounded 16); a single consumer thread dispatches each 80 ms frame through IDLE / BEEPING / RECORDING / PLAYBACK. No locks, no `Interlocked` — `state` is consumer-thread-local; the only cross-thread datum is a `volatile bool` in a `PlaybackDoneFlag`.
|
||||||
|
- **Detector gating during PLAYBACK prevents self-trigger.** The recorded user audio comes back out the speaker and re-enters the input stream; because the consumer's PLAYBACK branch discards frames (never calls `model.Predict`), the playback never re-triggers detection.
|
||||||
|
- **`WakewordModel.Reset()` on PLAYBACK→IDLE re-arms the warmup guard correctly.** Five lines (`_rawRingFill = 0; _melRing.Clear(); _embRing.Clear(); _framesSeen = 0`) restore the model to the same state it has just after construction; the existing `WarmupFrames = 16` then suppresses the next ~1.3 s of `Predict` calls until the rings refill with fresh post-playback audio. Next "alexa" after playback ends is detected reliably and quickly (within target).
|
||||||
|
- **3+ cycles back-to-back, no degradation.** Detection rate holds, CPU stays in IDLE-state range between cycles, no stuck states.
|
||||||
|
- **No `[status]` or `[error]` warnings during normal operation.** Input doesn't overflow during PLAYBACK (the consumer keeps draining the queue into the discard branch), consumer doesn't fall behind, no fire-and-forget timeouts after the fix below.
|
||||||
|
- **Ctrl-C from the workstation cleanly exits the probe.** Same `ssh -t` + `Console.CancelKeyPress` + `cts.Cancel()` pattern as Test 2.
|
||||||
|
|
||||||
|
### Key gotchas
|
||||||
|
|
||||||
|
- **The fire-and-forget cleanup task's `done.Wait` timeout must scale to the buffer length, not be a flat constant.** Test 2's `FireAndForgetBeep` used `done.Wait(TimeSpan.FromSeconds(2))` to guard against silently-faulted PortAudio callbacks (PortAudio swallows callback exceptions). For a 200 ms beep, 2 s is generous; the cleanup task's wait always returns "set", then sleeps 200 ms and disposes. Test 3's `FireAndForgetPlayback` initially inherited the same flat 2 s timeout — but the recording playback runs for ~5 s, so the timeout fired mid-playback, the cleanup task called `stream.Stop()` 200 ms later, and the audio cut at ~2.2 s. **First hardware run produced "[error] recording playback timed out — callback may have faulted" on every cycle.** Fix: compute `expectedSeconds = buf.Length / SampleRate` and use `TimeSpan.FromSeconds(expectedSeconds + 2.0)` as the cleanup wait. Commit `a7d408e`.
|
||||||
|
- **Partial deploys are silently wrong: a self-contained .NET publish uses `Probe` (native launcher) + `Probe.dll` (managed code).** Copying only `Probe` after a rebuild leaves the old `.dll` on the target, which the new launcher loads — so the binary on the Pi reflects code from the previous build. Always wipe-and-replace the whole publish directory (the pattern already in `bin/probe-cs-3`: `rm -rf ~/probe-cs-3 && mkdir ~/probe-cs-3 && scp -r $PUBLISH_DIR/. pi:~/probe-cs-3/`). Don't do incremental `scp Probe`.
|
||||||
|
- **`PlaybackDoneFlag` should be created per cycle, not reused.** A theoretical late `Set()` from a delayed cleanup `Task.Run` (cycle N) could land on the shared flag while the consumer is in cycle N+1's PLAYBACK, causing a spurious immediate PLAYBACK→IDLE that skips playback. Allocating a fresh `PlaybackDoneFlag()` inside the RECORDING→PLAYBACK transition (and reading with `playbackDone!.Consume()` in PLAYBACK) sends the stale `Set()` to a now-unreachable object. Commit `070cc07`. Not observed in practice but cheap to defend.
|
||||||
|
- **Log the cleanup-task's `done.Wait` timeout return.** Without the log, a silently-faulted PortAudio callback produces "PLAYBACK→IDLE with no audio" with no diagnostic — exactly the failure mode the timeout was supposed to defend against. Both `FireAndForgetBeep` and `FireAndForgetPlayback` now log `[error] ... timed out — callback may have faulted` on `!done.Wait(timeout)`. Commit `070cc07`.
|
||||||
|
|
||||||
|
### Implications for the main assistant
|
||||||
|
|
||||||
|
- The whole shape — persistent input stream, single-consumer state machine, per-cycle output streams, fire-and-forget cleanup with timeout scaled to buffer length — is good to lift directly into the main Pi client. The main assistant adds: Opus encoding on the recorded buffer before sending to the Realtime WebSocket, an Opus decoder feeding the output stream during TTS playback, and replacing the local 5 s recording window with VAD-driven endpointing. None of those replace the state-machine shape; they slot in.
|
||||||
|
- `WakewordModel` should be promoted out of `tests/` into a real source tree (e.g. `src/Audio/Wakeword/`) when the main assistant starts. Test 3's copy is the live version (it has `Reset()`); Test 2's is frozen.
|
||||||
|
- The "always wipe-and-replace the whole publish directory" deploy pattern needs to be the default in the main assistant's deploy script too. A partial `scp` of `Probe` is a footgun every time the .dll changes (i.e. every code change).
|
||||||
|
|
||||||
|
### Reference paths (C# full-cycle probe)
|
||||||
|
|
||||||
|
- Spec: `docs/superpowers/specs/2026-06-12-test-3-full-cycle-csharp-design.md`
|
||||||
|
- Plan: `docs/superpowers/plans/2026-06-12-test-3-full-cycle-csharp.md`
|
||||||
|
- Code: `tests/03-full-cycle-cs/`
|
||||||
|
- Deploy: `bin/probe-cs-3`
|
||||||
|
|
||||||
|
## Test 4a outcome (2026-06-12 — OpenAI Realtime one-shot voice round-trip in C#)
|
||||||
|
|
||||||
|
Test 4a opens a WebSocket to `wss://api.openai.com/v1/realtime?model=gpt-realtime`, plays a startup beep, streams mic audio to the server VAD endpoint, plays the assistant's audio reply back through the same USB Speaker Phone, and exits. Probe lives at `tests/04a-realtime-oneshot-cs/`; deploy script `bin/probe-cs-4a`. All 8 pass criteria met on hardware after a four-round iteration to match the GA Realtime API surface (the spec was written against beta-era docs from openai/openai-realtime-api-beta).
|
||||||
|
|
||||||
|
### What we proved
|
||||||
|
|
||||||
|
- **`ClientWebSocket` + two background `Task.Run` loops (ReceiveLoop, UpstreamSendLoop) + bounded `Channel<short[]>` on each direction** is the right shape on .NET 9 / linux-arm64 for a Realtime API client. No third-party WebSocket lib needed; `System.Text.Json` is fine for event parsing via `JsonDocument`.
|
||||||
|
- **One PortAudio input stream + one persistent output stream at 24 kHz mono Int16, both on the USB Speaker Phone, coexist cleanly.** PortAudio plug-resamples 48↔24 at ALSA on this device. No client-side resampling.
|
||||||
|
- **Server VAD endpointing works out of the box.** `turn_detection: { type: "server_vad", threshold: 0.5, prefix_padding_ms: 300, silence_duration_ms: 500, create_response: true }` (nested under `session.audio.input`) fires `speech_started` / `speech_stopped` correctly on real Pi mic input; server auto-commits the buffer and creates the response.
|
||||||
|
- **Round-trip latency is excellent.** End-of-speech (server VAD `speech_stopped`) to first response audio sample arriving at the probe: **0.44 s** measured on the test "hello" turn. Spec target was under 2 s; we beat it by 4×.
|
||||||
|
- **Streaming-length cleanup discipline works.** Output callback signals `doneFlag` when `CurrentChunk == null && Reader.Count == 0 && NoMoreDeltas` (the `NoMoreDeltas` `volatile bool` is set on `response.done` after `Writer.Complete()`; reading it is cheaper than touching `Reader.Completion`). Probe exits ~immediately after `response.done`.
|
||||||
|
- **`~/.openai_key` precondition pattern in `bin/probe-cs-4a` works.** Pre-publish check via `sshpass ... 'test -f ~/.openai_key && test -r ~/.openai_key'` catches a missing key at the deploy step rather than 30 s into publish + scp. Set with `umask 077 && cat > ~/.openai_key <<KEY` on the Pi.
|
||||||
|
|
||||||
|
### Key gotchas — Realtime API GA migration
|
||||||
|
|
||||||
|
The spec was written against the older beta API surface (the openai/openai-realtime-api-beta library docs, which were the largest indexed source). The GA API rejected every beta-era piece of our session.update payload, surfacing one error per iteration. The full diff (commit `4cfb0d4`):
|
||||||
|
|
||||||
|
1. **No `OpenAI-Beta: realtime=v1` request header.** GA rejects it as `beta_api_shape_disabled`. Only `Authorization: Bearer <key>` is required for GA.
|
||||||
|
2. **`session.update` payload is nested under `session.audio.{input,output}` instead of flat at the top of `session`.** Beta had `session.input_audio_format`, `session.output_audio_format`, `session.voice`, `session.turn_detection`. GA has `session.audio.input.format`, `session.audio.input.turn_detection`, `session.audio.output.format`, `session.audio.output.voice`. Also `session.type = "realtime"` and `session.model = "gpt-realtime"` go inline at the top of `session`.
|
||||||
|
3. **`session.audio.{input,output}.format` is now an OBJECT, not a bare string.** Beta accepted `"pcm16"`. GA requires `{ "type": "audio/pcm", "rate": 24000 }`. The error was `Invalid type for 'session.audio.input.format': expected an object, but got a string instead.`
|
||||||
|
4. **`session.modalities` is no longer accepted.** Server rejected with `Unknown parameter: 'session.modalities'`. Dropped from our payload; server defaults are fine (audio + text output). The GA equivalent for constraining output is likely `output_modalities` (matches Responses API convention) — not needed for the probe.
|
||||||
|
5. **Server events for response audio are renamed `response.audio.*` → `response.output_audio.*`.** Beta had `response.audio.delta` / `.done` / `response.audio_transcript.delta` / `.done`. GA has `response.output_audio.delta` / `.done` / `response.output_audio_transcript.delta` / `.done`. **This is the silent failure mode**: the receive loop's switch default branch quietly logs `[ws] ignored type=…` instead of writing the decoded audio to the output channel. The probe logs "session pinned", VAD fires, response completes, transcript is empty, no audio is heard. Took one round of hardware verification to surface; trivially diagnosed via the `ignored type=…` log lines.
|
||||||
|
6. **GA emits a stream of conversation/response framing events** that don't exist in beta: `conversation.item.added` / `.done`, `response.output_item.added` / `.done`, `response.content_part.added` / `.done`, `rate_limits.updated`. Logged as `[ws] ignored type=…` initially; now silenced in our switch so they don't bury real signal.
|
||||||
|
|
||||||
|
The current Realtime model name `gpt-realtime` is still accepted (the dev docs mention `gpt-realtime-2` as well; we didn't need to swap).
|
||||||
|
|
||||||
|
### Other gotchas
|
||||||
|
|
||||||
|
- **Mid-startup WS error must route through cleanup.** Initial implementation had a single `try { … } finally { … }` around main. When the receive loop processed an error event and called `cts.Cancel()` mid-`SendSessionUpdate`, the `await ws.SendAsync(…, cts.Token)` threw `TaskCanceledException`, which propagated past the finally as an unhandled exception (PortAudio + ws Dispose still ran from finally, but the process exited non-zero with a stack trace). Fix: add `catch (OperationCanceledException) { /* expected on cancel */ } catch (Exception ex) { log; ExitCode = 7 }` between the outer try and finally. Probe now logs the `[error] WS error event:` from the receive loop and exits cleanly. Commit `4cfb0d4`.
|
||||||
|
- **`Environment.Exit(N)` skips the outer `finally` and leaks PortAudio**, but is OK in the LoadApiKey path because it runs before `PortAudio.Initialize()`. The `Environment.ExitCode = N; return;` pattern in WS-connect / stream-Start failure paths preserves the finally correctly.
|
||||||
|
- **CA1416 warning on `File.GetUnixFileMode`** is benign; the call is guarded by `try { … } catch (PlatformNotSupportedException) { }`. The build emits one warning consistently; no action needed.
|
||||||
|
- **ALSA shutdown noise on first run.** Two `alsa_snd_pcm_drop` "failed" lines appeared from PortAudio's cleanup on the *failed* runs (cts.Cancel from an error event tore down both streams mid-init). Did NOT appear on the successful run. Looks like a PortAudio cleanup-on-abort quirk; only present on error paths.
|
||||||
|
|
||||||
|
### Pass criteria
|
||||||
|
|
||||||
|
| # | Criterion | Result |
|
||||||
|
|-|-|-|
|
||||||
|
| 1 | End-to-end round-trip works once | ✅ Probe boots, beeps, user speaks, audible reply, exits 0 |
|
||||||
|
| 2 | Response intelligible | ✅ Clean speech, no clipping, no dropouts |
|
||||||
|
| 3 | Latency < 2 s | ✅ **0.44 s** end-of-speech to first delta |
|
||||||
|
| 4 | No `[status] upstream behind` | ✅ None |
|
||||||
|
| 5 | No `[error] WS …` during normal flow | ✅ None |
|
||||||
|
| 6 | No PortAudio `InputOverflow` | ✅ None |
|
||||||
|
| 7 | Ctrl-C clean exit | ✅ Same cleanup path as the successful run, which exited cleanly |
|
||||||
|
| 8 | Exits within ~500 ms of last audio sample | ✅ `response.done` → `[ws] closing` → `bye` is instant |
|
||||||
|
|
||||||
|
### Implications for Test 4b
|
||||||
|
|
||||||
|
- **GA API surface is now proven.** Test 4b inherits the migrated `SendSessionUpdate` payload, the renamed receive-loop switch cases, the silenced GA framing events, and the outer try/catch shape. Lift them directly.
|
||||||
|
- **The "spec written against beta docs" failure mode is a real risk for any future Realtime API work.** When LLM-fetched docs (context7, training data) disagree with the live server, **only the server is authoritative**. Pattern: send a minimal session.update, observe the error response, iterate. Our four-round iteration took ~10 min total.
|
||||||
|
- **The "[ws] ignored type=…" log line on the default branch is the diagnostic** that surfaced the renames. Keep it (or an equivalent) in 4b's receive loop. Without it, missing audio could have been silently attributed to a thousand other causes.
|
||||||
|
- **Server VAD parameters (`threshold: 0.5`, `silence_duration_ms: 500`, `prefix_padding_ms: 300`) work well on this hardware** — natural-feeling end-of-turn after ~500 ms of silence, no clipped openings of user speech. Lift unchanged into 4b.
|
||||||
|
- **`Environment.Exit` vs `Environment.ExitCode = N; return;`** — the latter is the correct pattern for failure paths that need PortAudio + WS cleanup. 4b will have more failure paths (wakeword model init, audio rate conversion, etc.); use the ExitCode pattern consistently.
|
||||||
|
- **The startup beep pattern (queue beep onto the persistent output channel before `_micArmed = true`)** works — beep does not get captured and shipped to OpenAI. Carry into 4b.
|
||||||
|
- **The 16↔24 kHz coexistence question is the one major thing 4a did not address.** 4a is 24 kHz end-to-end. 4b must deal with the wakeword model requiring 16 kHz input while OpenAI requires 24 kHz pcm16 — either a second input stream, an in-process resampler, or open one stream at 24 kHz and downsample for the wakeword. The spec/plan for 4b will need to pick.
|
||||||
|
|
||||||
|
### Reference paths (Test 4a)
|
||||||
|
|
||||||
|
- Spec: `docs/superpowers/specs/2026-06-12-test-4a-openai-realtime-design.md`
|
||||||
|
- Plan: `docs/superpowers/plans/2026-06-12-test-4a-openai-realtime.md`
|
||||||
|
- Code: `tests/04a-realtime-oneshot-cs/`
|
||||||
|
- Deploy: `bin/probe-cs-4a`
|
||||||
|
- GA Realtime API headers / URL / session.update shape (as discovered against the live server, 2026-06-12):
|
||||||
|
- `wss://api.openai.com/v1/realtime?model=gpt-realtime`
|
||||||
|
- Header `Authorization: Bearer <key>` (NO `OpenAI-Beta`)
|
||||||
|
- `session.update` payload: `{ type: "session.update", session: { type: "realtime", model: "gpt-realtime", instructions: "...", audio: { input: { format: { type: "audio/pcm", rate: 24000 }, turn_detection: { type: "server_vad", threshold, prefix_padding_ms, silence_duration_ms, create_response } }, output: { format: { type: "audio/pcm", rate: 24000 }, voice: "alloy" } } } }`
|
||||||
|
- Server events to handle: `session.created`, `session.updated`, `input_audio_buffer.speech_started`, `input_audio_buffer.speech_stopped`, `input_audio_buffer.committed`, `response.created`, `response.output_audio_transcript.delta`, `response.output_audio.delta`, `response.output_audio.done`, `response.output_audio_transcript.done`, `response.done`, `error`
|
||||||
|
- Server events safe to ignore for a probe: `conversation.item.added`, `conversation.item.done`, `response.output_item.added`, `response.output_item.done`, `response.content_part.added`, `response.content_part.done`, `rate_limits.updated`
|
||||||
|
|
||||||
## Reference paths
|
## Reference paths
|
||||||
|
|
||||||
- Spec: `docs/superpowers/specs/2026-06-11-voice-assistant-prototypes-design.md`
|
- Spec: `docs/superpowers/specs/2026-06-11-voice-assistant-prototypes-design.md`
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
openwakeword==0.6.0
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
sounddevice==0.5.1
|
|
||||||
numpy>=2.0
|
|
||||||
scipy>=1.13
|
|
||||||
onnxruntime>=1.18
|
|
||||||
scikit-learn
|
|
||||||
tqdm
|
|
||||||
requests
|
|
||||||
websockets>=12.0
|
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>RecordPlayProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using RecordPlayProbe;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
const int SampleRate = 16_000;
|
||||||
|
const int Channels = 1;
|
||||||
|
const int DurationS = 5;
|
||||||
|
const uint BlockFrames = 1024;
|
||||||
|
const string OutWav = "out.wav";
|
||||||
|
|
||||||
|
// Tell PortAudio's ALSA host API to wrap devices in `plughw:` so it does
|
||||||
|
// rate/format conversion (USB Speaker Phone does not natively offer 16 kHz).
|
||||||
|
// Environment.SetEnvironmentVariable updates the .NET table but the libc
|
||||||
|
// getenv() view PortAudio actually reads can lag, so call setenv directly too.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine(
|
||||||
|
$"Recording {DurationS}s from device {device} " +
|
||||||
|
$"('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
short[] buf = new short[SampleRate * DurationS];
|
||||||
|
int writeOffset = 0;
|
||||||
|
object gate = new();
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
int remaining = buf.Length - writeOffset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &buf[writeOffset])
|
||||||
|
{
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeOffset += take;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
|
||||||
|
var start = DateTime.UtcNow;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
int written;
|
||||||
|
lock (gate) { written = writeOffset; }
|
||||||
|
if (written >= buf.Length) break;
|
||||||
|
|
||||||
|
double level = Rms(buf, Math.Max(0, written - 1600), written);
|
||||||
|
int bars = (int)(level * 60);
|
||||||
|
double elapsed = (DateTime.UtcNow - start).TotalSeconds;
|
||||||
|
Console.Write($"\r{elapsed,4:0.0}s |{new string('#', bars).PadRight(60)}|");
|
||||||
|
Thread.Sleep(50);
|
||||||
|
}
|
||||||
|
Console.WriteLine();
|
||||||
|
stream.Stop();
|
||||||
|
|
||||||
|
WavWriter.Write(OutWav, buf, SampleRate);
|
||||||
|
Console.WriteLine($"Wrote {OutWav} ({buf.Length} frames @ {SampleRate} Hz)");
|
||||||
|
|
||||||
|
Console.WriteLine($"Playing back through device {device}");
|
||||||
|
|
||||||
|
int playOffset = 0;
|
||||||
|
object playGate = new();
|
||||||
|
var doneEvent = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
lock (playGate)
|
||||||
|
{
|
||||||
|
int remaining = buf.Length - playOffset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
int silence = (int)frameCount - take;
|
||||||
|
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &buf[playOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
playOffset += take;
|
||||||
|
}
|
||||||
|
if (silence > 0)
|
||||||
|
{
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (playOffset >= buf.Length)
|
||||||
|
{
|
||||||
|
doneEvent.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var playStream = new Stream(
|
||||||
|
null, outParams, SampleRate, BlockFrames, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
playStream.Start();
|
||||||
|
doneEvent.Wait();
|
||||||
|
// Give PortAudio a moment to drain the device buffer before Stop().
|
||||||
|
Thread.Sleep(200);
|
||||||
|
playStream.Stop();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static double Rms(short[] data, int from, int to)
|
||||||
|
{
|
||||||
|
if (to <= from) return 0.0;
|
||||||
|
double sumSq = 0.0;
|
||||||
|
for (int i = from; i < to; i++)
|
||||||
|
{
|
||||||
|
double x = data[i] / 32768.0;
|
||||||
|
sumSq += x * x;
|
||||||
|
}
|
||||||
|
return Math.Sqrt(sumSq / (to - from) + 1e-12);
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
namespace RecordPlayProbe;
|
||||||
|
|
||||||
|
internal static class WavWriter
|
||||||
|
{
|
||||||
|
public static void Write(string path, ReadOnlySpan<short> samples, int sampleRate)
|
||||||
|
{
|
||||||
|
const short channels = 1;
|
||||||
|
const short bitsPerSample = 16;
|
||||||
|
const short pcmFormat = 1;
|
||||||
|
|
||||||
|
int byteRate = sampleRate * channels * bitsPerSample / 8;
|
||||||
|
short blockAlign = (short)(channels * bitsPerSample / 8);
|
||||||
|
int dataBytes = samples.Length * sizeof(short);
|
||||||
|
int riffChunkSize = 36 + dataBytes;
|
||||||
|
|
||||||
|
using var fs = new FileStream(path, FileMode.Create, FileAccess.Write);
|
||||||
|
using var bw = new BinaryWriter(fs);
|
||||||
|
|
||||||
|
// RIFF header
|
||||||
|
bw.Write(new[] { (byte)'R', (byte)'I', (byte)'F', (byte)'F' });
|
||||||
|
bw.Write(riffChunkSize);
|
||||||
|
bw.Write(new[] { (byte)'W', (byte)'A', (byte)'V', (byte)'E' });
|
||||||
|
|
||||||
|
// fmt chunk
|
||||||
|
bw.Write(new[] { (byte)'f', (byte)'m', (byte)'t', (byte)' ' });
|
||||||
|
bw.Write(16); // fmt chunk size for PCM
|
||||||
|
bw.Write(pcmFormat);
|
||||||
|
bw.Write(channels);
|
||||||
|
bw.Write(sampleRate);
|
||||||
|
bw.Write(byteRate);
|
||||||
|
bw.Write(blockAlign);
|
||||||
|
bw.Write(bitsPerSample);
|
||||||
|
|
||||||
|
// data chunk
|
||||||
|
bw.Write(new[] { (byte)'d', (byte)'a', (byte)'t', (byte)'a' });
|
||||||
|
bw.Write(dataBytes);
|
||||||
|
foreach (short s in samples)
|
||||||
|
bw.Write(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>WakewordProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.26.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="models/*.onnx">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using WakewordProbe;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
const int SampleRate = WakewordModel.SampleRate;
|
||||||
|
const int Channels = 1;
|
||||||
|
const uint BlockFrames = WakewordModel.FrameSamples; // 1280
|
||||||
|
const float Threshold = 0.5f;
|
||||||
|
const double CooldownSeconds = 1.0;
|
||||||
|
const double BeepHz = 880.0;
|
||||||
|
const double BeepSeconds = 0.2;
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio / onnxruntime (which use getenv()) see the values.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
Environment.SetEnvironmentVariable("ORT_LOGGING_LEVEL", "3");
|
||||||
|
Libc.setenv("ORT_LOGGING_LEVEL", "3", 1);
|
||||||
|
// ORT 1.26.0: the env var does NOT suppress early GPU-discovery warnings from device_discovery.cc.
|
||||||
|
// CreateInstanceWithOptions sets the log level at OrtEnv creation time, before EP discovery runs.
|
||||||
|
var ortEnvOpts = new EnvironmentCreationOptions { logId = "WakewordProbe", logLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR };
|
||||||
|
OrtEnv.CreateInstanceWithOptions(ref ortEnvOpts);
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
WakewordModel? model = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
Console.WriteLine("Loading WakewordModel...");
|
||||||
|
var t0 = DateTime.UtcNow;
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
model = new WakewordModel(
|
||||||
|
Path.Combine(modelsDir, "melspectrogram.onnx"),
|
||||||
|
Path.Combine(modelsDir, "embedding_model.onnx"),
|
||||||
|
Path.Combine(modelsDir, "alexa.onnx"));
|
||||||
|
Console.WriteLine($"Loaded in {(DateTime.UtcNow - t0).TotalSeconds:0.00}s.");
|
||||||
|
|
||||||
|
var queue = new BlockingCollection<short[]>(boundedCapacity: 16);
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (frameCount != BlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected callback frameCount={frameCount} (want {BlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[BlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!queue.TryAdd(frame, 0))
|
||||||
|
Console.Error.WriteLine("[status] consumer behind, dropping frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
short[] beepBuffer = MakeBeep(BeepHz, BeepSeconds, SampleRate);
|
||||||
|
Console.WriteLine("Listening. Say 'alexa'. Ctrl-C to exit.");
|
||||||
|
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
TimeSpan lastTrigger = TimeSpan.FromSeconds(-CooldownSeconds);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
short[] frame = queue.Take(cts.Token);
|
||||||
|
float score = model.Predict(frame);
|
||||||
|
var now = sw.Elapsed;
|
||||||
|
if (score >= Threshold && (now - lastTrigger).TotalSeconds >= CooldownSeconds)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"DETECTED alexa score={score:0.000} t={now.TotalSeconds:0.0}s");
|
||||||
|
lastTrigger = now;
|
||||||
|
FireAndForgetBeep(device, beepBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
stream.Stop();
|
||||||
|
Console.WriteLine("\nbye");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
model?.Dispose();
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetBeep(int device, short[] beepBuffer)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = beepBuffer.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &beepBuffer[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= beepBuffer.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
// 2 s timeout so a failed callback (which PortAudio silently swallows
|
||||||
|
// and would leave 'done' unset) eventually releases the stream + handle
|
||||||
|
// instead of leaking them for the process lifetime.
|
||||||
|
done.Wait(TimeSpan.FromSeconds(2));
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||||
|
|
||||||
|
namespace WakewordProbe;
|
||||||
|
|
||||||
|
// Streaming wakeword inference port of openwakeword 0.6.0's predict pipeline.
|
||||||
|
// References (read alongside this file):
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_features (mel + embedding stages)
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_melspectrogram
|
||||||
|
// openwakeword/utils.py:AudioFeatures._get_embeddings
|
||||||
|
// openwakeword/model.py:Model.predict (classifier stage)
|
||||||
|
internal sealed class WakewordModel : IDisposable
|
||||||
|
{
|
||||||
|
// Audio I/O geometry — per-call input contract.
|
||||||
|
public const int FrameSamples = 1280; // 80 ms @ 16 kHz; each Predict() call.
|
||||||
|
public const int SampleRate = 16_000;
|
||||||
|
|
||||||
|
// Mel model: input is last (FrameSamples + MelContextSamples) raw samples
|
||||||
|
// -- the +480 is openwakeword's `-n_samples-160*3:` slice (3 hops of context).
|
||||||
|
public const int MelContextSamples = 480; // 160 * 3
|
||||||
|
public const int MelInputSamples = FrameSamples + MelContextSamples; // 1760
|
||||||
|
public const int MelBins = 32; // melspectrogram model output dim
|
||||||
|
public const int MelBufferMaxFrames = 970; // 10 * 97 (openwakeword `melspectrogram_max_len`)
|
||||||
|
|
||||||
|
// Embedding model: 76-mel-frame window in, 96-d embedding out.
|
||||||
|
public const int EmbeddingWindowMelFrames = 76;
|
||||||
|
public const int EmbeddingDim = 96;
|
||||||
|
public const int EmbeddingBufferMax = 120; // openwakeword `feature_buffer_max_len`
|
||||||
|
public const string EmbeddingInputName = "input_1"; // openwakeword convention; assert at startup
|
||||||
|
|
||||||
|
// Classifier: 16 embeddings in, scalar score out.
|
||||||
|
public const int ClassifierEmbeddings = 16;
|
||||||
|
|
||||||
|
// Skip the first N Predict() calls — buffer fill-up window.
|
||||||
|
public const int WarmupFrames = ClassifierEmbeddings; // 16 frames ≈ 1.28 s
|
||||||
|
|
||||||
|
private readonly InferenceSession _mel;
|
||||||
|
private readonly InferenceSession _emb;
|
||||||
|
private readonly InferenceSession _cls;
|
||||||
|
|
||||||
|
private readonly string _melInputName; // discovered at startup
|
||||||
|
private readonly string _clsInputName; // discovered at startup
|
||||||
|
|
||||||
|
private readonly short[] _rawRing = new short[MelInputSamples];
|
||||||
|
private int _rawRingFill = 0; // samples buffered (≤ MelInputSamples)
|
||||||
|
|
||||||
|
private readonly List<float[]> _melRing = new(MelBufferMaxFrames); // each entry is a length-32 mel frame
|
||||||
|
private readonly List<float[]> _embRing = new(EmbeddingBufferMax); // each entry is a length-96 embedding
|
||||||
|
|
||||||
|
private int _framesSeen = 0;
|
||||||
|
|
||||||
|
public WakewordModel(string melPath, string embeddingPath, string classifierPath)
|
||||||
|
{
|
||||||
|
var opts = new SessionOptions
|
||||||
|
{
|
||||||
|
IntraOpNumThreads = 1,
|
||||||
|
InterOpNumThreads = 1,
|
||||||
|
LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR,
|
||||||
|
};
|
||||||
|
|
||||||
|
_mel = new InferenceSession(melPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(melPath), _mel);
|
||||||
|
_emb = new InferenceSession(embeddingPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(embeddingPath), _emb);
|
||||||
|
_cls = new InferenceSession(classifierPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(classifierPath), _cls);
|
||||||
|
|
||||||
|
// Discover input names + assert shape geometry.
|
||||||
|
_melInputName = _mel.InputMetadata.Keys.Single();
|
||||||
|
|
||||||
|
AssertEmbeddingShape(_emb); // input_1: [batch, 76, 32, 1], dtype float
|
||||||
|
_clsInputName = _cls.InputMetadata.Keys.Single();
|
||||||
|
AssertClassifierShape(_cls); // [batch, 16, 96], dtype float
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void PrintShapes(string filename, InferenceSession sess)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[wakeword-model] {filename}");
|
||||||
|
foreach (var kv in sess.InputMetadata)
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" input '{kv.Key}': shape=[{string.Join(",", kv.Value.Dimensions)}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
foreach (var kv in sess.OutputMetadata)
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" output '{kv.Key}': shape=[{string.Join(",", kv.Value.Dimensions)}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AssertEmbeddingShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
if (!sess.InputMetadata.TryGetValue(EmbeddingInputName, out var meta))
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input named '{EmbeddingInputName}', got [{string.Join(",", sess.InputMetadata.Keys)}]");
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 76, 32, 1] — batch may be -1 (dynamic).
|
||||||
|
if (d.Length != 4 || d[1] != EmbeddingWindowMelFrames || d[2] != MelBins || d[3] != 1)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input shape [batch,{EmbeddingWindowMelFrames},{MelBins},1], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"embedding_model.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AssertClassifierShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
var inputName = sess.InputMetadata.Keys.Single();
|
||||||
|
var meta = sess.InputMetadata[inputName];
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 16, 96] — batch may be -1.
|
||||||
|
if (d.Length != 3 || d[1] != ClassifierEmbeddings || d[2] != EmbeddingDim)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"alexa.onnx: expected input shape [batch,{ClassifierEmbeddings},{EmbeddingDim}], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"alexa.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
public float Predict(short[] frame1280)
|
||||||
|
{
|
||||||
|
if (frame1280.Length != FrameSamples)
|
||||||
|
throw new ArgumentException($"Expected {FrameSamples} samples, got {frame1280.Length}");
|
||||||
|
|
||||||
|
// 1. Append 1280 new samples to the raw ring (shift older samples down if full).
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
int copyToFront = Math.Min(MelInputSamples - _rawRingFill, FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, _rawRingFill, copyToFront);
|
||||||
|
_rawRingFill += copyToFront;
|
||||||
|
if (copyToFront < FrameSamples)
|
||||||
|
{
|
||||||
|
// Boundary case: ring was partially full and the new frame overshoots
|
||||||
|
// remaining capacity. Fires exactly once during warm-up (typically call 2,
|
||||||
|
// when _rawRingFill = 1280 and the incoming 1280 samples overshoot the
|
||||||
|
// remaining 480 capacity). Shift the older samples left to make room,
|
||||||
|
// then write the leftover at the tail.
|
||||||
|
int leftover = FrameSamples - copyToFront;
|
||||||
|
Array.Copy(_rawRing, leftover, _rawRing, 0, MelInputSamples - leftover);
|
||||||
|
Array.Copy(frame1280, copyToFront, _rawRing, MelInputSamples - leftover, leftover);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Shift older samples left by FrameSamples, then append new at the tail.
|
||||||
|
Array.Copy(_rawRing, FrameSamples, _rawRing, 0, MelInputSamples - FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, MelInputSamples - FrameSamples, FrameSamples);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip everything until we have the full mel-context window primed.
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Mel stage: feed the full _rawRing as float32 (1, MelInputSamples) into mel model.
|
||||||
|
var melInputData = new float[MelInputSamples];
|
||||||
|
for (int i = 0; i < MelInputSamples; i++) melInputData[i] = _rawRing[i]; // int16 → float32, NO normalisation
|
||||||
|
var melInputTensor = new DenseTensor<float>(melInputData, new[] { 1, MelInputSamples });
|
||||||
|
using var melResults = _mel.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_melInputName, melInputTensor)
|
||||||
|
});
|
||||||
|
var melOut = melResults.First().AsTensor<float>(); // shape (1, 1, n_frames, 32)
|
||||||
|
|
||||||
|
// Apply openwakeword's `x / 10 + 2` transform and append each new frame to _melRing.
|
||||||
|
// Mel output shape is (1, 1, n_frames, 32). Note n_frames is at Dimensions[2], not [1].
|
||||||
|
int nFrames = melOut.Dimensions[2];
|
||||||
|
for (int f = 0; f < nFrames; f++)
|
||||||
|
{
|
||||||
|
var bin = new float[MelBins];
|
||||||
|
for (int b = 0; b < MelBins; b++)
|
||||||
|
bin[b] = melOut[0, 0, f, b] / 10f + 2f;
|
||||||
|
_melRing.Add(bin);
|
||||||
|
}
|
||||||
|
if (_melRing.Count > MelBufferMaxFrames)
|
||||||
|
_melRing.RemoveRange(0, _melRing.Count - MelBufferMaxFrames);
|
||||||
|
|
||||||
|
// 3. Embedding stage: need ≥ 76 mel frames; slice the last 76 → (1, 76, 32, 1).
|
||||||
|
if (_melRing.Count < EmbeddingWindowMelFrames)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
var embInputData = new float[EmbeddingWindowMelFrames * MelBins];
|
||||||
|
int startMel = _melRing.Count - EmbeddingWindowMelFrames;
|
||||||
|
for (int f = 0; f < EmbeddingWindowMelFrames; f++)
|
||||||
|
Array.Copy(_melRing[startMel + f], 0, embInputData, f * MelBins, MelBins);
|
||||||
|
var embInputTensor = new DenseTensor<float>(embInputData, new[] { 1, EmbeddingWindowMelFrames, MelBins, 1 });
|
||||||
|
using var embResults = _emb.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(EmbeddingInputName, embInputTensor)
|
||||||
|
});
|
||||||
|
var embOut = embResults.First().AsTensor<float>(); // shape (1, 1, 1, 96)
|
||||||
|
var newEmb = new float[EmbeddingDim];
|
||||||
|
for (int i = 0; i < EmbeddingDim; i++) newEmb[i] = embOut[0, 0, 0, i];
|
||||||
|
_embRing.Add(newEmb);
|
||||||
|
if (_embRing.Count > EmbeddingBufferMax)
|
||||||
|
_embRing.RemoveRange(0, _embRing.Count - EmbeddingBufferMax);
|
||||||
|
|
||||||
|
_framesSeen++;
|
||||||
|
|
||||||
|
// 4. Warm-up + classifier stage.
|
||||||
|
if (_embRing.Count < ClassifierEmbeddings || _framesSeen <= WarmupFrames)
|
||||||
|
return 0f;
|
||||||
|
|
||||||
|
var clsInputData = new float[ClassifierEmbeddings * EmbeddingDim];
|
||||||
|
int startEmb = _embRing.Count - ClassifierEmbeddings;
|
||||||
|
for (int i = 0; i < ClassifierEmbeddings; i++)
|
||||||
|
Array.Copy(_embRing[startEmb + i], 0, clsInputData, i * EmbeddingDim, EmbeddingDim);
|
||||||
|
var clsInputTensor = new DenseTensor<float>(clsInputData, new[] { 1, ClassifierEmbeddings, EmbeddingDim });
|
||||||
|
using var clsResults = _cls.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_clsInputName, clsInputTensor)
|
||||||
|
});
|
||||||
|
var clsOut = clsResults.First().AsTensor<float>(); // shape (1, 1)
|
||||||
|
return clsOut[0, 0];
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_mel.Dispose();
|
||||||
|
_emb.Dispose();
|
||||||
|
_cls.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,24 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>FullCycleProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.26.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="models/*.onnx">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,387 @@
|
|||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using FullCycleProbe;
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
const int SampleRate = WakewordModel.SampleRate; // 16 000
|
||||||
|
const int Channels = 1;
|
||||||
|
const uint BlockFrames = WakewordModel.FrameSamples; // 1280 (80 ms)
|
||||||
|
const float Threshold = 0.5f;
|
||||||
|
const double CooldownSeconds = 1.0;
|
||||||
|
const double BeepHz = 880.0;
|
||||||
|
const double BeepSeconds = 0.2;
|
||||||
|
|
||||||
|
const int BeepingFrames = 3; // 3 * 80 ms = 240 ms (covers 200 ms beep + 40 ms drain/reverb margin)
|
||||||
|
const int RecordingFrames = 63; // 63 * 80 ms = 5.04 s (rounded up from 5 s to avoid partial-frame handling)
|
||||||
|
const int RecordBufSamples = RecordingFrames * (int)BlockFrames; // 80640
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio / onnxruntime (which use getenv()) see the values.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
Environment.SetEnvironmentVariable("ORT_LOGGING_LEVEL", "3");
|
||||||
|
Libc.setenv("ORT_LOGGING_LEVEL", "3", 1);
|
||||||
|
// ORT 1.26.0: the env var does NOT suppress early GPU-discovery warnings from device_discovery.cc.
|
||||||
|
// CreateInstanceWithOptions sets the log level at OrtEnv creation time, before EP discovery runs.
|
||||||
|
var ortEnvOpts = new EnvironmentCreationOptions { logId = "FullCycleProbe", logLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR };
|
||||||
|
OrtEnv.CreateInstanceWithOptions(ref ortEnvOpts);
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
WakewordModel? model = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
Console.WriteLine("Loading WakewordModel...");
|
||||||
|
var t0 = DateTime.UtcNow;
|
||||||
|
string modelsDir = Path.Combine(AppContext.BaseDirectory, "models");
|
||||||
|
model = new WakewordModel(
|
||||||
|
Path.Combine(modelsDir, "melspectrogram.onnx"),
|
||||||
|
Path.Combine(modelsDir, "embedding_model.onnx"),
|
||||||
|
Path.Combine(modelsDir, "alexa.onnx"));
|
||||||
|
Console.WriteLine($"Loaded in {(DateTime.UtcNow - t0).TotalSeconds:0.00}s.");
|
||||||
|
|
||||||
|
var queue = new BlockingCollection<short[]>(boundedCapacity: 16);
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (frameCount != BlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected callback frameCount={frameCount} (want {BlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[BlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!queue.TryAdd(frame, 0))
|
||||||
|
Console.Error.WriteLine("[status] consumer behind, dropping frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
using var stream = new Stream(
|
||||||
|
inParams, null, SampleRate, BlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
stream.Start();
|
||||||
|
short[] beepBuffer = MakeBeep(BeepHz, BeepSeconds, SampleRate);
|
||||||
|
Console.WriteLine("Listening. Say 'alexa'. Ctrl-C to exit.");
|
||||||
|
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
TimeSpan lastTrigger = TimeSpan.FromSeconds(-CooldownSeconds);
|
||||||
|
|
||||||
|
// State machine (single-writer: this consumer thread).
|
||||||
|
State state = State.Idle;
|
||||||
|
int beepFrames = 0;
|
||||||
|
int recordOffset = 0;
|
||||||
|
short[] recordBuf = null!; // assigned on BEEPING→RECORDING transition
|
||||||
|
PlaybackDoneFlag? playbackDone = null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
short[] frame = queue.Take(cts.Token);
|
||||||
|
|
||||||
|
switch (state)
|
||||||
|
{
|
||||||
|
case State.Idle:
|
||||||
|
{
|
||||||
|
float score = model.Predict(frame);
|
||||||
|
var now = sw.Elapsed;
|
||||||
|
if (score >= Threshold && (now - lastTrigger).TotalSeconds >= CooldownSeconds)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"DETECTED alexa score={score:0.000} t={now.TotalSeconds:0.0}s");
|
||||||
|
lastTrigger = now;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
FireAndForgetBeep(device, beepBuffer);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] beep stream failed to start: {ex.Message}");
|
||||||
|
// Stay in IDLE — no cycle to abort yet.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[state] BEEPING");
|
||||||
|
state = State.Beeping;
|
||||||
|
beepFrames = 0;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Beeping:
|
||||||
|
{
|
||||||
|
beepFrames++;
|
||||||
|
if (beepFrames >= BeepingFrames)
|
||||||
|
{
|
||||||
|
Console.WriteLine("[state] RECORDING");
|
||||||
|
state = State.Recording;
|
||||||
|
recordBuf = new short[RecordBufSamples];
|
||||||
|
recordOffset = 0;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Recording:
|
||||||
|
{
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
fixed (short* src = &frame[0])
|
||||||
|
fixed (short* dst = &recordBuf[recordOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst, BlockFrames * sizeof(short), BlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
recordOffset += (int)BlockFrames;
|
||||||
|
if (recordOffset >= RecordBufSamples)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
playbackDone = new PlaybackDoneFlag();
|
||||||
|
FireAndForgetPlayback(device, recordBuf, playbackDone);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] playback stream failed to start: {ex.Message}");
|
||||||
|
// Drop the recording, reset model, return to IDLE.
|
||||||
|
model.Reset();
|
||||||
|
Console.WriteLine("[state] IDLE");
|
||||||
|
state = State.Idle;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[state] PLAYBACK");
|
||||||
|
state = State.Playback;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case State.Playback:
|
||||||
|
{
|
||||||
|
// Discard input during playback (gate the detector).
|
||||||
|
if (playbackDone!.Consume())
|
||||||
|
{
|
||||||
|
model.Reset();
|
||||||
|
Console.WriteLine("[state] IDLE");
|
||||||
|
state = State.Idle;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
stream.Stop();
|
||||||
|
Console.WriteLine("\nbye");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
model?.Dispose();
|
||||||
|
PortAudio.Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetBeep(int device, short[] beepBuffer)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = beepBuffer.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &beepBuffer[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= beepBuffer.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
// 2 s timeout so a failed callback (which PortAudio silently swallows
|
||||||
|
// and would leave 'done' unset) eventually releases the stream + handle
|
||||||
|
// instead of leaking them for the process lifetime.
|
||||||
|
if (!done.Wait(TimeSpan.FromSeconds(2)))
|
||||||
|
Console.Error.WriteLine("[error] beep playback timed out — callback may have faulted");
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static void FireAndForgetPlayback(int device, short[] recordBuf, PlaybackDoneFlag doneFlag)
|
||||||
|
{
|
||||||
|
int offset = 0;
|
||||||
|
var done = new ManualResetEventSlim(false);
|
||||||
|
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = 1,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback playCb = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int remaining = recordBuf.Length - offset;
|
||||||
|
int take = (int)Math.Min(frameCount, (uint)remaining);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
if (take > 0)
|
||||||
|
{
|
||||||
|
fixed (short* src = &recordBuf[offset])
|
||||||
|
Buffer.MemoryCopy(src, dst, take * sizeof(short), take * sizeof(short));
|
||||||
|
offset += take;
|
||||||
|
}
|
||||||
|
for (int i = take; i < frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
if (offset >= recordBuf.Length)
|
||||||
|
{
|
||||||
|
done.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
var stream = new Stream(
|
||||||
|
null, outParams, WakewordModel.SampleRate, 1024, StreamFlags.NoFlag, playCb, IntPtr.Zero);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
stream.Start();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
// Timeout = playback wall-clock + 2 s safety margin. Test 2's beep cleanup uses
|
||||||
|
// a flat 2 s because beep is 200 ms; for a 5 s recording the same value would
|
||||||
|
// fire mid-playback, Stop the stream, and cut the audio.
|
||||||
|
double expectedSeconds = recordBuf.Length / (double)WakewordModel.SampleRate;
|
||||||
|
var waitTimeout = TimeSpan.FromSeconds(expectedSeconds + 2.0);
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
if (!done.Wait(waitTimeout))
|
||||||
|
Console.Error.WriteLine("[error] recording playback timed out — callback may have faulted");
|
||||||
|
Thread.Sleep(200);
|
||||||
|
stream.Stop();
|
||||||
|
stream.Dispose();
|
||||||
|
done.Dispose();
|
||||||
|
doneFlag.Set(); // tell the consumer it's safe to flip PLAYBACK→IDLE
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
enum State { Idle, Beeping, Recording, Playback }
|
||||||
|
|
||||||
|
// Volatile flag set by the playback cleanup task and consumed by the consumer thread.
|
||||||
|
// Wrapped in a tiny class so we can pass it by reference into the helper.
|
||||||
|
sealed class PlaybackDoneFlag
|
||||||
|
{
|
||||||
|
private volatile bool _set;
|
||||||
|
public void Set() => _set = true;
|
||||||
|
public bool Consume()
|
||||||
|
{
|
||||||
|
if (!_set) return false;
|
||||||
|
_set = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||||
|
|
||||||
|
namespace FullCycleProbe;
|
||||||
|
|
||||||
|
// Streaming wakeword inference port of openwakeword 0.6.0's predict pipeline.
|
||||||
|
// References (read alongside this file):
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_features (mel + embedding stages)
|
||||||
|
// openwakeword/utils.py:AudioFeatures._streaming_melspectrogram
|
||||||
|
// openwakeword/utils.py:AudioFeatures._get_embeddings
|
||||||
|
// openwakeword/model.py:Model.predict (classifier stage)
|
||||||
|
internal sealed class WakewordModel : IDisposable
|
||||||
|
{
|
||||||
|
// Audio I/O geometry — per-call input contract.
|
||||||
|
public const int FrameSamples = 1280; // 80 ms @ 16 kHz; each Predict() call.
|
||||||
|
public const int SampleRate = 16_000;
|
||||||
|
|
||||||
|
// Mel model: input is last (FrameSamples + MelContextSamples) raw samples
|
||||||
|
// -- the +480 is openwakeword's `-n_samples-160*3:` slice (3 hops of context).
|
||||||
|
public const int MelContextSamples = 480; // 160 * 3
|
||||||
|
public const int MelInputSamples = FrameSamples + MelContextSamples; // 1760
|
||||||
|
public const int MelBins = 32; // melspectrogram model output dim
|
||||||
|
public const int MelBufferMaxFrames = 970; // 10 * 97 (openwakeword `melspectrogram_max_len`)
|
||||||
|
|
||||||
|
// Embedding model: 76-mel-frame window in, 96-d embedding out.
|
||||||
|
public const int EmbeddingWindowMelFrames = 76;
|
||||||
|
public const int EmbeddingDim = 96;
|
||||||
|
public const int EmbeddingBufferMax = 120; // openwakeword `feature_buffer_max_len`
|
||||||
|
public const string EmbeddingInputName = "input_1"; // openwakeword convention; assert at startup
|
||||||
|
|
||||||
|
// Classifier: 16 embeddings in, scalar score out.
|
||||||
|
public const int ClassifierEmbeddings = 16;
|
||||||
|
|
||||||
|
// Skip the first N Predict() calls — buffer fill-up window.
|
||||||
|
public const int WarmupFrames = ClassifierEmbeddings; // 16 frames ≈ 1.28 s
|
||||||
|
|
||||||
|
private readonly InferenceSession _mel;
|
||||||
|
private readonly InferenceSession _emb;
|
||||||
|
private readonly InferenceSession _cls;
|
||||||
|
|
||||||
|
private readonly string _melInputName; // discovered at startup
|
||||||
|
private readonly string _clsInputName; // discovered at startup
|
||||||
|
|
||||||
|
private readonly short[] _rawRing = new short[MelInputSamples];
|
||||||
|
private int _rawRingFill = 0; // samples buffered (≤ MelInputSamples)
|
||||||
|
|
||||||
|
private readonly List<float[]> _melRing = new(MelBufferMaxFrames); // each entry is a length-32 mel frame
|
||||||
|
private readonly List<float[]> _embRing = new(EmbeddingBufferMax); // each entry is a length-96 embedding
|
||||||
|
|
||||||
|
private int _framesSeen = 0;
|
||||||
|
|
||||||
|
public WakewordModel(string melPath, string embeddingPath, string classifierPath)
|
||||||
|
{
|
||||||
|
var opts = new SessionOptions
|
||||||
|
{
|
||||||
|
IntraOpNumThreads = 1,
|
||||||
|
InterOpNumThreads = 1,
|
||||||
|
LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR,
|
||||||
|
};
|
||||||
|
|
||||||
|
_mel = new InferenceSession(melPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(melPath), _mel);
|
||||||
|
_emb = new InferenceSession(embeddingPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(embeddingPath), _emb);
|
||||||
|
_cls = new InferenceSession(classifierPath, opts);
|
||||||
|
PrintShapes(Path.GetFileName(classifierPath), _cls);
|
||||||
|
|
||||||
|
// Discover input names + assert shape geometry.
|
||||||
|
_melInputName = _mel.InputMetadata.Keys.Single();
|
||||||
|
|
||||||
|
AssertEmbeddingShape(_emb); // input_1: [batch, 76, 32, 1], dtype float
|
||||||
|
_clsInputName = _cls.InputMetadata.Keys.Single();
|
||||||
|
AssertClassifierShape(_cls); // [batch, 16, 96], dtype float
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void PrintShapes(string filename, InferenceSession sess)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[wakeword-model] {filename}");
|
||||||
|
foreach (var kv in sess.InputMetadata)
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" input '{kv.Key}': shape=[{string.Join(",", kv.Value.Dimensions)}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
foreach (var kv in sess.OutputMetadata)
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" output '{kv.Key}': shape=[{string.Join(",", kv.Value.Dimensions)}] dtype={kv.Value.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AssertEmbeddingShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
if (!sess.InputMetadata.TryGetValue(EmbeddingInputName, out var meta))
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input named '{EmbeddingInputName}', got [{string.Join(",", sess.InputMetadata.Keys)}]");
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 76, 32, 1] — batch may be -1 (dynamic).
|
||||||
|
if (d.Length != 4 || d[1] != EmbeddingWindowMelFrames || d[2] != MelBins || d[3] != 1)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"embedding_model.onnx: expected input shape [batch,{EmbeddingWindowMelFrames},{MelBins},1], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"embedding_model.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AssertClassifierShape(InferenceSession sess)
|
||||||
|
{
|
||||||
|
var inputName = sess.InputMetadata.Keys.Single();
|
||||||
|
var meta = sess.InputMetadata[inputName];
|
||||||
|
var d = meta.Dimensions;
|
||||||
|
// Expected: [batch, 16, 96] — batch may be -1.
|
||||||
|
if (d.Length != 3 || d[1] != ClassifierEmbeddings || d[2] != EmbeddingDim)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"alexa.onnx: expected input shape [batch,{ClassifierEmbeddings},{EmbeddingDim}], got [{string.Join(",", d)}]");
|
||||||
|
if (meta.ElementType != typeof(float))
|
||||||
|
throw new InvalidOperationException($"alexa.onnx: expected Single input, got {meta.ElementType.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
public float Predict(short[] frame1280)
|
||||||
|
{
|
||||||
|
if (frame1280.Length != FrameSamples)
|
||||||
|
throw new ArgumentException($"Expected {FrameSamples} samples, got {frame1280.Length}");
|
||||||
|
|
||||||
|
// 1. Append 1280 new samples to the raw ring (shift older samples down if full).
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
int copyToFront = Math.Min(MelInputSamples - _rawRingFill, FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, _rawRingFill, copyToFront);
|
||||||
|
_rawRingFill += copyToFront;
|
||||||
|
if (copyToFront < FrameSamples)
|
||||||
|
{
|
||||||
|
// Boundary case: ring was partially full and the new frame overshoots
|
||||||
|
// remaining capacity. Fires exactly once during warm-up (typically call 2,
|
||||||
|
// when _rawRingFill = 1280 and the incoming 1280 samples overshoot the
|
||||||
|
// remaining 480 capacity). Shift the older samples left to make room,
|
||||||
|
// then write the leftover at the tail.
|
||||||
|
int leftover = FrameSamples - copyToFront;
|
||||||
|
Array.Copy(_rawRing, leftover, _rawRing, 0, MelInputSamples - leftover);
|
||||||
|
Array.Copy(frame1280, copyToFront, _rawRing, MelInputSamples - leftover, leftover);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Shift older samples left by FrameSamples, then append new at the tail.
|
||||||
|
Array.Copy(_rawRing, FrameSamples, _rawRing, 0, MelInputSamples - FrameSamples);
|
||||||
|
Array.Copy(frame1280, 0, _rawRing, MelInputSamples - FrameSamples, FrameSamples);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip everything until we have the full mel-context window primed.
|
||||||
|
if (_rawRingFill < MelInputSamples)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Mel stage: feed the full _rawRing as float32 (1, MelInputSamples) into mel model.
|
||||||
|
var melInputData = new float[MelInputSamples];
|
||||||
|
for (int i = 0; i < MelInputSamples; i++) melInputData[i] = _rawRing[i]; // int16 → float32, NO normalisation
|
||||||
|
var melInputTensor = new DenseTensor<float>(melInputData, new[] { 1, MelInputSamples });
|
||||||
|
using var melResults = _mel.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_melInputName, melInputTensor)
|
||||||
|
});
|
||||||
|
var melOut = melResults.First().AsTensor<float>(); // shape (1, 1, n_frames, 32)
|
||||||
|
|
||||||
|
// Apply openwakeword's `x / 10 + 2` transform and append each new frame to _melRing.
|
||||||
|
// Mel output shape is (1, 1, n_frames, 32). Note n_frames is at Dimensions[2], not [1].
|
||||||
|
int nFrames = melOut.Dimensions[2];
|
||||||
|
for (int f = 0; f < nFrames; f++)
|
||||||
|
{
|
||||||
|
var bin = new float[MelBins];
|
||||||
|
for (int b = 0; b < MelBins; b++)
|
||||||
|
bin[b] = melOut[0, 0, f, b] / 10f + 2f;
|
||||||
|
_melRing.Add(bin);
|
||||||
|
}
|
||||||
|
if (_melRing.Count > MelBufferMaxFrames)
|
||||||
|
_melRing.RemoveRange(0, _melRing.Count - MelBufferMaxFrames);
|
||||||
|
|
||||||
|
// 3. Embedding stage: need ≥ 76 mel frames; slice the last 76 → (1, 76, 32, 1).
|
||||||
|
if (_melRing.Count < EmbeddingWindowMelFrames)
|
||||||
|
{
|
||||||
|
_framesSeen++;
|
||||||
|
return 0f;
|
||||||
|
}
|
||||||
|
var embInputData = new float[EmbeddingWindowMelFrames * MelBins];
|
||||||
|
int startMel = _melRing.Count - EmbeddingWindowMelFrames;
|
||||||
|
for (int f = 0; f < EmbeddingWindowMelFrames; f++)
|
||||||
|
Array.Copy(_melRing[startMel + f], 0, embInputData, f * MelBins, MelBins);
|
||||||
|
var embInputTensor = new DenseTensor<float>(embInputData, new[] { 1, EmbeddingWindowMelFrames, MelBins, 1 });
|
||||||
|
using var embResults = _emb.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(EmbeddingInputName, embInputTensor)
|
||||||
|
});
|
||||||
|
var embOut = embResults.First().AsTensor<float>(); // shape (1, 1, 1, 96)
|
||||||
|
var newEmb = new float[EmbeddingDim];
|
||||||
|
for (int i = 0; i < EmbeddingDim; i++) newEmb[i] = embOut[0, 0, 0, i];
|
||||||
|
_embRing.Add(newEmb);
|
||||||
|
if (_embRing.Count > EmbeddingBufferMax)
|
||||||
|
_embRing.RemoveRange(0, _embRing.Count - EmbeddingBufferMax);
|
||||||
|
|
||||||
|
_framesSeen++;
|
||||||
|
|
||||||
|
// 4. Warm-up + classifier stage.
|
||||||
|
if (_embRing.Count < ClassifierEmbeddings || _framesSeen <= WarmupFrames)
|
||||||
|
return 0f;
|
||||||
|
|
||||||
|
var clsInputData = new float[ClassifierEmbeddings * EmbeddingDim];
|
||||||
|
int startEmb = _embRing.Count - ClassifierEmbeddings;
|
||||||
|
for (int i = 0; i < ClassifierEmbeddings; i++)
|
||||||
|
Array.Copy(_embRing[startEmb + i], 0, clsInputData, i * EmbeddingDim, EmbeddingDim);
|
||||||
|
var clsInputTensor = new DenseTensor<float>(clsInputData, new[] { 1, ClassifierEmbeddings, EmbeddingDim });
|
||||||
|
using var clsResults = _cls.Run(new[] {
|
||||||
|
NamedOnnxValue.CreateFromTensor(_clsInputName, clsInputTensor)
|
||||||
|
});
|
||||||
|
var clsOut = clsResults.First().AsTensor<float>(); // shape (1, 1)
|
||||||
|
return clsOut[0, 0];
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Reset()
|
||||||
|
{
|
||||||
|
_rawRingFill = 0;
|
||||||
|
_melRing.Clear();
|
||||||
|
_embRing.Clear();
|
||||||
|
_framesSeen = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_mel.Dispose();
|
||||||
|
_emb.Dispose();
|
||||||
|
_cls.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,18 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<RootNamespace>RealtimeOneShotProbe</RootNamespace>
|
||||||
|
<AssemblyName>Probe</AssemblyName>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RuntimeIdentifier>linux-arm64</RuntimeIdentifier>
|
||||||
|
<SelfContained>true</SelfContained>
|
||||||
|
<PublishSingleFile>false</PublishSingleFile>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PortAudioSharp2" Version="1.0.6" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,542 @@
|
|||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Threading.Channels;
|
||||||
|
using PortAudioSharp;
|
||||||
|
using Stream = PortAudioSharp.Stream;
|
||||||
|
|
||||||
|
// .NET-side mirror for any readers that go through Environment.GetEnvironmentVariable,
|
||||||
|
// AND libc setenv so PortAudio (which uses getenv()) sees the value.
|
||||||
|
Environment.SetEnvironmentVariable("PA_ALSA_PLUGHW", "1");
|
||||||
|
Libc.setenv("PA_ALSA_PLUGHW", "1", 1);
|
||||||
|
|
||||||
|
string apiKey = LoadApiKey();
|
||||||
|
|
||||||
|
PortAudio.Initialize();
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||||
|
|
||||||
|
var upstreamChannel = Channel.CreateBounded<short[]>(new BoundedChannelOptions(C.UpstreamChannelCapacity)
|
||||||
|
{
|
||||||
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
|
SingleWriter = true,
|
||||||
|
SingleReader = true,
|
||||||
|
});
|
||||||
|
var downstreamChannel = Channel.CreateBounded<short[]>(new BoundedChannelOptions(C.DownstreamChannelCapacity)
|
||||||
|
{
|
||||||
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
|
SingleWriter = false,
|
||||||
|
SingleReader = true,
|
||||||
|
});
|
||||||
|
|
||||||
|
var doneFlag = new ManualResetEventSlim(false);
|
||||||
|
var state = new SharedState();
|
||||||
|
|
||||||
|
var ws = new ClientWebSocket();
|
||||||
|
ws.Options.SetRequestHeader("Authorization", $"Bearer {apiKey}");
|
||||||
|
// No OpenAI-Beta header — the GA Realtime API rejects beta-shaped requests with
|
||||||
|
// "beta_api_shape_disabled". Documented at developers.openai.com/api/docs/guides/realtime-websocket.
|
||||||
|
|
||||||
|
Stream? inputStream = null;
|
||||||
|
Stream? outputStream = null;
|
||||||
|
Task? receiveTask = null;
|
||||||
|
Task? sendTask = null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var uri = new Uri($"wss://api.openai.com/v1/realtime?model={C.ModelName}");
|
||||||
|
Console.WriteLine($"[ws] connecting to {uri}");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await ws.ConnectAsync(uri, cts.Token);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] WS connect failed: {ex.Message}");
|
||||||
|
if (ex.InnerException is HttpRequestException httpEx)
|
||||||
|
Console.Error.WriteLine($"[error] inner: HTTP {(int?)httpEx.StatusCode}");
|
||||||
|
Environment.ExitCode = 3;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Console.WriteLine("[ws] connected");
|
||||||
|
|
||||||
|
receiveTask = Task.Run(() => ReceiveLoop(ws, downstreamChannel, cts, state));
|
||||||
|
|
||||||
|
await SendSessionUpdate(ws, cts.Token);
|
||||||
|
|
||||||
|
int device = FindUsbDevice();
|
||||||
|
Console.WriteLine($"Using device {device} ('{PortAudio.GetDeviceInfo(device).name}')");
|
||||||
|
|
||||||
|
inputStream = OpenInputStream(device, upstreamChannel, state);
|
||||||
|
try { inputStream.Start(); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] input stream start failed: {ex.Message}");
|
||||||
|
Environment.ExitCode = 5;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
outputStream = OpenOutputStream(device, downstreamChannel, state, doneFlag);
|
||||||
|
try { outputStream.Start(); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] output stream start failed: {ex.Message}");
|
||||||
|
Environment.ExitCode = 6;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
sendTask = Task.Run(() => UpstreamSendLoop(ws, upstreamChannel, cts, state));
|
||||||
|
|
||||||
|
// Startup beep — queued onto downstreamChannel before the receive loop produces any audio.
|
||||||
|
short[] beep = MakeBeep(C.BeepHz, C.BeepSeconds, C.SampleRate);
|
||||||
|
Console.WriteLine("[beep]");
|
||||||
|
await downstreamChannel.Writer.WriteAsync(beep, cts.Token);
|
||||||
|
|
||||||
|
// Wait for beep to drain (200 ms beep + 100 ms margin) so it doesn't get captured
|
||||||
|
// by the mic and shipped to OpenAI.
|
||||||
|
Thread.Sleep((int)(C.BeepSeconds * 1000) + C.BeepDrainMs);
|
||||||
|
state.MicArmed = true;
|
||||||
|
Console.WriteLine("Speak now.");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
doneFlag.Wait(cts.Token);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* Ctrl-C path */ }
|
||||||
|
|
||||||
|
Console.WriteLine("[ws] closing");
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
// Mid-startup cancellation (typically a WS error event fired cts.Cancel while
|
||||||
|
// the main thread was awaiting SendSessionUpdate / a channel write).
|
||||||
|
// The receive loop has already logged the underlying [error]; we just route through cleanup.
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] unexpected: {ex.GetType().Name}: {ex.Message}");
|
||||||
|
Environment.ExitCode = 7;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// Cancel any background work first so receive/send loops exit cleanly.
|
||||||
|
cts.Cancel();
|
||||||
|
|
||||||
|
if (ws.State == WebSocketState.Open || ws.State == WebSocketState.CloseReceived)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
|
||||||
|
await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "done", closeTimeout.Token);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[ws] close failed: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try { inputStream?.Stop(); } catch { }
|
||||||
|
try { outputStream?.Stop(); } catch { }
|
||||||
|
try { inputStream?.Dispose(); } catch { }
|
||||||
|
try { outputStream?.Dispose(); } catch { }
|
||||||
|
try { ws.Dispose(); } catch { }
|
||||||
|
try { PortAudio.Terminate(); } catch { }
|
||||||
|
|
||||||
|
Console.WriteLine("bye");
|
||||||
|
}
|
||||||
|
|
||||||
|
static string LoadApiKey()
|
||||||
|
{
|
||||||
|
string home = Environment.GetEnvironmentVariable("HOME") ?? "/root";
|
||||||
|
string path = Path.Combine(home, ".openai_key");
|
||||||
|
if (!File.Exists(path))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key file not found at {path}");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-fatal mode-600 check.
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var mode = File.GetUnixFileMode(path);
|
||||||
|
var leaky = mode & (UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute
|
||||||
|
| UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute);
|
||||||
|
if (leaky != UnixFileMode.None)
|
||||||
|
Console.Error.WriteLine($"[warn] {path} mode includes group/other perms ({mode}); expected 600");
|
||||||
|
}
|
||||||
|
catch (PlatformNotSupportedException) { /* not Unix */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[warn] could not check {path} mode: {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
string key = File.ReadAllText(path).Trim();
|
||||||
|
if (string.IsNullOrEmpty(key))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key file empty");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
if (!key.StartsWith("sk-"))
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] API key doesn't start with sk-");
|
||||||
|
Environment.Exit(2);
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int FindUsbDevice()
|
||||||
|
{
|
||||||
|
int count = PortAudio.DeviceCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
if (info.name.ToLowerInvariant().Contains("usb") && info.maxInputChannels >= 1)
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
Console.Error.WriteLine("USB audio device not found. Devices:");
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var info = PortAudio.GetDeviceInfo(i);
|
||||||
|
Console.Error.WriteLine(
|
||||||
|
$" [{i}] {info.name} in={info.maxInputChannels} out={info.maxOutputChannels}");
|
||||||
|
}
|
||||||
|
Environment.Exit(1);
|
||||||
|
return -1; // unreachable
|
||||||
|
}
|
||||||
|
|
||||||
|
static short[] MakeBeep(double freqHz, double durationS, int sampleRate)
|
||||||
|
{
|
||||||
|
int n = (int)(sampleRate * durationS);
|
||||||
|
var buf = new short[n];
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
{
|
||||||
|
double t = i / (double)sampleRate;
|
||||||
|
double v = 0.3 * Math.Sin(2.0 * Math.PI * freqHz * t);
|
||||||
|
buf[i] = (short)(v * short.MaxValue);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task SendSessionUpdate(ClientWebSocket ws, CancellationToken ct)
|
||||||
|
{
|
||||||
|
// GA Realtime API shape: nested session.audio.{input,output} with format + voice + turn_detection
|
||||||
|
// moved under audio.input. session.type = "realtime" and session.model inline.
|
||||||
|
// (Beta-era flat shape with session.input_audio_format/turn_detection at the top is rejected.)
|
||||||
|
var msg = new
|
||||||
|
{
|
||||||
|
type = "session.update",
|
||||||
|
session = new
|
||||||
|
{
|
||||||
|
type = "realtime",
|
||||||
|
model = C.ModelName,
|
||||||
|
instructions = C.Instructions,
|
||||||
|
// No 'modalities' field — the beta name. GA defaults are fine; if we ever need to
|
||||||
|
// constrain to audio-only we'd use 'output_modalities' (the documented GA name in
|
||||||
|
// newer OpenAI APIs).
|
||||||
|
audio = new
|
||||||
|
{
|
||||||
|
input = new
|
||||||
|
{
|
||||||
|
// GA shape: format is now an object with a "type" field, not a bare string.
|
||||||
|
format = new { type = "audio/pcm", rate = 24_000 },
|
||||||
|
turn_detection = new
|
||||||
|
{
|
||||||
|
type = "server_vad",
|
||||||
|
threshold = 0.5,
|
||||||
|
prefix_padding_ms = 300,
|
||||||
|
silence_duration_ms = 500,
|
||||||
|
create_response = true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
output = new
|
||||||
|
{
|
||||||
|
format = new { type = "audio/pcm", rate = 24_000 },
|
||||||
|
voice = C.Voice,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(msg);
|
||||||
|
await ws.SendAsync(new ArraySegment<byte>(bytes), WebSocketMessageType.Text, endOfMessage: true, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Stream OpenInputStream(int device, Channel<short[]> upstream, SharedState state)
|
||||||
|
{
|
||||||
|
var inParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = C.Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowInputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr input, IntPtr _, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags status, IntPtr _3) =>
|
||||||
|
{
|
||||||
|
if (status.HasFlag(StreamCallbackFlags.InputOverflow))
|
||||||
|
Console.Error.WriteLine("[status] input overflow");
|
||||||
|
|
||||||
|
if (!state.MicArmed || state.StopSending)
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
|
||||||
|
if (frameCount != C.MicBlockFrames)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[status] unexpected input frameCount={frameCount} (want {C.MicBlockFrames})");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var frame = new short[C.MicBlockFrames];
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* src = (short*)input.ToPointer();
|
||||||
|
fixed (short* dst = &frame[0])
|
||||||
|
Buffer.MemoryCopy(src, dst, C.MicBlockFrames * sizeof(short), C.MicBlockFrames * sizeof(short));
|
||||||
|
}
|
||||||
|
if (!upstream.Writer.TryWrite(frame))
|
||||||
|
Console.Error.WriteLine("[status] upstream behind, dropping mic frame");
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Stream(inParams, null, C.SampleRate, C.MicBlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Stream OpenOutputStream(int device, Channel<short[]> downstream, SharedState state, ManualResetEventSlim doneFlag)
|
||||||
|
{
|
||||||
|
var outParams = new StreamParameters
|
||||||
|
{
|
||||||
|
device = device,
|
||||||
|
channelCount = C.Channels,
|
||||||
|
sampleFormat = SampleFormat.Int16,
|
||||||
|
suggestedLatency = PortAudio.GetDeviceInfo(device).defaultLowOutputLatency,
|
||||||
|
hostApiSpecificStreamInfo = IntPtr.Zero,
|
||||||
|
};
|
||||||
|
|
||||||
|
Stream.Callback callback = (IntPtr _, IntPtr output, uint frameCount,
|
||||||
|
ref StreamCallbackTimeInfo _2, StreamCallbackFlags _3, IntPtr _4) =>
|
||||||
|
{
|
||||||
|
int fill = 0;
|
||||||
|
|
||||||
|
while (fill < frameCount)
|
||||||
|
{
|
||||||
|
// Pull next chunk if we have no working one.
|
||||||
|
if (state.CurrentChunk == null)
|
||||||
|
{
|
||||||
|
if (!downstream.Reader.TryRead(out var nextChunk))
|
||||||
|
break; // ring empty
|
||||||
|
state.CurrentChunk = nextChunk;
|
||||||
|
state.ChunkOffset = 0;
|
||||||
|
}
|
||||||
|
// Local non-null alias so the compiler doesn't warn on dereference inside `fixed`.
|
||||||
|
short[] chunk = state.CurrentChunk!;
|
||||||
|
int slack = (int)frameCount - fill;
|
||||||
|
int remaining = chunk.Length - state.ChunkOffset;
|
||||||
|
int take = Math.Min(remaining, slack);
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
fixed (short* src = &chunk[state.ChunkOffset])
|
||||||
|
Buffer.MemoryCopy(src, dst + fill, take * sizeof(short), take * sizeof(short));
|
||||||
|
}
|
||||||
|
state.ChunkOffset += take;
|
||||||
|
fill += take;
|
||||||
|
if (state.ChunkOffset >= chunk.Length)
|
||||||
|
state.CurrentChunk = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe
|
||||||
|
{
|
||||||
|
short* dst = (short*)output.ToPointer();
|
||||||
|
for (int i = fill; i < (int)frameCount; i++) dst[i] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Completion = working chunk drained AND channel empty AND no more deltas expected.
|
||||||
|
// _noMoreDeltas is the spec's "redundant fast path" — set immediately on response.done,
|
||||||
|
// avoids touching downstream.Reader.Completion (a Task allocation per callback).
|
||||||
|
if (state.CurrentChunk == null
|
||||||
|
&& downstream.Reader.Count == 0
|
||||||
|
&& state.NoMoreDeltas)
|
||||||
|
{
|
||||||
|
doneFlag.Set();
|
||||||
|
return StreamCallbackResult.Complete;
|
||||||
|
}
|
||||||
|
return StreamCallbackResult.Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Stream(null, outParams, C.SampleRate, C.OutBlockFrames, StreamFlags.NoFlag, callback, IntPtr.Zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task UpstreamSendLoop(ClientWebSocket ws, Channel<short[]> upstream, CancellationTokenSource cts, SharedState state)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await foreach (var frame in upstream.Reader.ReadAllAsync(cts.Token))
|
||||||
|
{
|
||||||
|
if (state.StopSending)
|
||||||
|
{
|
||||||
|
// After server VAD said the user is done, drop in-flight mic frames and exit
|
||||||
|
// once the channel is drained. Matches the spec's "_stopSending && empty" exit.
|
||||||
|
if (upstream.Reader.Count == 0) break;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ReadOnlySpan<byte> bytes = MemoryMarshal.AsBytes(frame.AsSpan());
|
||||||
|
string base64 = Convert.ToBase64String(bytes);
|
||||||
|
var msg = new { type = "input_audio_buffer.append", audio = base64 };
|
||||||
|
byte[] json = JsonSerializer.SerializeToUtf8Bytes(msg);
|
||||||
|
await ws.SendAsync(new ArraySegment<byte>(json), WebSocketMessageType.Text, endOfMessage: true, cts.Token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* expected on Cancel */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] upstream send: {ex.Message}");
|
||||||
|
cts.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task ReceiveLoop(ClientWebSocket ws, Channel<short[]> downstream, CancellationTokenSource cts, SharedState state)
|
||||||
|
{
|
||||||
|
var transcript = new StringBuilder();
|
||||||
|
var buffer = new byte[4096];
|
||||||
|
bool firstDeltaSeen = false;
|
||||||
|
DateTime speechStoppedTime = DateTime.MinValue;
|
||||||
|
using var msgBuffer = new MemoryStream();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
msgBuffer.SetLength(0);
|
||||||
|
WebSocketReceiveResult result;
|
||||||
|
do
|
||||||
|
{
|
||||||
|
result = await ws.ReceiveAsync(new ArraySegment<byte>(buffer), cts.Token);
|
||||||
|
if (result.MessageType == WebSocketMessageType.Close) return;
|
||||||
|
msgBuffer.Write(buffer, 0, result.Count);
|
||||||
|
} while (!result.EndOfMessage);
|
||||||
|
|
||||||
|
msgBuffer.Position = 0;
|
||||||
|
using var doc = JsonDocument.Parse(msgBuffer);
|
||||||
|
string type = doc.RootElement.GetProperty("type").GetString() ?? "";
|
||||||
|
|
||||||
|
switch (type)
|
||||||
|
{
|
||||||
|
case "session.created":
|
||||||
|
{
|
||||||
|
string? sid = doc.RootElement.TryGetProperty("session", out var sess)
|
||||||
|
&& sess.TryGetProperty("id", out var idEl) ? idEl.GetString() : null;
|
||||||
|
Console.WriteLine($"[ws] session.created {sid}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "session.updated":
|
||||||
|
Console.WriteLine("[ws] session.updated — session pinned");
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.speech_started":
|
||||||
|
Console.WriteLine("[vad] speech started");
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.speech_stopped":
|
||||||
|
Console.WriteLine("[vad] speech stopped");
|
||||||
|
speechStoppedTime = DateTime.UtcNow;
|
||||||
|
state.StopSending = true;
|
||||||
|
break;
|
||||||
|
case "input_audio_buffer.committed":
|
||||||
|
Console.WriteLine("[ws] input_audio_buffer.committed");
|
||||||
|
break;
|
||||||
|
case "response.created":
|
||||||
|
Console.WriteLine("[ws] response.created");
|
||||||
|
break;
|
||||||
|
// GA event names are response.output_audio* (beta was response.audio*).
|
||||||
|
case "response.output_audio_transcript.delta":
|
||||||
|
if (doc.RootElement.TryGetProperty("delta", out var tEl))
|
||||||
|
transcript.Append(tEl.GetString());
|
||||||
|
break;
|
||||||
|
case "response.output_audio.delta":
|
||||||
|
{
|
||||||
|
string b64 = doc.RootElement.GetProperty("delta").GetString() ?? "";
|
||||||
|
byte[] audioBytes = Convert.FromBase64String(b64);
|
||||||
|
short[] samples = new short[audioBytes.Length / 2];
|
||||||
|
Buffer.BlockCopy(audioBytes, 0, samples, 0, audioBytes.Length);
|
||||||
|
if (!firstDeltaSeen)
|
||||||
|
{
|
||||||
|
firstDeltaSeen = true;
|
||||||
|
double elapsed = speechStoppedTime == DateTime.MinValue
|
||||||
|
? -1
|
||||||
|
: (DateTime.UtcNow - speechStoppedTime).TotalSeconds;
|
||||||
|
Console.WriteLine($"[audio] first delta received (t = {elapsed:0.00} s since speech_stopped)");
|
||||||
|
}
|
||||||
|
await downstream.Writer.WriteAsync(samples, cts.Token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "response.output_audio.done":
|
||||||
|
Console.WriteLine("[ws] response.output_audio.done");
|
||||||
|
break;
|
||||||
|
case "response.output_audio_transcript.done":
|
||||||
|
// transcript already accumulated; nothing to do here, server signals end of stream.
|
||||||
|
break;
|
||||||
|
// GA emits a stream of conversation/response framing events we don't need for the
|
||||||
|
// probe — quiet them so they don't bury real signal in the log.
|
||||||
|
case "conversation.item.added":
|
||||||
|
case "conversation.item.done":
|
||||||
|
case "response.output_item.added":
|
||||||
|
case "response.output_item.done":
|
||||||
|
case "response.content_part.added":
|
||||||
|
case "response.content_part.done":
|
||||||
|
case "rate_limits.updated":
|
||||||
|
break;
|
||||||
|
case "response.done":
|
||||||
|
state.NoMoreDeltas = true;
|
||||||
|
downstream.Writer.Complete();
|
||||||
|
Console.WriteLine("[ws] response.done");
|
||||||
|
Console.WriteLine($"[transcript] {transcript}");
|
||||||
|
break;
|
||||||
|
case "error":
|
||||||
|
Console.Error.WriteLine($"[error] WS error event: {doc.RootElement.GetRawText()}");
|
||||||
|
cts.Cancel();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
Console.Error.WriteLine($"[ws] ignored type={type}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { /* expected on Cancel */ }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine($"[error] receive loop: {ex.Message}");
|
||||||
|
cts.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sealed class SharedState
|
||||||
|
{
|
||||||
|
public volatile bool MicArmed;
|
||||||
|
public volatile bool StopSending;
|
||||||
|
public volatile bool NoMoreDeltas;
|
||||||
|
// Output-callback-thread-local; do NOT touch from anywhere else.
|
||||||
|
public short[]? CurrentChunk;
|
||||||
|
public int ChunkOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
static class C
|
||||||
|
{
|
||||||
|
public const int SampleRate = 24_000;
|
||||||
|
public const int Channels = 1;
|
||||||
|
public const uint MicBlockFrames = 1920; // 40 ms @ 24 kHz
|
||||||
|
public const uint OutBlockFrames = 1024; // ~43 ms @ 24 kHz
|
||||||
|
public const string ModelName = "gpt-realtime";
|
||||||
|
public const string Voice = "alloy";
|
||||||
|
public const string Instructions = "Reply in one short sentence.";
|
||||||
|
public const double BeepHz = 880.0;
|
||||||
|
public const double BeepSeconds = 0.2;
|
||||||
|
public const int BeepDrainMs = 100;
|
||||||
|
public const int UpstreamChannelCapacity = 64;
|
||||||
|
public const int DownstreamChannelCapacity = 64;
|
||||||
|
}
|
||||||
|
|
||||||
|
static class Libc
|
||||||
|
{
|
||||||
|
[DllImport("libc", EntryPoint = "setenv")]
|
||||||
|
public static extern int setenv(string name, string value, int overwrite);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user