Sprint 6d — Migrate Gemini Live to @google/genai SDK
feat(geminiLive): rewrite with GoogleGenAI SDK (vertexai: true, apiKey) replaces raw WebSocket to generativelanguage.googleapis.com feat(geminiLive): restore full setup config (systemInstruction, inputAudioTranscription, outputAudioTranscription, VAD) fix(geminiLive): buildSetupFrame → SDK config object (no manual JSON) fix(useT2LiveSession): cancelTokenRef for idempotent startDialogue, closeAllRef for stable unmount cleanup chore: add @google/genai@^1.50.1 dependency test: 11 geminiLive tests rewritten with SDK mock 292/292 backend tests green
This commit is contained in:
parent
d89b0b1e89
commit
0662e766d4
6 changed files with 970 additions and 331 deletions
|
|
@ -1,5 +1,50 @@
|
|||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
// ─── Mock du SDK @google/genai ───────────────────────────────────────────────
|
||||
//
|
||||
// On capture les callbacks passés à `ai.live.connect` pour pouvoir simuler les
|
||||
// événements (onopen, onmessage, onerror, onclose) depuis les tests. La
|
||||
// fabrique `clientFactory` injectée dans openGeminiLiveSession permet de
|
||||
// remplacer `new GoogleGenAI(...)` par un stub.
|
||||
|
||||
interface CapturedConnect {
|
||||
model: string;
|
||||
config: Record<string, unknown>;
|
||||
callbacks: {
|
||||
onopen?: () => void;
|
||||
onmessage?: (msg: unknown) => void;
|
||||
onerror?: (err: unknown) => void;
|
||||
onclose?: (evt: unknown) => void;
|
||||
};
|
||||
session: {
|
||||
sendRealtimeInput: ReturnType<typeof vi.fn>;
|
||||
close: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
}
|
||||
|
||||
let capturedConnect: CapturedConnect | null = null;
|
||||
|
||||
function makeFakeClient() {
|
||||
return {
|
||||
live: {
|
||||
connect: vi.fn(async (params: CapturedConnect) => {
|
||||
const session = {
|
||||
sendRealtimeInput: vi.fn(),
|
||||
close: vi.fn(),
|
||||
};
|
||||
capturedConnect = {
|
||||
model: params.model,
|
||||
config: params.config,
|
||||
callbacks: params.callbacks,
|
||||
session,
|
||||
};
|
||||
return session;
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
import {
|
||||
openGeminiLiveSession,
|
||||
buildT2SystemPrompt,
|
||||
|
|
@ -30,6 +75,33 @@ const SUJET_OPTS = {
|
|||
"Vous cherchez un appartement de 2 pièces dans le centre-ville, votre budget est limité et vous souhaitez emménager le mois prochain.",
|
||||
};
|
||||
|
||||
/** Helper : ouvre une session avec un client mocké et retourne la capture. */
|
||||
async function openWithMock(
|
||||
client: FakeWs,
|
||||
extra: Partial<{
|
||||
onSessionEnd: (transcript: string) => void | Promise<void>;
|
||||
timeoutMs: number;
|
||||
warningMs: number;
|
||||
}> = {},
|
||||
) {
|
||||
capturedConnect = null;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
apiKey: "test-key",
|
||||
clientFactory: () => makeFakeClient() as any,
|
||||
...extra,
|
||||
});
|
||||
// Le `await live.connect()` est dans un `.then()` du code prod ; on laisse
|
||||
// les microtasks se vider avant de retourner la capture.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
if (!capturedConnect) {
|
||||
throw new Error("Le mock du SDK n'a pas capturé de connect()");
|
||||
}
|
||||
return capturedConnect;
|
||||
}
|
||||
|
||||
describe("buildT2SystemPrompt", () => {
|
||||
it("substitue role et contexte dans le template", () => {
|
||||
const prompt = buildT2SystemPrompt(SUJET_OPTS);
|
||||
|
|
@ -42,170 +114,120 @@ describe("buildT2SystemPrompt", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("openGeminiLiveSession", () => {
|
||||
let originalKey: string | undefined;
|
||||
|
||||
describe("openGeminiLiveSession (SDK)", () => {
|
||||
beforeEach(() => {
|
||||
originalKey = process.env.GEMINI_API_KEY;
|
||||
process.env.GEMINI_API_KEY = "test-key";
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalKey === undefined) {
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
} else {
|
||||
process.env.GEMINI_API_KEY = originalKey;
|
||||
}
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
capturedConnect = null;
|
||||
});
|
||||
|
||||
it("envoie le setup frame avec prompt dynamique + VAD + transcriptions", () => {
|
||||
it("appelle live.connect avec le modèle + config Live (audio + system + transcripts + VAD)", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const capture = await openWithMock(client);
|
||||
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
});
|
||||
gemini.emit("open");
|
||||
|
||||
expect(gemini.sent).toHaveLength(1);
|
||||
const setup = JSON.parse(gemini.sent[0] as string);
|
||||
expect(setup.setup.model).toMatch(/gemini/);
|
||||
expect(setup.setup.systemInstruction.parts[0].text).toContain(
|
||||
expect(capture.model).toMatch(/gemini/);
|
||||
const config = capture.config;
|
||||
expect(config.responseModalities).toContain("AUDIO");
|
||||
expect(config.systemInstruction).toContain(
|
||||
"un bailleur qui propose un appartement",
|
||||
);
|
||||
expect(setup.setup.generationConfig.responseModalities).toContain("AUDIO");
|
||||
expect(setup.setup.inputAudioTranscription).toEqual({});
|
||||
expect(setup.setup.outputAudioTranscription).toEqual({});
|
||||
expect(
|
||||
setup.setup.realtimeInputConfig.automaticActivityDetection,
|
||||
).toMatchObject({
|
||||
disabled: false,
|
||||
startOfSpeechSensitivity: "START_SENSITIVITY_LOW",
|
||||
endOfSpeechSensitivity: "END_SENSITIVITY_LOW",
|
||||
silenceDurationMs: 2000,
|
||||
expect(config.inputAudioTranscription).toEqual({});
|
||||
expect(config.outputAudioTranscription).toEqual({});
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const vad: any = (config.realtimeInputConfig as any)
|
||||
?.automaticActivityDetection;
|
||||
expect(vad?.disabled).toBe(false);
|
||||
expect(vad?.silenceDurationMs).toBe(2000);
|
||||
});
|
||||
|
||||
it("forwarde un chunk audio client {type:'audio'} via session.sendRealtimeInput (PCM 16k base64)", async () => {
|
||||
const client = new FakeWs();
|
||||
const capture = await openWithMock(client);
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
const base64 = "AQIDBA=="; // base64 de [1,2,3,4]
|
||||
client.emit("message", JSON.stringify({ type: "audio", data: base64 }));
|
||||
|
||||
expect(capture.session.sendRealtimeInput).toHaveBeenCalledTimes(1);
|
||||
expect(capture.session.sendRealtimeInput).toHaveBeenCalledWith({
|
||||
audio: { data: base64, mimeType: "audio/pcm;rate=16000" },
|
||||
});
|
||||
});
|
||||
|
||||
it("forwarde un chunk audio client (Buffer) vers Gemini", () => {
|
||||
it("forwarde un message Gemini (audio inlineData) au client en JSON", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client);
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
const audioChunk = Buffer.from([0x01, 0x02, 0x03, 0x04]);
|
||||
client.emit("message", audioChunk);
|
||||
const geminiMsg = {
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: { data: "EAYE", mimeType: "audio/pcm;rate=24000" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
capture.callbacks.onmessage?.(geminiMsg);
|
||||
|
||||
// [0] = setup, [1] = chunk audio
|
||||
expect(gemini.sent).toHaveLength(2);
|
||||
expect(gemini.sent[1]).toBe(audioChunk);
|
||||
});
|
||||
|
||||
it("forwarde un chunk audio Gemini (Buffer non-JSON) vers le client sans accumuler de transcript", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const onSessionEnd = vi.fn();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
onSessionEnd,
|
||||
});
|
||||
gemini.emit("open");
|
||||
|
||||
const examinerAudio = Buffer.from([0x10, 0x20, 0x30]);
|
||||
gemini.emit("message", examinerAudio);
|
||||
expect(client.sent).toHaveLength(1);
|
||||
expect(client.sent[0]).toBe(examinerAudio);
|
||||
|
||||
// Fin de session via signal client → transcript vide
|
||||
client.emit("message", JSON.stringify({ type: "end" }));
|
||||
await vi.runAllTimersAsync();
|
||||
expect(onSessionEnd).toHaveBeenCalledWith("");
|
||||
expect(JSON.parse(client.sent[0] as string)).toEqual(geminiMsg);
|
||||
});
|
||||
|
||||
it("accumule inputTranscription et outputTranscription depuis Gemini", async () => {
|
||||
it("accumule input/outputTranscription et reconstruit le transcript chronologique", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const onSessionEnd = vi.fn();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
onSessionEnd,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client, { onSessionEnd });
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
gemini.emit(
|
||||
"message",
|
||||
JSON.stringify({
|
||||
serverContent: {
|
||||
inputTranscription: { text: "Bonjour, je voudrais louer." },
|
||||
},
|
||||
}),
|
||||
);
|
||||
gemini.emit(
|
||||
"message",
|
||||
JSON.stringify({
|
||||
serverContent: {
|
||||
outputTranscription: { text: "Bonjour, c’est pour quel quartier ?" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
gemini.emit(
|
||||
"message",
|
||||
JSON.stringify({
|
||||
serverContent: {
|
||||
inputTranscription: { text: "Le centre-ville." },
|
||||
},
|
||||
}),
|
||||
);
|
||||
capture.callbacks.onmessage?.({
|
||||
serverContent: {
|
||||
inputTranscription: { text: "Bonjour, je voudrais louer." },
|
||||
},
|
||||
});
|
||||
capture.callbacks.onmessage?.({
|
||||
serverContent: {
|
||||
outputTranscription: { text: "Bonjour, c’est pour quel quartier ?" },
|
||||
},
|
||||
});
|
||||
capture.callbacks.onmessage?.({
|
||||
serverContent: { inputTranscription: { text: "Le centre-ville." } },
|
||||
});
|
||||
|
||||
client.emit("message", JSON.stringify({ type: "end" }));
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(onSessionEnd).toHaveBeenCalledTimes(1);
|
||||
const transcript = onSessionEnd.mock.calls[0][0] as string;
|
||||
expect(transcript).toBe(
|
||||
expect(onSessionEnd.mock.calls[0][0]).toBe(
|
||||
"Candidat : Bonjour, je voudrais louer.\nExaminateur : Bonjour, c’est pour quel quartier ?\nCandidat : Le centre-ville.",
|
||||
);
|
||||
});
|
||||
|
||||
it("ferme Gemini après onSessionEnd, sans fermer le client (réservé à l’appelant)", async () => {
|
||||
it("ferme la session SDK après onSessionEnd, sans fermer le client", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const onSessionEnd = vi.fn();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
onSessionEnd,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client, { onSessionEnd });
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
client.emit("message", JSON.stringify({ type: "end" }));
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(gemini.closed).toBe(true);
|
||||
expect(gemini.closeCode).toBe(1000);
|
||||
expect(capture.session.close).toHaveBeenCalledTimes(1);
|
||||
expect(client.closed).toBe(false);
|
||||
});
|
||||
|
||||
it("warning à 180 s puis timeout à 210 s déclenche endSession", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const onSessionEnd = vi.fn();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
onSessionEnd,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client, { onSessionEnd });
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
// Avancer à 180 s → warning au client
|
||||
await vi.advanceTimersByTimeAsync(180_000);
|
||||
const warningFrame = client.sent.find(
|
||||
(f) => typeof f === "string" && f.includes('"warning"'),
|
||||
|
|
@ -217,22 +239,16 @@ describe("openGeminiLiveSession", () => {
|
|||
});
|
||||
expect(onSessionEnd).not.toHaveBeenCalled();
|
||||
|
||||
// Avancer à 210 s total → timeout déclenche endSession
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(onSessionEnd).toHaveBeenCalledTimes(1);
|
||||
expect(gemini.closed).toBe(true);
|
||||
expect(capture.session.close).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("signal end client déclenche endSession une seule fois (idempotent)", async () => {
|
||||
it("signal end client est idempotent (un seul onSessionEnd)", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
const onSessionEnd = vi.fn();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
onSessionEnd,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client, { onSessionEnd });
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
client.emit("message", JSON.stringify({ type: "end" }));
|
||||
client.emit("message", JSON.stringify({ type: "end" }));
|
||||
|
|
@ -241,47 +257,47 @@ describe("openGeminiLiveSession", () => {
|
|||
expect(onSessionEnd).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fermeture Gemini avant fin → close client 4006 GEMINI_DISCONNECTED", () => {
|
||||
it("onclose SDK avant fin → close client 4006 GEMINI_DISCONNECTED", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client);
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
gemini.emit("close");
|
||||
capture.callbacks.onclose?.({ code: 1000 });
|
||||
|
||||
expect(client.closed).toBe(true);
|
||||
expect(client.closeCode).toBe(4006);
|
||||
expect(client.closeReason).toBe("GEMINI_DISCONNECTED");
|
||||
});
|
||||
|
||||
it("erreur Gemini → close client 4006 GEMINI_DISCONNECTED", () => {
|
||||
it("onerror SDK → close client 4006", async () => {
|
||||
const client = new FakeWs();
|
||||
const gemini = new FakeWs();
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
geminiFactory: () => gemini,
|
||||
});
|
||||
gemini.emit("open");
|
||||
const capture = await openWithMock(client);
|
||||
capture.callbacks.onopen?.();
|
||||
|
||||
gemini.emit("error", new Error("boom"));
|
||||
capture.callbacks.onerror?.(new Error("boom"));
|
||||
|
||||
expect(client.closed).toBe(true);
|
||||
expect(client.closeCode).toBe(4006);
|
||||
});
|
||||
|
||||
it("absence de GEMINI_API_KEY → close client 4005 GEMINI_CONFIG sans appel à la factory", () => {
|
||||
it("absence de GEMINI_API_KEY → close client 4005 GEMINI_CONFIG sans appel à live.connect", () => {
|
||||
const originalKey = process.env.GEMINI_API_KEY;
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
capturedConnect = null;
|
||||
const client = new FakeWs();
|
||||
const factory = vi.fn(() => new FakeWs());
|
||||
const factory = vi.fn(() => makeFakeClient());
|
||||
|
||||
openGeminiLiveSession(client, { ...SUJET_OPTS, geminiFactory: factory });
|
||||
openGeminiLiveSession(client, {
|
||||
...SUJET_OPTS,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
clientFactory: factory as any,
|
||||
});
|
||||
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
expect(client.closed).toBe(true);
|
||||
expect(client.closeCode).toBe(4005);
|
||||
expect(client.closeReason).toBe("GEMINI_CONFIG");
|
||||
|
||||
if (originalKey !== undefined) process.env.GEMINI_API_KEY = originalKey;
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,9 +1,38 @@
|
|||
import { WebSocket as NodeWebSocket } from "ws";
|
||||
/**
|
||||
* geminiLive.ts — Sprint 6d.
|
||||
*
|
||||
* Migration du WebSocket brut (`wss://generativelanguage.googleapis.com/...`)
|
||||
* vers le SDK officiel `@google/genai` v1.50.x. Motif : Google a migré les
|
||||
* clés API vers le mode "Vertex AI Express", incompatible avec l'endpoint WS
|
||||
* historique (réponse 403 systématique). Le SDK gère l'auth automatiquement
|
||||
* et accepte les clés Express bound à un service account.
|
||||
*
|
||||
* Interface publique (consommée par `routes/t2live.ts`) :
|
||||
* - openGeminiLiveSession(clientWs, opts) : ouvre une session Live et
|
||||
* proxifie les messages dans les deux sens entre le client (navigateur)
|
||||
* et Gemini, accumule les transcripts, gère timeouts + close codes.
|
||||
* - WebSocketLike : interface minimale pour le client WS (Hono adapter).
|
||||
* - buildT2SystemPrompt({role, contexte}) : prompt dynamique T2 Live.
|
||||
* - GEMINI_LIVE_MODEL, T2_SESSION_TIMEOUT_MS, T2_SESSION_WARNING_MS.
|
||||
*
|
||||
* Cf. docs/IMPLEMENTATION_T2_LIVE.md §3, docs/Prompt_t2live.md §3.
|
||||
*/
|
||||
|
||||
export const GEMINI_LIVE_URL =
|
||||
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
||||
import {
|
||||
GoogleGenAI,
|
||||
Modality,
|
||||
StartSensitivity,
|
||||
EndSensitivity,
|
||||
type Session,
|
||||
} from "@google/genai";
|
||||
|
||||
export const GEMINI_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-latest";
|
||||
/**
|
||||
* Modèle Live cible. `gemini-3.1-flash-live-preview` est le choix par défaut
|
||||
* (Sprint 6d), à valider sur Express Mode via `test-gemini-live.js`. Fallback
|
||||
* documenté : `gemini-2.0-flash-live-001` (modèle Live garanti sur Express
|
||||
* d'après la doc Vertex Express).
|
||||
*/
|
||||
export const GEMINI_LIVE_MODEL = "gemini-3.1-flash-live-preview";
|
||||
|
||||
/** Timeout total session WS T2 Live : 3 min 30 (durée TCF) + marge évaluation. */
|
||||
export const T2_SESSION_TIMEOUT_MS = 210_000;
|
||||
|
|
@ -36,7 +65,6 @@ Règles à respecter impérativement :
|
|||
/**
|
||||
* Subset minimal d'une WebSocket — compatible avec :
|
||||
* - le wrapper exposé par @hono/node-ws (côté client navigateur)
|
||||
* - la WebSocket de `ws` (côté Gemini)
|
||||
* - les fakes basés sur EventEmitter dans les tests
|
||||
*/
|
||||
export interface WebSocketLike {
|
||||
|
|
@ -59,34 +87,32 @@ export interface OpenGeminiLiveSessionOptions {
|
|||
timeoutMs?: number;
|
||||
/** Override warning (par défaut T2_SESSION_WARNING_MS). */
|
||||
warningMs?: number;
|
||||
/** Injection pour les tests — fabrique de WebSocket vers Gemini. */
|
||||
geminiFactory?: (url: string) => WebSocketLike;
|
||||
/** Surcharge la clé API (par défaut : process.env.GEMINI_API_KEY). */
|
||||
apiKey?: string;
|
||||
/**
|
||||
* Injection pour les tests — fabrique de client SDK. Permet de remplacer
|
||||
* `new GoogleGenAI(...)` par un mock dans les tests sans toucher au code prod.
|
||||
*/
|
||||
clientFactory?: (apiKey: string) => GoogleGenAI;
|
||||
}
|
||||
|
||||
function buildSetupFrame(systemPrompt: string): string {
|
||||
return JSON.stringify({
|
||||
setup: {
|
||||
model: GEMINI_LIVE_MODEL,
|
||||
systemInstruction: {
|
||||
parts: [{ text: systemPrompt }],
|
||||
},
|
||||
generationConfig: {
|
||||
responseModalities: ["AUDIO"],
|
||||
},
|
||||
inputAudioTranscription: {},
|
||||
outputAudioTranscription: {},
|
||||
realtimeInputConfig: {
|
||||
automaticActivityDetection: {
|
||||
disabled: false,
|
||||
startOfSpeechSensitivity: "START_SENSITIVITY_LOW",
|
||||
endOfSpeechSensitivity: "END_SENSITIVITY_LOW",
|
||||
silenceDurationMs: 2000,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
/**
|
||||
* Forme minimale d'un message Live retourné par le SDK. On n'exporte pas
|
||||
* `LiveServerMessage` du SDK pour ne pas coupler les tests à son shape exact.
|
||||
*/
|
||||
interface LiveServerMessage {
|
||||
serverContent?: {
|
||||
modelTurn?: {
|
||||
parts?: Array<{
|
||||
inlineData?: { data?: string; mimeType?: string };
|
||||
}>;
|
||||
};
|
||||
inputTranscription?: { text?: string };
|
||||
outputTranscription?: { text?: string };
|
||||
interrupted?: boolean;
|
||||
turnComplete?: boolean;
|
||||
};
|
||||
setupComplete?: unknown;
|
||||
}
|
||||
|
||||
interface TranscriptEntry {
|
||||
|
|
@ -104,54 +130,6 @@ function reconstructTranscript(entries: TranscriptEntry[]): string {
|
|||
.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Tente de parser un message Gemini en JSON pour en extraire les transcripts.
|
||||
* Retourne null si non-JSON (chunks audio binaires).
|
||||
*/
|
||||
function tryParseGeminiMessage(data: unknown): {
|
||||
inputText?: string;
|
||||
outputText?: string;
|
||||
} | null {
|
||||
let text: string;
|
||||
if (typeof data === "string") {
|
||||
text = data;
|
||||
} else if (data instanceof Buffer) {
|
||||
// Heuristique : tenter de parser comme JSON UTF-8 ; si ça échoue, c'est binaire.
|
||||
try {
|
||||
text = data.toString("utf8");
|
||||
if (!text.startsWith("{")) return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
} else if (typeof data === "object" && data !== null && "toString" in data) {
|
||||
try {
|
||||
text = (data as { toString: () => string }).toString();
|
||||
if (!text.startsWith("{")) return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(text) as {
|
||||
serverContent?: {
|
||||
inputTranscription?: { text?: string };
|
||||
outputTranscription?: { text?: string };
|
||||
};
|
||||
};
|
||||
const sc = parsed.serverContent;
|
||||
if (!sc) return {};
|
||||
return {
|
||||
inputText: sc.inputTranscription?.text,
|
||||
outputText: sc.outputTranscription?.text,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Détecte un signal de fin de session envoyé par le client : `{type:'end'}`.
|
||||
*/
|
||||
|
|
@ -178,19 +156,53 @@ function isEndSignal(data: unknown): boolean {
|
|||
}
|
||||
|
||||
/**
|
||||
* Ouvre une session Gemini Live et proxifie les messages
|
||||
* Parse un message client `{type:'audio', data: base64}` et renvoie le base64
|
||||
* si le format est valide, sinon null.
|
||||
*/
|
||||
function parseAudioChunk(data: unknown): string | null {
|
||||
let text: string;
|
||||
if (typeof data === "string") {
|
||||
text = data;
|
||||
} else if (data instanceof Buffer) {
|
||||
try {
|
||||
text = data.toString("utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
if (!text.startsWith("{")) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(text) as { type?: string; data?: unknown };
|
||||
if (parsed.type === "audio" && typeof parsed.data === "string") {
|
||||
return parsed.data;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ouvre une session Gemini Live via le SDK et proxifie les messages
|
||||
* dans les deux sens entre le client (navigateur) et Gemini.
|
||||
*
|
||||
* - À l'open Gemini : envoie le setup frame avec prompt dynamique + VAD
|
||||
* + inputAudioTranscription + outputAudioTranscription.
|
||||
* - Forward transparent des frames audio dans les deux directions.
|
||||
* - Accumule les transcripts (input = candidat, output = examinateur IA).
|
||||
* - Détecte signal client `{type:'end'}` → déclenche fin de session.
|
||||
* - Timeout 210 s : warning client à 180 s, fin auto à 210 s.
|
||||
* - En fin de session : appelle `onSessionEnd(transcript)` puis ferme Gemini.
|
||||
* Le client WS n'est PAS fermé ici — c'est l'appelant qui décide (envoi du
|
||||
* rapport puis close 1000).
|
||||
* - Erreur Gemini → close client 4006 GEMINI_DISCONNECTED.
|
||||
* - Init : `new GoogleGenAI({ vertexai: true, apiKey })` → mode Vertex Express
|
||||
* (compatible avec les clés API auto-bound à un service account).
|
||||
* - Setup config : modèle + responseModalities AUDIO + systemInstruction
|
||||
* + inputAudioTranscription + outputAudioTranscription + VAD.
|
||||
* - Forward client → Gemini : parse `{type:'audio', data: base64}` →
|
||||
* `session.sendRealtimeInput({audio: {data, mimeType: 'audio/pcm;rate=16000'}})`.
|
||||
* - Forward Gemini → client : `clientWs.send(JSON.stringify(msg))` (le frontend
|
||||
* parse `serverContent.modelTurn.parts[].inlineData.data`).
|
||||
* - Accumule input/outputTranscription pour la correction finale.
|
||||
* - Détecte `{type:'end'}` du client → fin de session.
|
||||
* - Timer 210 s : warning à 180 s, fin auto à 210 s.
|
||||
* - En fin : `onSessionEnd(transcript)` puis ferme la session SDK. Le client WS
|
||||
* n'est PAS fermé ici — c'est l'appelant qui décide (envoi du rapport puis
|
||||
* close 1000).
|
||||
* - Erreur SDK / close Gemini → close client 4006 GEMINI_DISCONNECTED.
|
||||
* - GEMINI_API_KEY absente → close client 4005 GEMINI_CONFIG.
|
||||
*/
|
||||
export function openGeminiLiveSession(
|
||||
|
|
@ -211,17 +223,14 @@ export function openGeminiLiveSession(
|
|||
contexte: opts.contexte,
|
||||
});
|
||||
|
||||
const url = `${GEMINI_LIVE_URL}?key=${apiKey}`;
|
||||
const factory =
|
||||
opts.geminiFactory ??
|
||||
((u: string) => new NodeWebSocket(u) as unknown as WebSocketLike);
|
||||
|
||||
const geminiWs = factory(url);
|
||||
const ai =
|
||||
opts.clientFactory?.(apiKey) ?? new GoogleGenAI({ vertexai: true, apiKey });
|
||||
|
||||
const transcriptEntries: TranscriptEntry[] = [];
|
||||
let sessionEnded = false;
|
||||
let warningTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let session: Session | null = null;
|
||||
|
||||
const clearTimers = () => {
|
||||
if (warningTimer !== null) {
|
||||
|
|
@ -238,10 +247,12 @@ export function openGeminiLiveSession(
|
|||
if (sessionEnded) return;
|
||||
sessionEnded = true;
|
||||
clearTimers();
|
||||
try {
|
||||
geminiWs.close(1000);
|
||||
} catch {
|
||||
/* ignore */
|
||||
if (session) {
|
||||
try {
|
||||
session.close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
if (opts.onSessionEnd) {
|
||||
try {
|
||||
|
|
@ -255,105 +266,153 @@ export function openGeminiLiveSession(
|
|||
}
|
||||
};
|
||||
|
||||
geminiWs.on("open", () => {
|
||||
console.log("[T2] Gemini WS opened");
|
||||
try {
|
||||
geminiWs.send(buildSetupFrame(systemPrompt));
|
||||
console.log("[T2] Setup frame sent");
|
||||
|
||||
// Démarrer les timers une fois la session Gemini effectivement ouverte.
|
||||
warningTimer = setTimeout(() => {
|
||||
if (sessionEnded) return;
|
||||
try {
|
||||
clientWs.send(
|
||||
JSON.stringify({
|
||||
type: "warning",
|
||||
message: "30 secondes restantes",
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}, warningMs);
|
||||
|
||||
timeoutTimer = setTimeout(() => {
|
||||
void endSession();
|
||||
}, timeoutMs);
|
||||
} catch {
|
||||
try {
|
||||
clientWs.close(4005, "GEMINI_CONFIG");
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
const handleSdkMessage = (msg: LiveServerMessage) => {
|
||||
// Accumuler transcripts pour la correction finale.
|
||||
const sc = msg.serverContent;
|
||||
if (sc?.inputTranscription?.text && sc.inputTranscription.text.length > 0) {
|
||||
transcriptEntries.push({
|
||||
speaker: "candidat",
|
||||
text: sc.inputTranscription.text,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
geminiWs.on("message", (data) => {
|
||||
// Tentative d'extraction des transcripts — si JSON, on accumule ;
|
||||
// dans tous les cas (JSON ou audio binaire), on forward au client.
|
||||
const parsed = tryParseGeminiMessage(data);
|
||||
if (parsed) {
|
||||
if (parsed.inputText && parsed.inputText.length > 0) {
|
||||
transcriptEntries.push({
|
||||
speaker: "candidat",
|
||||
text: parsed.inputText,
|
||||
});
|
||||
}
|
||||
if (parsed.outputText && parsed.outputText.length > 0) {
|
||||
transcriptEntries.push({
|
||||
speaker: "examinateur",
|
||||
text: parsed.outputText,
|
||||
});
|
||||
}
|
||||
if (
|
||||
sc?.outputTranscription?.text &&
|
||||
sc.outputTranscription.text.length > 0
|
||||
) {
|
||||
transcriptEntries.push({
|
||||
speaker: "examinateur",
|
||||
text: sc.outputTranscription.text,
|
||||
});
|
||||
}
|
||||
|
||||
// Forward verbatim au client. Le frontend parse serverContent.modelTurn.
|
||||
try {
|
||||
clientWs.send(data);
|
||||
clientWs.send(JSON.stringify(msg));
|
||||
} catch {
|
||||
void endSession();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// ── Ouverture de la session SDK ──────────────────────────────────────
|
||||
ai.live
|
||||
.connect({
|
||||
model: GEMINI_LIVE_MODEL,
|
||||
config: {
|
||||
responseModalities: [Modality.AUDIO],
|
||||
systemInstruction: systemPrompt,
|
||||
inputAudioTranscription: {},
|
||||
outputAudioTranscription: {},
|
||||
realtimeInputConfig: {
|
||||
automaticActivityDetection: {
|
||||
disabled: false,
|
||||
startOfSpeechSensitivity: StartSensitivity.START_SENSITIVITY_LOW,
|
||||
endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_LOW,
|
||||
silenceDurationMs: 2000,
|
||||
},
|
||||
},
|
||||
},
|
||||
callbacks: {
|
||||
onopen: () => {
|
||||
console.log("[T2] Session Gemini ouverte (SDK)");
|
||||
// Démarrer les timers une fois la session effectivement ouverte.
|
||||
warningTimer = setTimeout(() => {
|
||||
if (sessionEnded) return;
|
||||
try {
|
||||
clientWs.send(
|
||||
JSON.stringify({
|
||||
type: "warning",
|
||||
message: "30 secondes restantes",
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}, warningMs);
|
||||
|
||||
timeoutTimer = setTimeout(() => {
|
||||
void endSession();
|
||||
}, timeoutMs);
|
||||
},
|
||||
onmessage: (msg: LiveServerMessage) => {
|
||||
handleSdkMessage(msg);
|
||||
},
|
||||
onerror: (err: unknown) => {
|
||||
console.log(
|
||||
"[T2] Erreur SDK :",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
if (!sessionEnded) {
|
||||
clearTimers();
|
||||
sessionEnded = true;
|
||||
try {
|
||||
clientWs.close(4006, "GEMINI_DISCONNECTED");
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose: () => {
|
||||
console.log("[T2] Session Gemini fermée (SDK)");
|
||||
if (!sessionEnded) {
|
||||
clearTimers();
|
||||
try {
|
||||
clientWs.close(4006, "GEMINI_DISCONNECTED");
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((s: Session) => {
|
||||
session = s;
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
console.log(
|
||||
"[T2] live.connect a échoué :",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
sessionEnded = true;
|
||||
clearTimers();
|
||||
try {
|
||||
clientWs.close(4006, "GEMINI_DISCONNECTED");
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
});
|
||||
|
||||
// ── Forward client → Gemini ──────────────────────────────────────────
|
||||
clientWs.on("message", (data) => {
|
||||
if (isEndSignal(data)) {
|
||||
void endSession();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
geminiWs.send(data);
|
||||
} catch {
|
||||
void endSession();
|
||||
}
|
||||
});
|
||||
|
||||
geminiWs.on("close", () => {
|
||||
console.log("[T2] Gemini closed");
|
||||
if (!sessionEnded) {
|
||||
clearTimers();
|
||||
const audioBase64 = parseAudioChunk(data);
|
||||
if (audioBase64 !== null && session !== null && !sessionEnded) {
|
||||
try {
|
||||
clientWs.close(4006, "GEMINI_DISCONNECTED");
|
||||
} catch {
|
||||
/* ignore */
|
||||
session.sendRealtimeInput({
|
||||
audio: {
|
||||
data: audioBase64,
|
||||
mimeType: "audio/pcm;rate=16000",
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
console.log(
|
||||
"[T2] sendRealtimeInput a échoué :",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
void endSession();
|
||||
}
|
||||
}
|
||||
// Tout autre message client est ignoré (ex: ping keep-alive frontend).
|
||||
});
|
||||
|
||||
clientWs.on("close", () => {
|
||||
clearTimers();
|
||||
sessionEnded = true;
|
||||
try {
|
||||
geminiWs.close(1000);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
});
|
||||
|
||||
geminiWs.on("error", (err) => {
|
||||
console.log("[T2] Gemini error:", (err as Error)?.message);
|
||||
if (!sessionEnded) {
|
||||
clearTimers();
|
||||
sessionEnded = true;
|
||||
if (session) {
|
||||
try {
|
||||
clientWs.close(4006, "GEMINI_DISCONNECTED");
|
||||
session.close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
|
@ -363,10 +422,12 @@ export function openGeminiLiveSession(
|
|||
clientWs.on("error", () => {
|
||||
clearTimers();
|
||||
sessionEnded = true;
|
||||
try {
|
||||
geminiWs.close(1011);
|
||||
} catch {
|
||||
/* ignore */
|
||||
if (session) {
|
||||
try {
|
||||
session.close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -230,8 +230,8 @@ export async function runT2LiveCorrection(args: {
|
|||
}
|
||||
|
||||
export interface CreateT2LiveRoutesOptions {
|
||||
/** Injection pour les tests : fabrique de WebSocket vers Gemini. */
|
||||
geminiFactory?: OpenGeminiLiveSessionOptions["geminiFactory"];
|
||||
/** Injection pour les tests : fabrique de client SDK Gemini (Sprint 6d). */
|
||||
clientFactory?: OpenGeminiLiveSessionOptions["clientFactory"];
|
||||
/** Injection pour les tests : override timeout/warning. */
|
||||
timeoutMs?: number;
|
||||
warningMs?: number;
|
||||
|
|
@ -313,7 +313,7 @@ export default function createT2LiveRoutes(
|
|||
openGeminiLiveSession(adapter, {
|
||||
role: sujetNonNull.role!,
|
||||
contexte: sujetNonNull.contexte!,
|
||||
geminiFactory: opts.geminiFactory,
|
||||
clientFactory: opts.clientFactory,
|
||||
timeoutMs: opts.timeoutMs,
|
||||
warningMs: opts.warningMs,
|
||||
onSessionEnd: async (transcript) => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue