diff --git a/apps/browser-example/src/components/EventList.tsx b/apps/browser-example/src/components/EventList.tsx index 6788985..b14cc45 100644 --- a/apps/browser-example/src/components/EventList.tsx +++ b/apps/browser-example/src/components/EventList.tsx @@ -36,9 +36,21 @@ export function EventList({ events }: { events: any[] }) { } }, [events]) + function saveEvents() { + const blob = new Blob([JSON.stringify(events, null, 2)], { + type: "application/json", + }) + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + const timestamp = new Date().toISOString().replace(/[:.]/g, "-") + a.download = `events-${timestamp}.json` + a.click() + } + return (
-
+
+ Event Count: {events.length}
=4" } }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "license": "MIT" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -4517,10 +4540,12 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "lodash-es": "^4.17.21", "typescript-event-target": "^1.1.1" }, "devDependencies": { "@tsconfig/node20": "^20.1.4", + "@types/lodash-es": "^4.17.12", "type-fest": "^4.33.0" } }, diff --git a/packages/browser/package.json b/packages/browser/package.json index 1c22c53..4d58d59 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -23,6 +23,9 @@ "types": "./dist/WebRTC/index.d.ts", "import": "./dist/WebRTC/index.js" }, + "./openai": { + "types": "./dist/openai/index.d.ts" + }, "./*": { "types": "./dist/*.d.ts", "import": "./dist/*.js" @@ -35,9 +38,11 @@ }, "devDependencies": { "@tsconfig/node20": "^20.1.4", + "@types/lodash-es": "^4.17.12", "type-fest": "^4.33.0" }, "dependencies": { + "lodash-es": "^4.17.21", "typescript-event-target": "^1.1.1" } } diff --git a/packages/browser/scripts/gen-openai-from-openapi.sh b/packages/browser/scripts/gen-openai-from-openapi.sh index a729121..ed80711 100755 --- a/packages/browser/scripts/gen-openai-from-openapi.sh +++ b/packages/browser/scripts/gen-openai-from-openapi.sh @@ -9,4 +9,4 @@ parent_dir=$(cd "${this_dir}/.."; pwd) # https://github.com/openai/openai-openapi # https://raw.githubusercontent.com/openai/openai-openapi/25be47865ea2df1179a46be045c2f6b65f38e982/openapi.yaml -npx openapi-typescript "https://raw.githubusercontent.com/openai/openai-openapi/25be47865ea2df1179a46be045c2f6b65f38e982/openapi.yaml" -o "${parent_dir}/src/types/openai/openapi.d.ts" +npx openapi-typescript "https://raw.githubusercontent.com/openai/openai-openapi/25be47865ea2df1179a46be045c2f6b65f38e982/openapi.yaml" -o "${parent_dir}/src/openai/openapi.ts" diff --git a/packages/browser/src/WebRTC/RealtimeClient.ts b/packages/browser/src/WebRTC/RealtimeClient.ts index 0247f5a..bf01e98 100644 --- a/packages/browser/src/WebRTC/RealtimeClient.ts +++ b/packages/browser/src/WebRTC/RealtimeClient.ts @@ -3,6 +3,7 @@ import type { RealtimeConversationItem, RealtimeServerEvent, RealtimeServerEventConversationItemCreated, + RealtimeServerEventError, RealtimeServerEventResponseAudioTranscriptDelta, RealtimeServerEventResponseAudioTranscriptDone, RealtimeServerEventResponseDone, @@ -10,7 +11,7 @@ import type { RealtimeServerEventSessionUpdated, RealtimeSession, RealtimeSessionCreateRequest, -} from "../types/openai" +} from "../openai" import { findConversationItem, findConversationItemContent, @@ -27,6 +28,7 @@ import { EventTargetListener, RealtimeClientEventMap, } from "./events" +import { isEqual } from "lodash-es" const log = console @@ -121,20 +123,30 @@ export class RealtimeClient { } // Create a data channel from a peer connection + let dataChannelOpenedPromise: Promise try { - this.initializeDataChannel() + // this promise will resolve when the channel is open and it's safe to send client events. It must be opened by the server after we initialize the channel with the SDP + dataChannelOpenedPromise = this.initializeDataChannel() } catch (err) { log.error("Failed to initialize data channel", err) throw err } - // Start the session using the Session Description Protocol (SDP) try { + // Start the session using the Session Description Protocol (SDP) await this.initializeSession() } catch (err) { log.error("Failed to initialize session", err) throw err } + + // await data channel open so that clientEvents can be sent before we continue or return + try { + await dataChannelOpenedPromise + } catch (err) { + log.error("Failed to await data channel open", err) + throw err + } } catch (err) { log.error("Failed to start RealtimeClient", err) // call stop to cleanup anything partially initialized @@ -153,7 +165,7 @@ export class RealtimeClient { this.audioElement.autoplay = true } - // Add local audio track for microphone input in the browser + // Add local audio track for microphone input in the browser: this.localMediaStream = await this.navigator.mediaDevices.getUserMedia({ audio: { // see https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints @@ -232,18 +244,33 @@ export class RealtimeClient { await this.peerConnection.setRemoteDescription(answer) } - private initializeDataChannel() { + private async initializeDataChannel(): Promise { if (!this.peerConnection) { throw new Error("No peer connection") } - this.dataChannel = this.peerConnection.createDataChannel("oai-events") + const dataChannel = this.peerConnection.createDataChannel("oai-events") + + // we will let the caller resolve when the dataChannel is opened + const dataChannelOpenedPromise = new Promise((resolve) => { + dataChannel.addEventListener("open", () => { + log.debug("Data channel open") + resolve() + }) + }) + + this.dataChannel = dataChannel // Listen for server-sent events on the data channel this.dataChannel.addEventListener( "message", this.receiveServerMessage.bind(this) ) + this.dataChannel.addEventListener("error", (e) => { + log.error("Data channel error from server: %o", e.error) + }) + + return dataChannelOpenedPromise } public async stop(): Promise { @@ -291,7 +318,9 @@ export class RealtimeClient { } private processServerEvent(event: RealtimeServerEvent) { - const handler = RealtimeClient.privateServerEventHandlers[event.type] + const handler = RealtimeClient.privateServerEventHandlers[ + event.type + ] as RealtimeServerEventHandler<(typeof event)["type"]> if (handler) { handler(this, event) } @@ -331,192 +360,228 @@ export class RealtimeClient { return new Blob(this.audioChunks, { type: "audio/webm" }) } - // TODO: make these events provide typed events. - private static privateServerEventHandlers: Partial< - Record - > = { - "session.created": (client, event) => { - // when a conversation is created, set the conversation state: - const sessionEvent = event as RealtimeServerEventSessionCreated - - client.session = sessionEvent.session - client.emitter.dispatchTypedEvent( - "sessionCreated", - new SessionCreatedEvent(sessionEvent.session) - ) - - if (!client.sessionRequested) { - throw new Error("No session request") - } + // TODO: make these events provide typed events to each handler (should be able to use TS types to do that). + private static privateServerEventHandlers: Partial = + { + error: (_, event) => { + const errorEvent = event as RealtimeServerEventError + log.error("Error event from server: %o", errorEvent) + }, + "session.created": (client, event) => { + // when a conversation is created, set the conversation state: + const sessionEvent = event as RealtimeServerEventSessionCreated - // NOTE: When we create a session with OpenAI, it ignores things like input_audio_transcription?.model !== "whisper-1"; So we update it again if it doesn't match the session. - let updatedSession: RealtimeSessionCreateRequest = { - ...client.sessionRequested, - } - let hasSessionMismatch = false - - for (const key of Object.keys(client.sessionRequested) as Array< - keyof RealtimeSessionCreateRequest - >) { - const requestValue = client.sessionRequested[key] - const sessionValue = sessionEvent.session[key] - if (requestValue === sessionValue) { - delete updatedSession[key] - continue - } - hasSessionMismatch = true - } - if (hasSessionMismatch) { - log.warn( - "Updating mismatched session to match requested session: %o", - updatedSession + client.session = sessionEvent.session + client.emitter.dispatchTypedEvent( + "sessionCreated", + new SessionCreatedEvent(sessionEvent.session) ) - const updateSessionEvent: RealtimeClientEventSessionUpdate = { - type: "session.update", - session: updatedSession, + + if (!client.sessionRequested) { + throw new Error("No session request") } - client.sendClientEvent(updateSessionEvent) - } - }, - "session.updated": (client, event) => { - const sessionEvent = event as RealtimeServerEventSessionUpdated - client.session = sessionEvent.session - client.emitter.dispatchTypedEvent( - "sessionUpdated", - new SessionUpdatedEvent(sessionEvent.session) - ) - }, - "conversation.item.created": (client, event) => { - const conversationEvent = - event as RealtimeServerEventConversationItemCreated - client.conversation.push(conversationEvent.item) - client.emitter.dispatchTypedEvent( - "conversationChanged", - new ConversationChangedEvent(client.conversation) - ) - }, - "response.audio_transcript.delta": (client, event) => { - const deltaEvent = - event as RealtimeServerEventResponseAudioTranscriptDelta - const { foundContent, foundItem } = findConversationItemContent( - { log }, - client.conversation, - deltaEvent.item_id, - deltaEvent.content_index, - deltaEvent - ) - if (!foundItem) { - // error was logged in findConversationItemContent - return - } - if (!foundContent) { - if (!foundItem.content) { - foundItem.content = [] + + // NOTE: When we create a session with OpenAI, it ignores things like input_audio_transcription?.model !== "whisper-1"; So we update it again if it doesn't match the session. + let updatedSession: RealtimeSessionCreateRequest = { + ...client.sessionRequested, } - foundItem.content.push({ - type: "input_audio", - transcript: deltaEvent.delta, - }) - } else { - if (foundContent.type !== "input_audio") { - log.error( - `${event.type} Unexpected content type ${foundContent.type} for audio transcript` + let hasSessionMismatch = false + + for (const key of Object.keys(client.sessionRequested) as Array< + keyof RealtimeSessionCreateRequest + >) { + const requestValue = client.sessionRequested[key] + const sessionValue = sessionEvent.session[key] + + if (compareValuesIgnoreNullProperties(requestValue, sessionValue)) { + continue + } + log.debug( + `session mismatch on ${key}: %o !== %o`, + requestValue, + sessionValue ) - return + hasSessionMismatch = true } - foundContent.transcript += deltaEvent.delta - } - client.emitter.dispatchTypedEvent( - "conversationChanged", - new ConversationChangedEvent(client.conversation) - ) - }, - "response.text.delta": (client, event) => { - log.error( - `${event.type} TODO: Need to handle event to support text streaming.` - ) - // TODO: use these to stream the messages: https://platform.openai.com/docs/api-reference/realtime-server-events/response/text/delta & https://platform.openai.com/docs/api-reference/realtime-server-events/response/audio_transcript/delta - }, - "response.done": (client, event) => { - const responseEvent = event as RealtimeServerEventResponseDone - - // https://platform.openai.com/docs/api-reference/realtime-server-events/response/done - // for each response content item, find the conversation item patch it up: - const response = responseEvent.response - if (!response.output) { - log.error("No output in response.done") - return - } - for (const output of response.output) { - if (output.type != "message") { - // function? - log.error(`Unexpected output type ${output.type} in response.done`) - continue + if (hasSessionMismatch) { + const updateSessionEvent: RealtimeClientEventSessionUpdate = { + type: "session.update", + session: updatedSession, + } + client.sendClientEvent(updateSessionEvent) } - if (!output.content) { - log.error("No content in output in response.done") - continue - } - const conversationItem = findConversationItem( + }, + "session.updated": (client, event) => { + const sessionEvent = event as RealtimeServerEventSessionUpdated + client.session = sessionEvent.session + client.emitter.dispatchTypedEvent( + "sessionUpdated", + new SessionUpdatedEvent(sessionEvent.session) + ) + }, + "conversation.item.created": (client, event) => { + const conversationEvent = + event as RealtimeServerEventConversationItemCreated + client.conversation.push(conversationEvent.item) + client.emitter.dispatchTypedEvent( + "conversationChanged", + new ConversationChangedEvent(client.conversation) + ) + }, + "response.audio_transcript.delta": (client, event) => { + const deltaEvent = + event as RealtimeServerEventResponseAudioTranscriptDelta + const { foundContent, foundItem } = findConversationItemContent( { log }, client.conversation, - output.id!, - event + deltaEvent.item_id, + deltaEvent.content_index, + deltaEvent ) - if (!conversationItem) { - // TODO: findConversationItem already logged an error, we should probably pass in a value that tells it not to log - // no existing item is there, for some reason maybe we missed it in the stream somehow? We'll just add it: - client.conversation.push(output) + if (!foundItem) { + // error was logged in findConversationItemContent + return + } + if (!foundContent) { + if (!foundItem.content) { + foundItem.content = [] + } + foundItem.content.push({ + type: "input_audio", + transcript: deltaEvent.delta, + }) + } else { + if (foundContent.type !== "input_audio") { + log.error( + `${event.type} Unexpected content type ${foundContent.type} for audio transcript` + ) + return + } + foundContent.transcript += deltaEvent.delta + } + client.emitter.dispatchTypedEvent( + "conversationChanged", + new ConversationChangedEvent(client.conversation) + ) + }, + "response.text.delta": (client, event) => { + // TODO: Need to handle event to support text streaming text events (these are only for the input items where the input itself is text and not audio). + }, + "response.done": (client, event) => { + const responseEvent = event as RealtimeServerEventResponseDone + + // https://platform.openai.com/docs/api-reference/realtime-server-events/response/done + // for each response content item, find the conversation item patch it up: + const response = responseEvent.response + if (!response.output) { + log.error("No output in response.done") + return + } + for (const output of response.output) { + if (output.type != "message") { + // function? + log.error(`Unexpected output type ${output.type} in response.done`) + continue + } + if (!output.content) { + log.error("No content in output in response.done") + continue + } + const conversationItem = findConversationItem( + { log }, + client.conversation, + output.id!, + event + ) + if (!conversationItem) { + // TODO: findConversationItem already logged an error, we should probably pass in a value that tells it not to log + // no existing item is there, for some reason maybe we missed it in the stream somehow? We'll just add it: + client.conversation.push(output) + client.emitter.dispatchTypedEvent( + "conversationChanged", + new ConversationChangedEvent(client.conversation) + ) + continue + } + // TODO: we probably need to handle this better. Probably we need to overwrite the existing item with this new one since it is now "done". + // patch up the conversation item with the provided output: + if (!conversationItem.content) { + conversationItem.content = [] + } + for (const outputItem of output.content) { + conversationItem.content.push(outputItem) + } + // force update the conversation state: client.emitter.dispatchTypedEvent( "conversationChanged", new ConversationChangedEvent(client.conversation) ) - continue - } - // TODO: we probably need to handle this better. Probably we need to overwrite the existing item with this new one since it is now "done". - // patch up the conversation item with the provided output: - if (!conversationItem.content) { - conversationItem.content = [] } - for (const outputItem of output.content) { - conversationItem.content.push(outputItem) - } - // force update the conversation state: + }, + "response.audio_transcript.done": (client, event) => { + patchConversationItemWithCompletedTranscript( + { log }, + client.conversation, + event as RealtimeServerEventResponseAudioTranscriptDone + ) client.emitter.dispatchTypedEvent( "conversationChanged", new ConversationChangedEvent(client.conversation) ) - } - }, - "response.audio_transcript.done": (client, event) => { - patchConversationItemWithCompletedTranscript( - { log }, - client.conversation, - event as RealtimeServerEventResponseAudioTranscriptDone - ) - client.emitter.dispatchTypedEvent( - "conversationChanged", - new ConversationChangedEvent(client.conversation) - ) - }, - "conversation.item.input_audio_transcription.completed": ( - client, - event - ) => { - patchConversationItemWithCompletedTranscript( - { log }, - client.conversation, - event as RealtimeServerEventResponseAudioTranscriptDone - ) - client.emitter.dispatchTypedEvent( - "conversationChanged", - new ConversationChangedEvent(client.conversation) - ) - }, - } + }, + "conversation.item.input_audio_transcription.completed": ( + client, + event + ) => { + patchConversationItemWithCompletedTranscript( + { log }, + client.conversation, + event + ) + client.emitter.dispatchTypedEvent( + "conversationChanged", + new ConversationChangedEvent(client.conversation) + ) + }, + } } -type RealtimeClientServerEventHandler = ( +type RealtimeServerEventHandler< + TRealtimeServerEventType extends RealtimeServerEvent["type"] = RealtimeServerEvent["type"] +> = ( client: RealtimeClient, - event: RealtimeServerEvent + event: Extract ) => void + +type RealtimeServerEventNames = RealtimeServerEvent["type"] + +type RealtimeServerEventTypeToHandlerMap = { + [K in RealtimeServerEventNames]: RealtimeServerEventHandler +} + +/** + * If the two values are objects, and one has a property that is null or + * undefined and the other either does not have that property or the + * property is null or undefined, then the property will be considered equal. + * If the value is not an object, a normal deep-equal operation is done. + */ +function compareValuesIgnoreNullProperties(valueA: any, valueB: any): boolean { + if ( + typeof valueA === "object" && + typeof valueB === "object" && + valueA && + valueB + ) { + for (const key in valueB) { + if (valueB[key] == null) { + // if session has null/undefined property, ignore it if request doesn't have it or is also null/undefined + if (!(key in valueA) || valueA[key] == null) continue + return false + } + if (!compareValuesIgnoreNullProperties(valueA[key], valueB[key])) + return false + } + return true + } + return isEqual(valueA, valueB) +} diff --git a/packages/browser/src/WebRTC/events.ts b/packages/browser/src/WebRTC/events.ts index 36ce238..3daa51c 100644 --- a/packages/browser/src/WebRTC/events.ts +++ b/packages/browser/src/WebRTC/events.ts @@ -3,7 +3,7 @@ import type { RealtimeServerEvent, RealtimeConversationItem, RealtimeSession, -} from "../types/openai" +} from "../openai" // Add this index signature allows the key to be string // & { @@ -19,7 +19,7 @@ type RealtimeClientEventObjects = RealtimeClientEventMap[RealtimeClientEventNames] class BaseEvent< - TType extends keyof RealtimeClientEventMap | RealtimeServerEvent["type"] + TType extends keyof RealtimeClientEventMap | RealtimeServerEvent["type"], > extends Event { constructor(public readonly type: TType) { super(type) @@ -27,7 +27,7 @@ class BaseEvent< } export class RealtimeServerEventEvent< - T extends RealtimeServerEvent = RealtimeServerEvent + T extends RealtimeServerEvent = RealtimeServerEvent, > extends BaseEvent<"serverEvent"> { constructor(public readonly event: T) { super("serverEvent") @@ -60,7 +60,7 @@ export class SessionCreatedEvent extends BaseEvent<"sessionCreated"> { } export interface EventTargetListener< - TEvent extends RealtimeClientEventObjects + TEvent extends RealtimeClientEventObjects, > { (evt: TEvent): void } diff --git a/packages/browser/src/WebRTC/items.ts b/packages/browser/src/WebRTC/items.ts index 7c499db..2e43534 100644 --- a/packages/browser/src/WebRTC/items.ts +++ b/packages/browser/src/WebRTC/items.ts @@ -3,7 +3,7 @@ import type { RealtimeConversationItem, RealtimeConversationItemContent, RealtimeServerEventWithCompletedTranscript, -} from "../types/openai" +} from "../openai" /** * Finds the specified item in a conversation. @@ -19,7 +19,7 @@ export function findConversationItem( forEvent: { type: string event_id: string - } + }, ): RealtimeConversationItem | undefined { // get conversation item: const found = conversation.find((convItem) => { @@ -27,7 +27,7 @@ export function findConversationItem( }) if (!found) { context.log.error( - `No conversation item ${item_id} found for event ${forEvent.type} with id ${forEvent.event_id}. Existing conversation searched: ${conversation} (${conversation.length} items)` + `No conversation item ${item_id} found for event ${forEvent.type} with id ${forEvent.event_id}. Existing conversation searched: ${conversation} (${conversation.length} items)`, ) return undefined } @@ -50,7 +50,7 @@ export function findConversationItemContent( forEvent: { type: string event_id: string - } + }, ): { foundItem: RealtimeConversationItem | undefined foundContent: RealtimeConversationItemContent | undefined @@ -59,14 +59,14 @@ export function findConversationItemContent( context, conversation, item_id, - forEvent + forEvent, ) if (!foundItem) { return { foundItem, foundContent: undefined } } if (!foundItem.content) { context.log.error( - `Conversation item ${foundItem.id} has no content at index ${content_index}.` + `Conversation item ${foundItem.id} has no content at index ${content_index}.`, ) return { foundItem, foundContent: undefined } } @@ -79,7 +79,7 @@ export function findConversationItemContent( export function patchConversationItemWithCompletedTranscript( context: { log: Logger }, existingConversation: RealtimeConversationItem[], - audioEvent: RealtimeServerEventWithCompletedTranscript + audioEvent: RealtimeServerEventWithCompletedTranscript, ): void { // get conversation item & content: const { foundItem, foundContent } = findConversationItemContent( @@ -87,7 +87,7 @@ export function patchConversationItemWithCompletedTranscript( existingConversation, audioEvent.item_id, audioEvent.content_index, - audioEvent + audioEvent, ) if (!foundItem) { // error was logged in findConversationItemContent @@ -109,7 +109,7 @@ export function patchConversationItemWithCompletedTranscript( if (foundContent.type !== "input_audio") { // only this even has a transcript field context.log.error( - `Unexpected content type ${foundContent.type} for audio transcript` + `Unexpected content type ${foundContent.type} for audio transcript`, ) return } diff --git a/packages/browser/src/types/openai/index.ts b/packages/browser/src/openai/index.ts similarity index 98% rename from packages/browser/src/types/openai/index.ts rename to packages/browser/src/openai/index.ts index 4384581..596a12e 100644 --- a/packages/browser/src/types/openai/index.ts +++ b/packages/browser/src/openai/index.ts @@ -53,7 +53,8 @@ export type RealtimeServerEventTypeMap = { export type RealtimeServerEventConversationItemCreated = components["schemas"]["RealtimeServerEventConversationItemCreated"] - +export type RealtimeServerEventError = + components["schemas"]["RealtimeServerEventError"] export type RealtimeServerEventSessionCreated = components["schemas"]["RealtimeServerEventSessionCreated"] export type RealtimeServerEventSessionUpdated = diff --git a/packages/browser/src/types/openai/openapi.d.ts b/packages/browser/src/openai/openapi.ts similarity index 100% rename from packages/browser/src/types/openai/openapi.d.ts rename to packages/browser/src/openai/openapi.ts