Integrate into Website and Apps

1
type AudioOutControls = {
2
  enqueueAudioChunk: (chunk: Int16Array) => void;
3
  audioContext: AudioContext;
4
  stop: () => void;
5
};
6
7
async function startAudioOut(): Promise<AudioOutControls> {
8
  const ctx = new AudioContext({ sampleRate: 16000 });
9
  const gainNode = ctx.createGain();
10
  gainNode.connect(ctx.destination);
11
  await ctx.audioWorklet.addModule("/audio-out-worklet.js");
12
  const worklet = new AudioWorkletNode(ctx, "audio-out-worklet");
13
  worklet.connect(gainNode);
14
  return {
15
    enqueueAudioChunk: (chunk: Int16Array) =>
16
      worklet.port.postMessage({ buffer: chunk }),
17
    audioContext: ctx,
18
    stop: () => {
19
      worklet.disconnect();
20
      gainNode.disconnect();
21
      ctx.close();
22
    },
23
  };
24
}
25
26
async function startRecording(
27
  onAudioChunk: (chunk: ArrayBufferLike) => void,
28
  onRecordingStarted: () => Promise<void>,
29
): Promise<{ stop: () => void }> {
30
  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
31
  const audioContext = new window.AudioContext();
32
  const sampleRate = audioContext.sampleRate; // 48000
33
  const sourceNode = audioContext.createMediaStreamSource(stream);
34
  const scriptProcessor = audioContext.createScriptProcessor(8192, 1, 1);
35
36
  let hasRecordingStarted = false;
37
  scriptProcessor.onaudioprocess = (
38
    audioProcessingEvent: AudioProcessingEvent
39
  ) => {
40
    if (!hasRecordingStarted) {
41
      hasRecordingStarted = true;
42
      onRecordingStarted();
43
    }
44
    const floatData = audioProcessingEvent.inputBuffer.getChannelData(0);
45
    const int16Data = float32ToInt16(floatData);
46
    const byteBuffer = new Uint8Array(int16Data.buffer);
47
    onAudioChunk(byteBuffer);
48
  };
49
50
  sourceNode.connect(scriptProcessor);
51
  scriptProcessor.connect(audioContext.destination);
52
  return {
53
    stop: () => {
54
      scriptProcessor.disconnect();
55
      scriptProcessor.onaudioprocess = null;
56
      sourceNode.disconnect();
57
      audioContext.close();
58
      stream.getTracks().forEach((track) => track.stop());
59
    },
60
  };
61
}
62
63
function float32ToInt16(float32Array: Float32Array): Int16Array {
64
  const int16Array = new Int16Array(float32Array.length);
65
  for (let i = 0; i < float32Array.length; i++) {
66
    const s = Math.max(-1, Math.min(1, float32Array[i]));
67
    int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
68
  }
69
  return int16Array;
70
}

1
const MESSAGE_TYPE_STATUS_AGENT_READY = "status_agent_ready";
2
const MESSAGE_TYPE_STATUS_CLIENT_READY = "status_client_ready";
3
4
async function startSynthflow(
5
  url: string,
6
  playAudioChunk: (chunk: Int16Array) => void
7
): Promise<{
8
  stop: () => void;
9
}> {
10
  updateStatus("Connecting to server", "warning");
11
  
12
  const websocket = new WebSocket(
13
    url
14
  );
15
  websocket.binaryType = "arraybuffer";
16
17
  websocket.onmessage = async (event) => {
18
    // Handle audio
19
    if (event.data instanceof ArrayBuffer) {
20
      const arrayBuffer = event.data as ArrayBuffer;
21
      const pcmSamples = new Int16Array(arrayBuffer);
22
      playAudioChunk(pcmSamples);
23
    } else {
24
      const data = JSON.parse(event.data);
25
      switch (data.type) {
26
        case MESSAGE_TYPE_STATUS_AGENT_READY:
27
          updateStatus("Connected to call", "success");
28
          console.log("Received agent ready message");
29
          break;
30
        default:
31
          console.log("Received unknown message from server", data);
32
          break;
33
      }
34
    }
35
  };
36
37
  const recordingControls = await startRecording(
38
    (audio) => {
39
      if (websocket.readyState === WebSocket.OPEN) {
40
        websocket.send(audio);
41
      }
42
    },
43
    async () => {
44
      sendWhenReady(websocket, JSON.stringify({ type: MESSAGE_TYPE_STATUS_CLIENT_READY }))
45
      console.log("Scheduled send client ready message");
46
    }
47
  );
48
49
  websocket.onclose = () => {
50
    recordingControls.stop();
51
    updateStatus("Disconnected from server", "error");
52
  };
53
54
  return {
55
    stop: () => {
56
      websocket.close();
57
    },
58
  };
59
}
60
61
function base64ToArrayBuffer(base64: string): ArrayBuffer {
62
  const binaryString = window.atob(base64);
63
  const len = binaryString.length;
64
  const bytes = new Uint8Array(len);
65
  for (let i = 0; i < len; i++) {
66
    bytes[i] = binaryString.charCodeAt(i);
67
  }
68
  return bytes.buffer;
69
}
70
71
function sendWhenReady(websocket: WebSocket, message: string) {
72
  if (websocket.readyState === WebSocket.CLOSED) {
73
    console.log("WebSocket is closed, not sending message");
74
    return;
75
  } else if (websocket.readyState === WebSocket.OPEN) {
76
    websocket.send(message);
77
  } else {
78
    setTimeout(() => sendWhenReady(websocket, message), 50);
79
  }
80
}

1
async function makeCall() {
2
  const sessionURL = ... // make a call to your server to get a session URL
3
4
  const audioOutControls = await startAudioOut();
5
  const synthflowControls = await startSynthflow(sessionURL, (audio) => {
6
    audioOutControls.enqueueAudioChunk(audio);
7
  });
8
9
  // stop audio/call when needed
10
}