Audio WebSocket

// COMPUTER_ID is the computer's UUID. INSTANCE_ID is its instance_id, which
// fills the /desktops/{instance_id} slot. Create computer returns both.
const computerId = process.env.COMPUTER_ID;
const instanceId = process.env.INSTANCE_ID;
const apiKey = process.env.ORGO_API_KEY;

// Step 1: Get the computer password. This endpoint takes the UUID.
const res = await fetch(
  `https://www.orgo.ai/api/computers/${computerId}/vnc-password`,
  { headers: { 'Authorization': `Bearer ${apiKey}` } }
);
const { password } = await res.json();

// Step 2: Connect to the audio stream
const ws = new WebSocket(
  `wss://www.orgo.ai/desktops/${instanceId}/ws/audio?token=${password}`
);
ws.binaryType = 'arraybuffer';

// Step 3: Set up Web Audio API playback
const ctx = new AudioContext({ sampleRate: 24000 });
let nextPlayTime = 0;

ws.onmessage = (event) => {
  // Text frames are JSON control messages
  if (typeof event.data === 'string') {
    const msg = JSON.parse(event.data);
    if (msg.type === 'started') {
      console.log(`Audio stream: ${msg.sample_rate}Hz, ${msg.channels}ch`);
    }
    if (msg.type === 'error') {
      console.error('Audio error:', msg.message);
    }
    return;
  }

  // Binary frames are raw PCM audio
  const pcm = new Int16Array(event.data);
  const floats = new Float32Array(pcm.length);
  for (let i = 0; i < pcm.length; i++) {
    floats[i] = pcm[i] / 32768;
  }

  const buffer = ctx.createBuffer(1, floats.length, 24000);
  buffer.getChannelData(0).set(floats);

  const source = ctx.createBufferSource();
  source.buffer = buffer;
  source.connect(ctx.destination);

  // Schedule with drift correction
  const now = ctx.currentTime;
  if (nextPlayTime < now || nextPlayTime > now + 0.15) {
    nextPlayTime = now + 0.02;
  }
  source.start(nextPlayTime);
  nextPlayTime += buffer.duration;
};

import asyncio
import websockets
import json
import os
import struct
import requests

async def stream_audio(computer_id: str, instance_id: str, api_key: str):
    # Step 1: Get the computer password. This endpoint takes the UUID.
    res = requests.get(
        f"https://www.orgo.ai/api/computers/{computer_id}/vnc-password",
        headers={"Authorization": f"Bearer {api_key}"}
    )
    password = res.json()["password"]

    # Step 2: Connect to the audio stream
    url = f"wss://www.orgo.ai/desktops/{instance_id}/ws/audio?token={password}"

    async with websockets.connect(url) as ws:
        async for message in ws:
            if isinstance(message, str):
                data = json.loads(message)
                if data["type"] == "started":
                    print(f"Audio: {data['sample_rate']}Hz, {data['channels']}ch")
                elif data["type"] == "error":
                    print(f"Error: {data['message']}")
                    break
            else:
                # Binary frame: raw s16le PCM
                samples = struct.unpack(f"<{len(message)//2}h", message)
                # Process samples (save to file, play back, analyze, etc.)
                print(f"Received {len(samples)} audio samples")

# COMPUTER_ID is the UUID. INSTANCE_ID fills the /desktops/{instance_id} slot.
asyncio.run(stream_audio(
    os.environ["COMPUTER_ID"], os.environ["INSTANCE_ID"], os.environ["ORGO_API_KEY"]
))

interface AudioConfig {
  type: 'started';
  sample_rate: number;
  channels: number;
}

class AudioStream {
  private ws!: WebSocket;
  private ctx!: AudioContext;
  private nextPlayTime = 0;

  static async connect(computerId: string, instanceId: string, apiKey: string) {
    const stream = new AudioStream();

    // Get the computer password. This endpoint takes the UUID.
    const res = await fetch(
      `https://www.orgo.ai/api/computers/${computerId}/vnc-password`,
      { headers: { 'Authorization': `Bearer ${apiKey}` } }
    );
    const { password } = await res.json();

    // Connect
    stream.ctx = new AudioContext({ sampleRate: 24000 });
    stream.ws = new WebSocket(
      `wss://www.orgo.ai/desktops/${instanceId}/ws/audio?token=${password}`
    );
    stream.ws.binaryType = 'arraybuffer';

    stream.ws.onmessage = (event) => {
      if (typeof event.data === 'string') {
        const msg = JSON.parse(event.data);
        if (msg.type === 'started') {
          console.log(`Audio: ${msg.sample_rate}Hz, ${msg.channels}ch`);
        }
        if (msg.type === 'error') {
          console.error('Audio error:', msg.message);
        }
        return;
      }
      stream.playPCM(event.data);
    };

    return stream;
  }

  private playPCM(data: ArrayBuffer) {
    const pcm = new Int16Array(data);
    const floats = new Float32Array(pcm.length);
    for (let i = 0; i < pcm.length; i++) {
      floats[i] = pcm[i] / 32768;
    }

    const buffer = this.ctx.createBuffer(1, floats.length, 24000);
    buffer.getChannelData(0).set(floats);

    const source = this.ctx.createBufferSource();
    source.buffer = buffer;
    source.connect(this.ctx.destination);

    const now = this.ctx.currentTime;
    if (this.nextPlayTime < now || this.nextPlayTime > now + 0.15) {
      this.nextPlayTime = now + 0.02;
    }
    source.start(this.nextPlayTime);
    this.nextPlayTime += buffer.duration;
  }

  stop() {
    this.ws.send(JSON.stringify({ type: 'stop' }));
    this.ws.close();
    this.ctx.close();
  }
}

// COMPUTER_ID is the UUID. INSTANCE_ID fills the /desktops/{instance_id} slot.
const audio = await AudioStream.connect(
  process.env.COMPUTER_ID!,
  process.env.INSTANCE_ID!,
  process.env.ORGO_API_KEY!
);

// Later: stop streaming
audio.stop();