POST /v1/audio/speech

POST /v1/audio/transcriptions

{
  "text": "你好，欢迎来到我们的平台。"
}

{
  "text": "你好，欢迎来到我们的平台。",
  "usage": {
    "type": "tokens",
    "total_tokens": 34,
    "input_tokens": 30,
    "input_token_details": {
      "text_tokens": 0,
      "audio_tokens": 30
    },
    "output_tokens": 4
  }
}

{
  "task": "transcribe",
  "language": "chinese",
  "duration": 5.32,
  "text": "你好，欢迎来到我们的平台。",
  "segments": [
    {
      "id": 0,
      "seek": 0,
      "start": 0.0,
      "end": 2.5,
      "text": "你好，",
      "tokens": [1, 2, 3],
      "temperature": 0.0,
      "avg_logprob": -0.25,
      "compression_ratio": 0.8,
      "no_speech_prob": 0.01
    },
    {
      "id": 1,
      "seek": 250,
      "start": 2.5,
      "end": 5.32,
      "text": "欢迎来到我们的平台。",
      "tokens": [4, 5, 6, 7, 8],
      "temperature": 0.0,
      "avg_logprob": -0.18,
      "compression_ratio": 0.9,
      "no_speech_prob": 0.02
    }
  ]
}

1
00:00:00,000 --> 00:00:02,500
你好，

2
00:00:02,500 --> 00:00:05,320
欢迎来到我们的平台。

POST /v1/audio/translations

{
  "text": "Hello, welcome to our platform."
}

POST /v1/chat/completions

{
  "usage": {
    "prompt_tokens": 100,
    "completion_tokens": 200,
    "total_tokens": 300,
    "prompt_tokens_details": {
      "cached_tokens": 0,
      "text_tokens": 50,
      "audio_tokens": 50,
      "image_tokens": 0
    },
    "completion_tokens_details": {
      "text_tokens": 100,
      "audio_tokens": 100,
      "reasoning_tokens": 0
    }
  }
}

GET /v1/realtime (WebSocket Upgrade)

wss://cloud.dataeyes.ai/v1/realtime?model=gpt-4o-realtime-preview

Authorization: Bearer <您的key>
Sec-WebSocket-Protocol: realtime

{
  "event_id": "evt_001",
  "type": "session.update",
  "session": {
    "modalities": ["text", "audio"],
    "instructions": "你是一个友好的中英文翻译助手，用户说中文你翻译成英文，说英文你翻译成中文。",
    "voice": "alloy",
    "input_audio_format": "pcm16",
    "output_audio_format": "pcm16",
    "input_audio_transcription": {
      "model": "whisper-1"
    },
    "turn_detection": {
      "type": "server_vad",
      "threshold": 0.5,
      "prefix_padding_ms": 300,
      "silence_duration_ms": 500
    },
    "temperature": 0.8,
    "tools": [],
    "tool_choice": "auto"
  }
}

{
  "type": "input_audio_buffer.append",
  "audio": "<base64编码的PCM音频数据>"
}

{
  "type": "response.create",
  "response": {
    "modalities": ["text", "audio"]
  }
}

{
  "type": "session.created",
  "session": {
    "modalities": ["text", "audio"],
    "voice": "alloy",
    ...
  }
}

{
  "type": "response.audio.delta",
  "delta": "<base64编码的音频chunk>"
}

{
  "type": "response.audio_transcript.delta",
  "delta": "你好"
}

{
  "type": "response.done",
  "response": {
    "usage": {
      "total_tokens": 500,
      "input_tokens": 200,
      "output_tokens": 300,
      "input_token_details": {
        "cached_tokens": 0,
        "text_tokens": 50,
        "audio_tokens": 150
      },
      "output_token_details": {
        "text_tokens": 100,
        "audio_tokens": 200
      }
    }
  }
}

{
  "type": "error",
  "error": {
    "type": "invalid_request_error",
    "message": "Audio format not supported"
  }
}

客户端                                      服务端
  │                                           │
  │ ──── WebSocket Connect ───────────────→   │
  │ ←──── session.created ────────────────   │
  │                                           │
  │ ──── session.update ──────────────────→   │
  │ ←──── session.updated ────────────────   │
  │                                           │
  │ ──── input_audio_buffer.append ───────→   │
  │ ──── input_audio_buffer.append ───────→   │
  │ ──── input_audio_buffer.append ───────→   │
  │                                           │
  │ ──── response.create ─────────────────→   │
  │ ←──── response.audio.delta ───────────   │
  │ ←──── response.audio.delta ───────────   │
  │ ←──── response.audio_transcript.delta    │
  │ ←──── response.done ─────────────────   │
  │                                           │

{
  "error": {
    "message": "Invalid audio format. Supported formats: mp3, mp4, mpeg, mpga, m4a, wav, webm, ogg, flac",
    "type": "invalid_request_error",
    "code": "invalid_audio_format"
  }
}