From a60b71310645a7059777af17ded64d13a0a20e09 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:28:42 +0800 Subject: [PATCH] feat(tools): add MiniMax speech generation --- raven/agent/tools/media_gen.py | 265 ++++++++++++++++++++++++- raven/config/schema.py | 36 ++-- tests/test_media_gen_speech_minimax.py | 184 +++++++++++++++++ 3 files changed, 468 insertions(+), 17 deletions(-) create mode 100644 tests/test_media_gen_speech_minimax.py diff --git a/raven/agent/tools/media_gen.py b/raven/agent/tools/media_gen.py index 6f64baeb..3ecf6304 100644 --- a/raven/agent/tools/media_gen.py +++ b/raven/agent/tools/media_gen.py @@ -1,4 +1,4 @@ -"""Multimodal generation tools (OpenRouter backend). +"""Multimodal generation tools. OpenRouter exposes generation through the OpenAI-compatible chat-completions API with extra *output modalities* — there are no standalone @@ -17,6 +17,14 @@ only when ``ffmpeg`` is on PATH; otherwise we keep the WAV and say so. Default model ``openai/gpt-audio-mini``. +MiniMax speech uses its dedicated HTTP API: + +- ``text_to_speech`` → ``POST {base}/t2a_v2`` with ``{model, text, ...}``. + The global endpoint is used by default; setting ``tools.media.speech.apiBase`` + selects the China endpoint. Audio is returned through ``data.audio`` as + hex bytes or a short-lived URL, ``data.status`` reports generation state, + and success is reported by ``base_resp.status_code``. + Video uses a separate async endpoint (NOT chat-completions): - ``video_generate`` → ``POST {base}/videos`` with ``{model, prompt}`` → ``202`` @@ -54,6 +62,26 @@ from raven.config.schema import MediaToolConfig _DEFAULT_BASE = "https://openrouter.ai/api/v1" +_MINIMAX_SPEECH_ENDPOINTS = { + "global_en": "https://api.minimax.io/v1/t2a_v2", + "cn_zh": "https://api.minimaxi.com/v1/t2a_v2", +} +_MINIMAX_SPEECH_DEFAULT_ENDPOINT = _MINIMAX_SPEECH_ENDPOINTS["global_en"] +_MINIMAX_SPEECH_MODELS = frozenset( + { + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", + } +) +_MINIMAX_SPEECH_DEFAULT_MODEL = "speech-2.8-hd" +_MINIMAX_SPEECH_AUDIO_FORMATS = frozenset({"mp3", "wav", "flac", "pcm"}) +_MINIMAX_SPEECH_HOSTS = tuple(endpoint.split("/")[2] for endpoint in _MINIMAX_SPEECH_ENDPOINTS.values()) _EXT_MIME = { ".png": "image/png", @@ -248,7 +276,7 @@ async def execute( class SpeechGenerateTool(_OpenRouterMediaTool): - """Synthesize speech from text via an OpenRouter audio model (gpt-audio).""" + """Synthesize speech from text through a configured speech backend.""" name = "text_to_speech" default_model = "openai/gpt-audio-mini" @@ -279,6 +307,19 @@ class SpeechGenerateTool(_OpenRouterMediaTool): "type": "string", "description": "Optional OpenRouter audio model override (e.g. openai/gpt-audio)", }, + "stream": {"type": "boolean", "default": False, "description": "Stream generated audio"}, + "language_boost": {"type": "string", "description": "Optional language recognition hint"}, + "output_format": { + "type": "string", + "enum": ["hex", "url"], + "default": "hex", + "description": "MiniMax response encoding; streaming requires hex", + }, + "voice_setting": {"type": "object", "description": "MiniMax voice settings"}, + "pronunciation_dict": {"type": "object", "description": "MiniMax pronunciation replacements"}, + "audio_setting": {"type": "object", "description": "MiniMax audio encoding settings"}, + "voice_modify": {"type": "object", "description": "MiniMax voice effect settings"}, + "subtitle_enable": {"type": "boolean", "description": "Include subtitle metadata"}, }, "required": ["text"], } @@ -295,8 +336,31 @@ async def execute( voice: str = "alloy", format: str = "wav", model: str | None = None, + stream: bool = False, + language_boost: str | None = None, + output_format: str = "hex", + voice_setting: dict[str, Any] | None = None, + pronunciation_dict: dict[str, Any] | None = None, + audio_setting: dict[str, Any] | None = None, + voice_modify: dict[str, Any] | None = None, + subtitle_enable: bool | None = None, **kwargs: Any, ) -> str: + minimax_model = self._minimax_model(model) + if minimax_model: + return await self._execute_minimax( + text=text, + model_id=minimax_model, + stream=stream, + language_boost=language_boost, + output_format=output_format, + voice_setting=voice_setting, + pronunciation_dict=pronunciation_dict, + audio_setting=audio_setting, + voice_modify=voice_modify, + subtitle_enable=subtitle_enable, + ) + if not self.api_key: return self._no_key_error() @@ -349,6 +413,203 @@ async def execute( result["note"] = note return json.dumps(result, ensure_ascii=False) + def _minimax_model(self, override: str | None) -> str | None: + cfg_model = getattr(self._config, "model", "") if self._config else "" + candidate = override or cfg_model + bare_model = candidate.rsplit("/", 1)[-1] if candidate else "" + if bare_model in _MINIMAX_SPEECH_MODELS: + return bare_model + + cfg_base = getattr(self._config, "api_base", "") if self._config else "" + if any(host in cfg_base.lower() for host in _MINIMAX_SPEECH_HOSTS): + return candidate or _MINIMAX_SPEECH_DEFAULT_MODEL + return None + + @property + def _minimax_api_key(self) -> str: + cfg_key = getattr(self._config, "api_key", "") if self._config else "" + return cfg_key or os.environ.get("MINIMAX_API_KEY", "") + + @property + def _minimax_endpoint(self) -> str: + cfg_base = getattr(self._config, "api_base", "") if self._config else "" + if not cfg_base: + return _MINIMAX_SPEECH_DEFAULT_ENDPOINT + base = cfg_base.rstrip("/") + return base if base.endswith("/t2a_v2") else f"{base}/t2a_v2" + + def _no_minimax_key_error(self) -> str: + return json.dumps( + { + "error": ( + "text_to_speech: no API key configured. Set it under " + "tools.media.speech.apiKey or providers.minimax.apiKey, " + "or export MINIMAX_API_KEY, then restart the gateway." + ) + }, + ensure_ascii=False, + ) + + async def _execute_minimax( + self, + *, + text: str, + model_id: str, + stream: bool, + language_boost: str | None, + output_format: str, + voice_setting: dict[str, Any] | None, + pronunciation_dict: dict[str, Any] | None, + audio_setting: dict[str, Any] | None, + voice_modify: dict[str, Any] | None, + subtitle_enable: bool | None, + ) -> str: + if not self._minimax_api_key: + return self._no_minimax_key_error() + if output_format not in {"hex", "url"}: + return json.dumps({"error": f"unsupported output_format: {output_format}"}, ensure_ascii=False) + if stream and output_format != "hex": + return json.dumps({"error": "MiniMax streaming requires output_format='hex'"}, ensure_ascii=False) + + audio = dict(audio_setting or {}) + audio_format = str(audio.get("format") or "mp3").lower() + if audio_format not in _MINIMAX_SPEECH_AUDIO_FORMATS: + supported = ", ".join(sorted(_MINIMAX_SPEECH_AUDIO_FORMATS)) + return json.dumps( + {"error": f"unsupported MiniMax audio format: {audio_format}; expected one of {supported}"}, + ensure_ascii=False, + ) + if stream and audio_format != "mp3": + return json.dumps({"error": "MiniMax streaming supports only mp3 audio"}, ensure_ascii=False) + audio["format"] = audio_format + + payload: dict[str, Any] = { + "model": model_id, + "text": text, + "stream": stream, + "output_format": output_format, + "audio_setting": audio, + } + if voice_setting is not None: + payload["voice_setting"] = dict(voice_setting) + if language_boost is not None: + payload["language_boost"] = language_boost + if pronunciation_dict is not None: + payload["pronunciation_dict"] = pronunciation_dict + if voice_modify is not None: + payload["voice_modify"] = voice_modify + if subtitle_enable is not None: + payload["subtitle_enable"] = subtitle_enable + + headers = { + "Authorization": f"Bearer {self._minimax_api_key}", + "Content-Type": "application/json", + } + try: + async with httpx.AsyncClient(proxy=self._proxy, timeout=180.0) as client: + if stream: + response = await self._stream_minimax_response(client, headers, payload) + else: + r = await client.post(self._minimax_endpoint, headers=headers, json=payload) + r.raise_for_status() + response = r.json() + + if error := self._minimax_response_error(response): + return json.dumps({"error": error, "model": model_id}, ensure_ascii=False) + data = response.get("data") or {} + audio_value = data.get("audio") + if not audio_value: + status = data.get("status") + message = f"speech generation status={status}" if status is not None else "no audio returned" + return json.dumps({"error": message, "model": model_id}, ensure_ascii=False) + + reported_format = str((response.get("extra_info") or {}).get("audio_format") or audio_format).lower() + saved_format = reported_format if reported_format in _MINIMAX_SPEECH_AUDIO_FORMATS else audio_format + if output_format == "url": + out_path = await self._download_minimax_audio(client, audio_value, saved_format) + else: + if not isinstance(audio_value, str): + raise ValueError("MiniMax data.audio must be a hex string") + out_path = self._output_path(saved_format) + out_path.write_bytes(bytes.fromhex(audio_value)) + except httpx.HTTPStatusError as e: + return self._format_http_error(e) + except Exception as e: + logger.error("text_to_speech MiniMax error: {}", e) + return json.dumps({"error": str(e)}, ensure_ascii=False) + + logger.info("text_to_speech: via {} -> {}", model_id, out_path) + return json.dumps( + {"success": True, "model": model_id, "path": str(out_path), "format": saved_format}, + ensure_ascii=False, + ) + + async def _stream_minimax_response( + self, + client: httpx.AsyncClient, + headers: dict[str, str], + payload: dict[str, Any], + ) -> dict[str, Any]: + audio_parts: list[str] = [] + base_resp: dict[str, Any] = {} + extra_info: dict[str, Any] = {} + status: Any = None + + async with client.stream("POST", self._minimax_endpoint, headers=headers, json=payload) as r: + if r.status_code >= 400: + await r.aread() + r.raise_for_status() + async for line in r.aiter_lines(): + raw = line.strip() + if raw.startswith("data:"): + raw = raw[len("data:") :].strip() + if not raw or raw == "[DONE]": + continue + try: + obj = json.loads(raw) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + if isinstance(obj.get("base_resp"), dict): + base_resp = obj["base_resp"] + if isinstance(obj.get("extra_info"), dict): + extra_info = obj["extra_info"] + data = obj.get("data") or {} + if isinstance(data, dict): + if isinstance(data.get("audio"), str): + audio_parts.append(data["audio"]) + if "status" in data: + status = data["status"] + + return { + "data": {"audio": "".join(audio_parts), "status": status}, + "base_resp": base_resp, + "extra_info": extra_info, + } + + @staticmethod + def _minimax_response_error(response: dict[str, Any]) -> str: + base_resp = response.get("base_resp") or {} + status_code = base_resp.get("status_code") + if status_code in (None, 0): + return "" + return str(base_resp.get("status_msg") or f"status_code={status_code}") + + async def _download_minimax_audio(self, client: httpx.AsyncClient, audio: Any, audio_format: str) -> Path: + urls = audio if isinstance(audio, list) else [audio] + url = next( + (item for item in urls if isinstance(item, str) and item.startswith(("http://", "https://"))), + "", + ) + if not url: + raise ValueError("MiniMax data.audio did not contain a downloadable URL") + r = await client.get(url, timeout=180.0) + r.raise_for_status() + out_path = self._output_path(audio_format) + out_path.write_bytes(r.content) + return out_path + async def _stream_audio_pcm(self, payload: dict[str, Any]) -> tuple[bytes, str]: """Stream a chat-completions audio response → (pcm16 bytes, transcript). diff --git a/raven/config/schema.py b/raven/config/schema.py index 5c53414e..427ba000 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -735,21 +735,20 @@ class ExecToolConfig(Base): class MediaToolConfig(Base): """Config for a media-generation tool (key + base + model). - Empty fields fall back at call time: ``api_key`` → ``providers.openrouter`` - / ``OPENROUTER_API_KEY``; ``api_base`` → OpenRouter; ``model`` → the tool's - default (Nano Banana for images). + Empty fields fall back at call time to the selected backend's provider + configuration, environment variables, endpoint, and default model. """ api_key: str = "" - api_base: str = "" # defaults to https://openrouter.ai/api/v1 + api_base: str = "" model: str = "" class MediaGenConfig(Base): """Multimodal generation tools configuration. - OpenRouter is the only backend: image + speech via chat-completions output - modalities, and video via the async ``/videos`` endpoint (Kling). + Speech supports MiniMax's dedicated ``/v1/t2a_v2`` endpoint in addition + to the existing media generation routes. """ image: MediaToolConfig = Field(default_factory=MediaToolConfig) @@ -848,22 +847,29 @@ def effective_media_config(self) -> MediaGenConfig: """Media config resolved for registration and auth. A media tool (image/speech/video) counts as configured only when the - user set its ``model`` or ``apiKey`` under ``tools.media.``. For - each configured tool we default a missing key to - ``providers.openrouter.apiKey`` so the chat key can be reused without - re-declaring it. Tools the user did not configure are left untouched - (no key, no model) — ``AgentLoop`` registers a media tool only when it - has a key or model, so an OpenRouter key set for chat alone never - surfaces image/speech/video to the agent. Returns a copy so this - resolution never mutates the raw config. + user set its model, API key, or a supported endpoint. MiniMax speech + models and regional endpoints inherit ``providers.minimax.apiKey`` + when the tool-specific key is empty. Returns a copy so this resolution + never mutates the raw config. """ media = self.tools.media.model_copy(deep=True) openrouter = self.providers.get("openrouter") or_key = openrouter.api_key if openrouter else "" - for tool in (media.image, media.speech, media.video): + for tool in (media.image, media.video): configured = bool(tool.api_key or tool.model) if configured and or_key and not tool.api_key: tool.api_key = or_key + + speech = media.speech + speech_model = speech.model.rsplit("/", 1)[-1] + minimax_speech = speech_model.startswith("speech-") or any( + host in speech.api_base.lower() for host in ("api.minimax.io", "api.minimaxi.com") + ) + speech_configured = bool(speech.api_key or speech.model or speech.api_base) + if speech_configured and not speech.api_key: + provider = self.providers.get("minimax") if minimax_speech else openrouter + if provider and provider.api_key: + speech.api_key = provider.api_key return media def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | None", str | None]: diff --git a/tests/test_media_gen_speech_minimax.py b/tests/test_media_gen_speech_minimax.py new file mode 100644 index 00000000..da6dee68 --- /dev/null +++ b/tests/test_media_gen_speech_minimax.py @@ -0,0 +1,184 @@ +"""Unit tests for MiniMax speech generation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from raven.agent.tools import media_gen as mg +from raven.agent.tools.media_gen import SpeechGenerateTool +from raven.config.schema import Config, MediaToolConfig + + +def _patch_client(monkeypatch: pytest.MonkeyPatch, handler) -> None: + real_client = httpx.AsyncClient + + def factory(*args, **kwargs): + return real_client(transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(mg.httpx, "AsyncClient", factory) + + +async def test_minimax_global_hex_request_and_response(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == httpx.URL("https://api.minimax.io/v1/t2a_v2") + assert request.headers["Authorization"] == "Bearer test-key" + body = json.loads(request.content) + assert body == { + "model": "speech-2.8-hd", + "text": "Hello", + "stream": False, + "output_format": "hex", + "audio_setting": {"format": "wav", "sample_rate": 32000}, + "voice_setting": {"voice_id": "English_Graceful_Lady", "speed": 1.1}, + "language_boost": "English", + "pronunciation_dict": {"tone": ["Raven/(rei)ven"]}, + "voice_modify": {"pitch": 2}, + "subtitle_enable": True, + } + return httpx.Response( + 200, + json={ + "data": {"audio": "52494646", "status": 2}, + "extra_info": {"audio_format": "wav"}, + "base_resp": {"status_code": 0}, + }, + ) + + _patch_client(monkeypatch, handler) + tool = SpeechGenerateTool( + MediaToolConfig(api_key="test-key", model="speech-2.8-hd"), + workspace=tmp_path, + ) + result = json.loads( + await tool.execute( + text="Hello", + voice_setting={"voice_id": "English_Graceful_Lady", "speed": 1.1}, + language_boost="English", + pronunciation_dict={"tone": ["Raven/(rei)ven"]}, + audio_setting={"format": "wav", "sample_rate": 32000}, + voice_modify={"pitch": 2}, + subtitle_enable=True, + ) + ) + + assert result["success"] is True + assert result["model"] == "speech-2.8-hd" + path = Path(result["path"]) + assert path.suffix == ".wav" + assert path.read_bytes() == bytes.fromhex("52494646") + + +async def test_minimax_cn_url_response_is_downloaded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + audio_bytes = b"ID3 mini audio" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "api.minimaxi.com": + assert request.url.path == "/v1/t2a_v2" + body = json.loads(request.content) + assert body["model"] == "speech-2.8-hd" + assert body["output_format"] == "url" + return httpx.Response( + 200, + json={ + "data": {"audio": "https://cdn.example.com/speech.mp3", "status": 2}, + "base_resp": {"status_code": 0}, + }, + ) + if request.url.host == "cdn.example.com": + return httpx.Response(200, content=audio_bytes) + return httpx.Response(404) + + _patch_client(monkeypatch, handler) + tool = SpeechGenerateTool( + MediaToolConfig( + api_key="test-key", + api_base="https://api.minimaxi.com/v1", + ), + workspace=tmp_path, + ) + result = json.loads( + await tool.execute( + text="Hello from the China endpoint", + output_format="url", + audio_setting={"format": "mp3"}, + ) + ) + + assert result["success"] is True + path = Path(result["path"]) + assert path.suffix == ".mp3" + assert path.read_bytes() == audio_bytes + + +async def test_minimax_stream_concatenates_hex_audio(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["stream"] is True + assert body["output_format"] == "hex" + events = ( + 'data: {"data":{"audio":"0102","status":1},"base_resp":{"status_code":0}}\n\n' + 'data: {"data":{"audio":"0304","status":2},"base_resp":{"status_code":0}}\n\n' + "data: [DONE]\n\n" + ) + return httpx.Response(200, content=events.encode()) + + _patch_client(monkeypatch, handler) + tool = SpeechGenerateTool( + MediaToolConfig(api_key="test-key", model="speech-2.6-hd"), + workspace=tmp_path, + ) + result = json.loads( + await tool.execute( + text="Stream this", + stream=True, + output_format="hex", + audio_setting={"format": "mp3"}, + ) + ) + + assert result["success"] is True + assert Path(result["path"]).read_bytes() == bytes.fromhex("01020304") + + +async def test_minimax_api_error_uses_base_response(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "data": None, + "base_resp": {"status_code": 1004, "status_msg": "Authentication failed"}, + }, + ) + + _patch_client(monkeypatch, handler) + tool = SpeechGenerateTool( + MediaToolConfig(api_key="test-key", model="speech-02-hd"), + workspace=tmp_path, + ) + result = json.loads(await tool.execute(text="Hello")) + + assert result["error"] == "Authentication failed" + + +@pytest.mark.parametrize( + "speech_config", + [ + {"model": "speech-2.8-hd"}, + {"apiBase": "https://api.minimaxi.com/v1"}, + ], +) +def test_minimax_speech_inherits_provider_key(speech_config: dict[str, str]) -> None: + config = Config.model_validate( + { + "providers": {"minimax": {"apiKey": "test-key"}}, + "tools": {"media": {"speech": speech_config}}, + } + ) + + resolved = config.effective_media_config().speech + + assert resolved.api_key == "test-key"