diff --git a/raven/agent/tools/media_gen.py b/raven/agent/tools/media_gen.py index 6f64bae..6377b3a 100644 --- a/raven/agent/tools/media_gen.py +++ b/raven/agent/tools/media_gen.py @@ -24,6 +24,10 @@ to download) or ``failed`` (with ``error``). Default model ``kwaivgi/kling-v3.0-std`` (Kling v3 Standard). Requires postpaid billing / credits enabled on the OpenRouter account. +- MiniMax text-to-video models use the regional ``/v1/video_generation`` or + ``/v2/video_generation`` API, with the matching query and file retrieval + flow. The global endpoint is the default; set ``apiBase`` to the China + endpoint when needed. Generated files are written under ``/`` and the path is returned so the agent can forward it with the ``message`` tool's ``media`` @@ -54,6 +58,18 @@ from raven.config.schema import MediaToolConfig _DEFAULT_BASE = "https://openrouter.ai/api/v1" +_MINIMAX_GLOBAL_BASE = "https://api.minimax.io" +_MINIMAX_TEXT_VIDEO_MODELS = frozenset( + { + "MiniMax-H3", + "MiniMax-Hailuo-2.3", + "MiniMax-Hailuo-2.3-Fast", + "MiniMax-Hailuo-02", + "T2V-01-Director", + "T2V-01", + } +) +_MINIMAX_V2_MODEL = "MiniMax-H3" _EXT_MIME = { ".png": "image/png", @@ -493,8 +509,9 @@ class VideoGenerateTool(_OpenRouterMediaTool): "params": { "type": "object", "description": ( - "Optional extra provider params merged into the request, e.g. " - '{"duration": 5, "aspect_ratio": "16:9"} (3-15s; 16:9/9:16/1:1)' + "Optional provider-specific request fields. MiniMax supports " + "duration, resolution, ratio, callback_url, prompt_optimizer, " + "fast_pretreatment, and aigc_watermark where applicable." ), }, }, @@ -517,6 +534,9 @@ async def execute( return self._no_key_error() model_id = self._model(model) + if model_id in _MINIMAX_TEXT_VIDEO_MODELS: + return await self._execute_minimax(prompt, model_id, params) + body: dict[str, Any] = {"model": model_id, "prompt": prompt} if params: body.update(params) @@ -572,6 +592,154 @@ async def execute( logger.info("video_generate: {} bytes via {} -> {}", len(data), model_id, path) return json.dumps({"success": True, "model": model_id, "path": str(path)}, ensure_ascii=False) + def _minimax_api_base(self) -> str: + configured = getattr(self._config, "api_base", "") if self._config else "" + base = (configured or _MINIMAX_GLOBAL_BASE).rstrip("/") + for suffix in ("/v2/video_generation", "/v1/video_generation", "/v2", "/v1"): + if base.endswith(suffix): + return base[: -len(suffix)] + return base + + @staticmethod + def _minimax_error(payload: dict[str, Any]) -> str: + response = payload.get("base_resp") or {} + status_code = response.get("status_code") + if status_code in (None, 0): + return "" + return response.get("status_msg") or f"MiniMax API status {status_code}" + + async def _execute_minimax( + self, + prompt: str, + model_id: str, + params: dict[str, Any] | None, + ) -> str: + api_base = self._minimax_api_base() + headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"} + options = params or {} + is_v2 = model_id == _MINIMAX_V2_MODEL + if is_v2: + allowed = {"resolution", "duration", "ratio", "callback_url", "aigc_watermark"} + body: dict[str, Any] = { + "model": model_id, + "content": [{"type": "text", "text": prompt}], + "resolution": options.get("resolution", "2K"), + "duration": options.get("duration", 5), + } + body.update({key: value for key, value in options.items() if key in allowed}) + submit_url = f"{api_base}/v2/video_generation" + else: + allowed = {"prompt_optimizer", "fast_pretreatment", "duration", "resolution", "callback_url"} + body = {"model": model_id, "prompt": prompt} + body.update({key: value for key, value in options.items() if key in allowed}) + submit_url = f"{api_base}/v1/video_generation" + + try: + async with httpx.AsyncClient(proxy=self._proxy, timeout=120.0) as client: + response = await client.post(submit_url, headers=headers, json=body) + response.raise_for_status() + job = response.json() + if error := self._minimax_error(job): + return json.dumps({"error": error, "model": model_id}, ensure_ascii=False) + task_id = job.get("task_id") + if not task_id: + return json.dumps( + {"error": "video task response did not include task_id", "model": model_id}, + ensure_ascii=False, + ) + + status = await self._poll_minimax(client, api_base, str(task_id), headers, is_v2=is_v2) + if status.get("status") == "timeout": + return json.dumps({"error": "video job status=timeout", "model": model_id}, ensure_ascii=False) + if error := self._minimax_error(status): + return json.dumps({"error": error, "model": model_id}, ensure_ascii=False) + + if is_v2: + task = status.get("task") or {} + state = str(task.get("status", "")).lower() + video_url = (task.get("content") or {}).get("url") + detail = task.get("error") or status + succeeded = state == "succeeded" + else: + state = str(status.get("status", "")).lower() + detail = status + succeeded = state == "success" + video_url = None + if succeeded: + file_id = status.get("file_id") + file_response = await client.get( + f"{api_base}/v1/files/retrieve", + headers=headers, + params={"file_id": file_id}, + ) + file_response.raise_for_status() + file_payload = file_response.json() + if error := self._minimax_error(file_payload): + return json.dumps({"error": error, "model": model_id}, ensure_ascii=False) + video_url = (file_payload.get("file") or {}).get("download_url") + + if not succeeded: + return json.dumps( + {"error": f"video job status={state}", "detail": detail, "model": model_id}, + ensure_ascii=False, + ) + if not video_url: + return json.dumps( + {"error": "completed but no video URL found", "model": model_id}, + ensure_ascii=False, + ) + + video_origin = httpx.URL(video_url) + api_origin = httpx.URL(api_base) + same_origin = (video_origin.scheme, video_origin.host, video_origin.port) == ( + api_origin.scheme, + api_origin.host, + api_origin.port, + ) + download_headers = headers if same_origin else None + download = await client.get(video_url, headers=download_headers, timeout=180.0) + download.raise_for_status() + data = download.content + except httpx.HTTPStatusError as error: + return self._format_http_error(error) + except Exception as error: + logger.error("video_generate error: {}", error) + return json.dumps({"error": str(error)}, ensure_ascii=False) + + path = self._output_path("mp4") + path.write_bytes(data) + logger.info("video_generate: {} bytes via {} -> {}", len(data), model_id, path) + return json.dumps({"success": True, "model": model_id, "path": str(path)}, ensure_ascii=False) + + async def _poll_minimax( + self, + client: httpx.AsyncClient, + api_base: str, + task_id: str, + headers: dict[str, str], + *, + is_v2: bool, + ) -> dict[str, Any]: + waited = 0.0 + while waited < self._POLL_TIMEOUT_S: + if is_v2: + response = await client.get(f"{api_base}/v2/query/video_generation/{task_id}", headers=headers) + else: + response = await client.get( + f"{api_base}/v1/query/video_generation", + headers=headers, + params={"task_id": task_id}, + ) + response.raise_for_status() + job = response.json() + task = job.get("task") or {} + state = str(task.get("status") if is_v2 else job.get("status", "")).lower() + if state not in {"queued", "running", "preparing", "queueing", "processing"}: + return job + await asyncio.sleep(self._POLL_INTERVAL_S) + waited += self._POLL_INTERVAL_S + return {"status": "timeout"} + async def _poll(self, client: httpx.AsyncClient, poll_url: str, headers: dict[str, str]) -> dict[str, Any]: """Poll until the job leaves the pending/processing state or times out.""" waited = 0.0 diff --git a/raven/config/schema.py b/raven/config/schema.py index 5c53414..72a887b 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -735,9 +735,8 @@ class ExecToolConfig(Base): class MediaToolConfig(Base): """Config for a media-generation tool (key + base + model). - Empty fields fall back at call time: ``api_key`` → ``providers.openrouter`` - / ``OPENROUTER_API_KEY``; ``api_base`` → OpenRouter; ``model`` → the tool's - default (Nano Banana for images). + Empty fields fall back at call time according to the configured model and + tool backend. A missing model uses the tool's default. """ api_key: str = "" @@ -748,8 +747,8 @@ class MediaToolConfig(Base): class MediaGenConfig(Base): """Multimodal generation tools configuration. - OpenRouter is the only backend: image + speech via chat-completions output - modalities, and video via the async ``/videos`` endpoint (Kling). + Image, speech, and video tools resolve their backend from the configured + model. Video generation supports both regional MiniMax API endpoints. """ image: MediaToolConfig = Field(default_factory=MediaToolConfig) @@ -849,21 +848,29 @@ def effective_media_config(self) -> MediaGenConfig: A media tool (image/speech/video) counts as configured only when the user set its ``model`` or ``apiKey`` under ``tools.media.``. For - each configured tool we default a missing key to - ``providers.openrouter.apiKey`` so the chat key can be reused without - re-declaring it. Tools the user did not configure are left untouched - (no key, no model) — ``AgentLoop`` registers a media tool only when it - has a key or model, so an OpenRouter key set for chat alone never - surfaces image/speech/video to the agent. Returns a copy so this - resolution never mutates the raw config. + each configured tool we default a missing key from its matching provider + section so the key can be reused without re-declaring it. Tools the user + did not configure are left untouched (no key, no model) — ``AgentLoop`` + registers a media tool only when it has a key or model, so a provider key + set for chat alone never surfaces image/speech/video to the agent. Returns + a copy so this resolution never mutates the raw config. """ media = self.tools.media.model_copy(deep=True) openrouter = self.providers.get("openrouter") or_key = openrouter.api_key if openrouter else "" - for tool in (media.image, media.speech, media.video): + for tool in (media.image, media.speech): configured = bool(tool.api_key or tool.model) if configured and or_key and not tool.api_key: tool.api_key = or_key + + video_configured = bool(media.video.api_key or media.video.model) + uses_minimax = media.video.model.startswith(("MiniMax-", "T2V-")) + video_provider = self.providers.get("minimax" if uses_minimax else "openrouter") + if video_configured and video_provider: + if not media.video.api_key: + media.video.api_key = video_provider.effective_api_key + if uses_minimax and not media.video.api_base and video_provider.api_base: + media.video.api_base = video_provider.api_base return media def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | None", str | None]: diff --git a/tests/test_media_gen.py b/tests/test_media_gen.py new file mode 100644 index 0000000..a70c81a --- /dev/null +++ b/tests/test_media_gen.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from raven.agent.tools import media_gen as media_gen_module +from raven.agent.tools.media_gen import VideoGenerateTool +from raven.config.schema import Config, MediaToolConfig + +_VIDEO_BYTES = b"generated-video" + + +def _patch_client(monkeypatch: pytest.MonkeyPatch, handler) -> None: + real_client = httpx.AsyncClient + + def factory(*args, **kwargs): + kwargs.pop("proxy", None) + kwargs["transport"] = httpx.MockTransport(handler) + return real_client(*args, **kwargs) + + monkeypatch.setattr(media_gen_module.httpx, "AsyncClient", factory) + + +@pytest.mark.parametrize( + ("configured_base", "expected_base"), + [ + ("", "https://api.minimax.io"), + ("https://api.minimaxi.com/v2/video_generation", "https://api.minimaxi.com"), + ], +) +async def test_minimax_v2_text_to_video_uses_regional_endpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + configured_base: str, + expected_base: str, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.method == "POST": + assert str(request.url) == f"{expected_base}/v2/video_generation" + assert json.loads(request.content) == { + "model": "MiniMax-H3", + "content": [{"type": "text", "text": "A ship crosses the horizon"}], + "resolution": "2K", + "duration": 6, + "ratio": "16:9", + } + return httpx.Response(200, json={"task_id": "task-v2"}) + if request.url.host in {"api.minimax.io", "api.minimaxi.com"}: + assert str(request.url) == f"{expected_base}/v2/query/video_generation/task-v2" + return httpx.Response( + 200, + json={ + "task": { + "id": "task-v2", + "status": "succeeded", + "content": {"url": "https://cdn.example/video.mp4"}, + } + }, + ) + assert request.headers.get("authorization") is None + return httpx.Response(200, content=_VIDEO_BYTES) + + _patch_client(monkeypatch, handler) + tool = VideoGenerateTool( + MediaToolConfig(api_key="test-key", api_base=configured_base, model="MiniMax-H3"), + workspace=tmp_path, + ) + + result = json.loads( + await tool.execute( + prompt="A ship crosses the horizon", + params={"duration": 6, "ratio": "16:9", "ignored": True}, + ) + ) + + assert result["success"] is True + assert result["model"] == "MiniMax-H3" + assert Path(result["path"]).read_bytes() == _VIDEO_BYTES + assert len(requests) == 3 + + +async def test_minimax_v1_text_to_video_retrieves_generated_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected_base = "https://api.minimaxi.com" + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "POST": + assert str(request.url) == f"{expected_base}/v1/video_generation" + assert json.loads(request.content) == { + "model": "T2V-01", + "prompt": "Clouds gather above a mountain", + "duration": 6, + "resolution": "720P", + } + return httpx.Response( + 200, + json={"task_id": "task-v1", "base_resp": {"status_code": 0, "status_msg": "success"}}, + ) + if request.url.path == "/v1/query/video_generation": + assert request.url.params["task_id"] == "task-v1" + return httpx.Response( + 200, + json={ + "task_id": "task-v1", + "status": "Success", + "file_id": "file-v1", + "base_resp": {"status_code": 0, "status_msg": "success"}, + }, + ) + if request.url.path == "/v1/files/retrieve": + assert request.url.params["file_id"] == "file-v1" + return httpx.Response( + 200, + json={ + "file": {"download_url": "https://cdn.example/video.mp4"}, + "base_resp": {"status_code": 0, "status_msg": "success"}, + }, + ) + assert request.headers.get("authorization") is None + return httpx.Response(200, content=_VIDEO_BYTES) + + _patch_client(monkeypatch, handler) + tool = VideoGenerateTool( + MediaToolConfig( + api_key="test-key", + api_base="https://api.minimaxi.com/v1/video_generation", + model="T2V-01", + ), + workspace=tmp_path, + ) + + result = json.loads( + await tool.execute( + prompt="Clouds gather above a mountain", + params={"duration": 6, "resolution": "720P", "ratio": "16:9"}, + ) + ) + + assert result["success"] is True + assert result["model"] == "T2V-01" + assert Path(result["path"]).read_bytes() == _VIDEO_BYTES + + +async def test_minimax_v2_returns_task_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "POST": + return httpx.Response(200, json={"task_id": "failed-task"}) + return httpx.Response( + 200, + json={ + "task": { + "id": "failed-task", + "status": "failed", + "error": {"code": "1026", "message": "video description contains sensitive content"}, + } + }, + ) + + _patch_client(monkeypatch, handler) + tool = VideoGenerateTool( + MediaToolConfig(api_key="test-key", model="MiniMax-H3"), + workspace=tmp_path, + ) + + result = json.loads(await tool.execute(prompt="A prompt")) + + assert result["error"] == "video job status=failed" + assert result["detail"]["code"] == "1026" + + +def test_minimax_video_reuses_provider_endpoint_and_key() -> None: + config = Config.model_validate( + { + "providers": { + "minimax": { + "apiKey": "test-provider-key", + "apiBase": "https://api.minimaxi.com/v1", + } + }, + "tools": {"media": {"video": {"model": "MiniMax-H3"}}}, + } + ) + + video = config.effective_media_config().video + + assert video.api_key == "test-provider-key" + assert video.api_base == "https://api.minimaxi.com/v1"