From b900c670d46d17b73af9a21fd791a035e2755de4 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:01:54 +0800 Subject: [PATCH] feat(tools): add MiniMax image generation backend --- raven/agent/tools/media_gen.py | 207 ++++++++++++++++++++++++++++++++- raven/config/schema.py | 34 ++++-- tests/test_media_gen_tool.py | 127 ++++++++++++++++++++ 3 files changed, 356 insertions(+), 12 deletions(-) create mode 100644 tests/test_media_gen_tool.py diff --git a/raven/agent/tools/media_gen.py b/raven/agent/tools/media_gen.py index 6f64baeb..3e2e1a76 100644 --- a/raven/agent/tools/media_gen.py +++ b/raven/agent/tools/media_gen.py @@ -1,4 +1,4 @@ -"""Multimodal generation tools (OpenRouter backend). +"""Multimodal generation tools. OpenRouter exposes generation through the OpenAI-compatible chat-completions API with extra *output modalities* — there are no standalone @@ -9,6 +9,8 @@ - ``image_generate`` → ``modalities:["image","text"]`` → ``message.images[].image_url.url`` (a ``data:image/...;base64,...`` URI). Default model **Nano Banana** (``google/gemini-2.5-flash-image``). +- ``image_generate`` can instead call MiniMax ``/v1/image_generation`` in the + global or China region and persist URL or base64 results. - ``text_to_speech`` → ``modalities:["audio","text"]`` + ``audio:{voice,format}``. OpenRouter only returns audio when ``stream:true`` AND only as raw ``pcm16`` in that mode, so we stream, concatenate the base64 ``delta.audio.data`` chunks, @@ -51,9 +53,13 @@ from raven.utils.helpers import image_block if TYPE_CHECKING: - from raven.config.schema import MediaToolConfig + from raven.config.schema import ImageToolConfig, MediaToolConfig _DEFAULT_BASE = "https://openrouter.ai/api/v1" +_MINIMAX_IMAGE_BASES = { + "global": "https://api.minimax.io/v1", + "cn": "https://api.minimaxi.com/v1", +} _EXT_MIME = { ".png": "image/png", @@ -145,7 +151,7 @@ def _format_http_error(self, e: httpx.HTTPStatusError) -> str: class ImageGenerateTool(_OpenRouterMediaTool): - """Generate (or edit) an image from a text prompt via Nano Banana on OpenRouter.""" + """Generate an image from a text prompt through the configured image API.""" name = "image_generate" default_model = "google/gemini-2.5-flash-image" # Nano Banana @@ -173,10 +179,84 @@ class ImageGenerateTool(_OpenRouterMediaTool): "Optional input images to edit/vary: local file paths, http(s) URLs, or data: URIs (max 6)" ), }, + "aspect_ratio": { + "type": "string", + "enum": ["1:1", "16:9", "4:3", "3:2", "2:3", "3:4", "9:16", "21:9"], + "description": "Optional output aspect ratio for MiniMax image generation", + }, + "width": {"type": "integer", "description": "Optional output width in pixels"}, + "height": {"type": "integer", "description": "Optional output height in pixels"}, + "response_format": { + "type": "string", + "enum": ["url", "base64"], + "default": "url", + "description": "MiniMax image response format", + }, + "seed": {"type": "integer", "description": "Optional deterministic generation seed"}, + "n": { + "type": "integer", + "minimum": 1, + "maximum": 9, + "default": 1, + "description": "Number of images to generate", + }, + "prompt_optimizer": { + "type": "boolean", + "default": False, + "description": "Whether MiniMax should optimize the prompt", + }, }, "required": ["prompt"], } + def __init__( + self, + config: "ImageToolConfig | None" = None, + *, + workspace: Path | None = None, + proxy: str | None = None, + output_subdir: str = "generated", + ): + super().__init__(config, workspace=workspace, proxy=proxy, output_subdir=output_subdir) + + @property + def _uses_minimax(self) -> bool: + return bool(self._config and getattr(self._config, "provider", "") == "minimax") + + @property + def api_key(self) -> str: + if not self._uses_minimax: + return super().api_key + cfg_key = getattr(self._config, "api_key", "") if self._config else "" + return cfg_key or os.environ.get("MINIMAX_API_KEY", "") + + @property + def api_base(self) -> str: + if not self._uses_minimax: + return super().api_base + cfg_base = getattr(self._config, "api_base", "") if self._config else "" + region = getattr(self._config, "region", "global") if self._config else "global" + return (cfg_base or _MINIMAX_IMAGE_BASES[region]).rstrip("/") + + def _model(self, override: str | None) -> str: + if not self._uses_minimax: + return super()._model(override) + cfg_model = getattr(self._config, "model", "") if self._config else "" + return override or cfg_model or "image-01" + + def _no_key_error(self) -> str: + if not self._uses_minimax: + return super()._no_key_error() + return json.dumps( + { + "error": ( + "image_generate: no API key configured. Set tools.media.image.apiKey, " + "providers.minimax.apiKey, or MINIMAX_API_KEY, then restart the gateway." + ) + }, + ensure_ascii=False, + ) + def _image_part(self, ref: str) -> dict[str, Any]: """Build an OpenAI-style image_url content part from a path/URL/data URI.""" if ref.startswith(("http://", "https://", "data:")): @@ -192,12 +272,32 @@ async def execute( prompt: str, model: str | None = None, images: list[str] | None = None, + aspect_ratio: str | None = None, + width: int | None = None, + height: int | None = None, + response_format: str = "url", + seed: int | None = None, + n: int = 1, + prompt_optimizer: bool = False, **kwargs: Any, ) -> str: if not self.api_key: return self._no_key_error() model_id = self._model(model) + if self._uses_minimax: + return await self._execute_minimax( + prompt=prompt, + model=model_id, + images=images, + aspect_ratio=aspect_ratio, + width=width, + height=height, + response_format=response_format, + seed=seed, + n=n, + prompt_optimizer=prompt_optimizer, + ) if images: content: Any = [{"type": "text", "text": prompt}] try: @@ -246,6 +346,107 @@ async def execute( logger.info("image_generate: {} image(s) via {} -> {}", len(paths), model_id, paths) return json.dumps({"success": True, "model": model_id, "paths": paths}, ensure_ascii=False) + async def _execute_minimax( + self, + *, + prompt: str, + model: str, + images: list[str] | None, + aspect_ratio: str | None, + width: int | None, + height: int | None, + response_format: str, + seed: int | None, + n: int, + prompt_optimizer: bool, + ) -> str: + if images: + return json.dumps( + {"error": "input images are not supported by the configured text-to-image API"}, + ensure_ascii=False, + ) + payload: dict[str, Any] = { + "model": model, + "prompt": prompt, + "response_format": response_format, + "n": n, + "prompt_optimizer": prompt_optimizer, + } + optional = { + "aspect_ratio": aspect_ratio, + "width": width, + "height": height, + "seed": seed, + } + payload.update({key: value for key, value in optional.items() if value is not None}) + + headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"} + try: + async with httpx.AsyncClient(proxy=self._proxy, timeout=180.0) as client: + response = await client.post( + f"{self.api_base}/image_generation", + headers=headers, + json=payload, + ) + response.raise_for_status() + data = response.json() + base_resp = data.get("base_resp") or {} + if base_resp.get("status_code", 0) != 0: + return json.dumps( + { + "error": base_resp.get("status_msg") or "image generation failed", + "status_code": base_resp.get("status_code"), + "model": model, + }, + ensure_ascii=False, + ) + output = data.get("data") or {} + paths = await self._save_minimax_images( + client, + output.get("image_urls") or [], + output.get("image_base64") or [], + ) + except httpx.HTTPStatusError as e: + return self._format_http_error(e) + except Exception as e: + logger.error("image_generate error: {}", e) + return json.dumps({"error": str(e)}, ensure_ascii=False) + + if not paths: + return json.dumps( + {"error": "no image returned", "model": model, "metadata": data.get("metadata") or {}}, + ensure_ascii=False, + ) + logger.info("image_generate: {} image(s) via {} -> {}", len(paths), model, paths) + return json.dumps( + { + "success": True, + "model": model, + "paths": paths, + "metadata": data.get("metadata") or {}, + }, + ensure_ascii=False, + ) + + async def _save_minimax_images( + self, + client: httpx.AsyncClient, + urls: list[str], + encoded_images: list[str], + ) -> list[str]: + paths: list[str] = [] + for encoded in encoded_images: + path = self._output_path("png") + path.write_bytes(base64.b64decode(encoded)) + paths.append(str(path)) + for url in urls: + download = await client.get(url) + download.raise_for_status() + path = self._output_path("png") + path.write_bytes(download.content) + paths.append(str(path)) + return paths + class SpeechGenerateTool(_OpenRouterMediaTool): """Synthesize speech from text via an OpenRouter audio model (gpt-audio).""" diff --git a/raven/config/schema.py b/raven/config/schema.py index 5c53414e..f15e4eda 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -745,14 +745,21 @@ class MediaToolConfig(Base): model: str = "" +class ImageToolConfig(MediaToolConfig): + """Image generation config with an explicit API protocol and region.""" + + provider: Literal["openrouter", "minimax"] = "openrouter" + region: Literal["global", "cn"] = "global" + + class MediaGenConfig(Base): """Multimodal generation tools configuration. - OpenRouter is the only backend: image + speech via chat-completions output - modalities, and video via the async ``/videos`` endpoint (Kling). + Image generation can use MiniMax directly; the other configured protocols + use their existing media endpoints. """ - image: MediaToolConfig = Field(default_factory=MediaToolConfig) + image: ImageToolConfig = Field(default_factory=ImageToolConfig) speech: MediaToolConfig = Field(default_factory=MediaToolConfig) video: MediaToolConfig = Field(default_factory=MediaToolConfig) proxy: str | None = None # HTTP/SOCKS proxy for media API calls @@ -847,11 +854,11 @@ def workspace_path(self) -> Path: def effective_media_config(self) -> MediaGenConfig: """Media config resolved for registration and auth. - A media tool (image/speech/video) counts as configured only when the - user set its ``model`` or ``apiKey`` under ``tools.media.``. For - each configured tool we default a missing key to - ``providers.openrouter.apiKey`` so the chat key can be reused without - re-declaring it. Tools the user did not configure are left untouched + A media tool (image/speech/video) counts as configured when the user + selects its provider or sets ``model`` or ``apiKey`` under + ``tools.media.``. A configured image tool resolves the key for its + selected provider, while the other media tools retain their existing key + fallback. Tools the user did not configure are left untouched (no key, no model) — ``AgentLoop`` registers a media tool only when it has a key or model, so an OpenRouter key set for chat alone never surfaces image/speech/video to the agent. Returns a copy so this @@ -860,7 +867,16 @@ def effective_media_config(self) -> MediaGenConfig: media = self.tools.media.model_copy(deep=True) openrouter = self.providers.get("openrouter") or_key = openrouter.api_key if openrouter else "" - for tool in (media.image, media.speech, media.video): + image_configured = bool(media.image.api_key or media.image.model or media.image.provider != "openrouter") + if image_configured and media.image.provider == "minimax" and not media.image.model: + media.image.model = "image-01" + if image_configured and not media.image.api_key: + if media.image.provider == "minimax": + minimax = self.providers.get("minimax") + media.image.api_key = minimax.api_key if minimax else "" + elif or_key: + media.image.api_key = or_key + for tool in (media.speech, media.video): configured = bool(tool.api_key or tool.model) if configured and or_key and not tool.api_key: tool.api_key = or_key diff --git a/tests/test_media_gen_tool.py b/tests/test_media_gen_tool.py new file mode 100644 index 00000000..83d2381f --- /dev/null +++ b/tests/test_media_gen_tool.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import httpx + +from raven.agent.tools import media_gen +from raven.agent.tools.media_gen import ImageGenerateTool +from raven.config.schema import Config, ImageToolConfig + + +def _patch_client(monkeypatch, handler) -> None: + real_client = httpx.AsyncClient + + def factory(*_args, **_kwargs): + return real_client(transport=httpx.MockTransport(handler)) + + monkeypatch.setattr(media_gen.httpx, "AsyncClient", factory) + + +async def test_minimax_image_generation_uses_global_endpoint_and_base64(tmp_path: Path, monkeypatch) -> None: + image = b"generated-image" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == "https://api.minimax.io/v1/image_generation" + assert request.headers["Authorization"] == "Bearer test-key" + payload = json.loads(request.content) + assert payload == { + "model": "image-01", + "prompt": "a lighthouse", + "response_format": "base64", + "n": 2, + "prompt_optimizer": True, + "aspect_ratio": "16:9", + "seed": 7, + } + return httpx.Response( + 200, + json={ + "data": {"image_base64": [base64.b64encode(image).decode("ascii")]}, + "metadata": {"success_count": 1, "failed_count": 0}, + "base_resp": {"status_code": 0, "status_msg": "success"}, + }, + ) + + _patch_client(monkeypatch, handler) + config = ImageToolConfig(provider="minimax", api_key="test-key", model="image-01") + tool = ImageGenerateTool(config, workspace=tmp_path) + + result = json.loads( + await tool.execute( + prompt="a lighthouse", + aspect_ratio="16:9", + response_format="base64", + seed=7, + n=2, + prompt_optimizer=True, + ) + ) + + assert result["success"] is True + assert result["model"] == "image-01" + assert result["metadata"] == {"success_count": 1, "failed_count": 0} + assert Path(result["paths"][0]).read_bytes() == image + + +async def test_minimax_image_generation_uses_cn_endpoint_and_downloads_url(tmp_path: Path, monkeypatch) -> None: + image_url = "https://example.test/generated.png" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url == "https://api.minimaxi.com/v1/image_generation": + return httpx.Response( + 200, + json={ + "data": {"image_urls": [image_url]}, + "metadata": {"success_count": 1, "failed_count": 0}, + "base_resp": {"status_code": 0, "status_msg": "success"}, + }, + ) + assert request.url == image_url + assert "Authorization" not in request.headers + return httpx.Response(200, content=b"downloaded-image") + + _patch_client(monkeypatch, handler) + config = ImageToolConfig(provider="minimax", region="cn", api_key="test-key") + result = json.loads(await ImageGenerateTool(config, workspace=tmp_path).execute(prompt="a garden")) + + assert result["success"] is True + assert result["model"] == "image-01" + assert Path(result["paths"][0]).read_bytes() == b"downloaded-image" + + +async def test_minimax_image_generation_surfaces_api_error(tmp_path: Path, monkeypatch) -> None: + _patch_client( + monkeypatch, + lambda _request: httpx.Response( + 200, + json={"base_resp": {"status_code": 2013, "status_msg": "invalid parameters"}}, + ), + ) + config = ImageToolConfig(provider="minimax", api_key="test-key") + + result = json.loads(await ImageGenerateTool(config, workspace=tmp_path).execute(prompt="test")) + + assert result == {"error": "invalid parameters", "status_code": 2013, "model": "image-01"} + + +def test_effective_media_config_resolves_minimax_key() -> None: + config = Config.model_validate( + { + "providers": {"minimax": {"apiKey": "provider-key"}}, + "tools": { + "media": { + "image": {"provider": "minimax", "region": "cn"}, + } + }, + } + ) + + image = config.effective_media_config().image + + assert image.api_key == "provider-key" + assert image.model == "image-01" + assert image.provider == "minimax" + assert image.region == "cn"