Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 204 additions & 3 deletions raven/agent/tools/media_gen.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Multimodal generation tools (OpenRouter backend).
"""Multimodal generation tools.

OpenRouter exposes generation through the OpenAI-compatible chat-completions
API with extra *output modalities* — there are no standalone
Expand All @@ -9,6 +9,8 @@
- ``image_generate`` → ``modalities:["image","text"]`` → ``message.images[].image_url.url``
(a ``data:image/...;base64,...`` URI). Default model **Nano Banana**
(``google/gemini-2.5-flash-image``).
- ``image_generate`` can instead call MiniMax ``/v1/image_generation`` in the
global or China region and persist URL or base64 results.
- ``text_to_speech`` → ``modalities:["audio","text"]`` + ``audio:{voice,format}``.
OpenRouter only returns audio when ``stream:true`` AND only as raw ``pcm16``
in that mode, so we stream, concatenate the base64 ``delta.audio.data`` chunks,
Expand Down Expand Up @@ -51,9 +53,13 @@
from raven.utils.helpers import image_block

if TYPE_CHECKING:
from raven.config.schema import MediaToolConfig
from raven.config.schema import ImageToolConfig, MediaToolConfig

_DEFAULT_BASE = "https://openrouter.ai/api/v1"
_MINIMAX_IMAGE_BASES = {
"global": "https://api.minimax.io/v1",
"cn": "https://api.minimaxi.com/v1",
}

_EXT_MIME = {
".png": "image/png",
Expand Down Expand Up @@ -145,7 +151,7 @@ def _format_http_error(self, e: httpx.HTTPStatusError) -> str:


class ImageGenerateTool(_OpenRouterMediaTool):
"""Generate (or edit) an image from a text prompt via Nano Banana on OpenRouter."""
"""Generate an image from a text prompt through the configured image API."""

name = "image_generate"
default_model = "google/gemini-2.5-flash-image" # Nano Banana
Expand Down Expand Up @@ -173,10 +179,84 @@ class ImageGenerateTool(_OpenRouterMediaTool):
"Optional input images to edit/vary: local file paths, http(s) URLs, or data: URIs (max 6)"
),
},
"aspect_ratio": {
"type": "string",
"enum": ["1:1", "16:9", "4:3", "3:2", "2:3", "3:4", "9:16", "21:9"],
"description": "Optional output aspect ratio for MiniMax image generation",
},
"width": {"type": "integer", "description": "Optional output width in pixels"},
"height": {"type": "integer", "description": "Optional output height in pixels"},
"response_format": {
"type": "string",
"enum": ["url", "base64"],
"default": "url",
"description": "MiniMax image response format",
},
"seed": {"type": "integer", "description": "Optional deterministic generation seed"},
"n": {
"type": "integer",
"minimum": 1,
"maximum": 9,
"default": 1,
"description": "Number of images to generate",
},
"prompt_optimizer": {
"type": "boolean",
"default": False,
"description": "Whether MiniMax should optimize the prompt",
},
},
"required": ["prompt"],
}

def __init__(
self,
config: "ImageToolConfig | None" = None,
*,
workspace: Path | None = None,
proxy: str | None = None,
output_subdir: str = "generated",
):
super().__init__(config, workspace=workspace, proxy=proxy, output_subdir=output_subdir)

@property
def _uses_minimax(self) -> bool:
return bool(self._config and getattr(self._config, "provider", "") == "minimax")

@property
def api_key(self) -> str:
if not self._uses_minimax:
return super().api_key
cfg_key = getattr(self._config, "api_key", "") if self._config else ""
return cfg_key or os.environ.get("MINIMAX_API_KEY", "")

@property
def api_base(self) -> str:
if not self._uses_minimax:
return super().api_base
cfg_base = getattr(self._config, "api_base", "") if self._config else ""
region = getattr(self._config, "region", "global") if self._config else "global"
return (cfg_base or _MINIMAX_IMAGE_BASES[region]).rstrip("/")

def _model(self, override: str | None) -> str:
if not self._uses_minimax:
return super()._model(override)
cfg_model = getattr(self._config, "model", "") if self._config else ""
return override or cfg_model or "image-01"

def _no_key_error(self) -> str:
if not self._uses_minimax:
return super()._no_key_error()
return json.dumps(
{
"error": (
"image_generate: no API key configured. Set tools.media.image.apiKey, "
"providers.minimax.apiKey, or MINIMAX_API_KEY, then restart the gateway."
)
},
ensure_ascii=False,
)

def _image_part(self, ref: str) -> dict[str, Any]:
"""Build an OpenAI-style image_url content part from a path/URL/data URI."""
if ref.startswith(("http://", "https://", "data:")):
Expand All @@ -192,12 +272,32 @@ async def execute(
prompt: str,
model: str | None = None,
images: list[str] | None = None,
aspect_ratio: str | None = None,
width: int | None = None,
height: int | None = None,
response_format: str = "url",
seed: int | None = None,
n: int = 1,
prompt_optimizer: bool = False,
**kwargs: Any,
) -> str:
if not self.api_key:
return self._no_key_error()

model_id = self._model(model)
if self._uses_minimax:
return await self._execute_minimax(
prompt=prompt,
model=model_id,
images=images,
aspect_ratio=aspect_ratio,
width=width,
height=height,
response_format=response_format,
seed=seed,
n=n,
prompt_optimizer=prompt_optimizer,
)
if images:
content: Any = [{"type": "text", "text": prompt}]
try:
Expand Down Expand Up @@ -246,6 +346,107 @@ async def execute(
logger.info("image_generate: {} image(s) via {} -> {}", len(paths), model_id, paths)
return json.dumps({"success": True, "model": model_id, "paths": paths}, ensure_ascii=False)

async def _execute_minimax(
self,
*,
prompt: str,
model: str,
images: list[str] | None,
aspect_ratio: str | None,
width: int | None,
height: int | None,
response_format: str,
seed: int | None,
n: int,
prompt_optimizer: bool,
) -> str:
if images:
return json.dumps(
{"error": "input images are not supported by the configured text-to-image API"},
ensure_ascii=False,
)
payload: dict[str, Any] = {
"model": model,
"prompt": prompt,
"response_format": response_format,
"n": n,
"prompt_optimizer": prompt_optimizer,
}
optional = {
"aspect_ratio": aspect_ratio,
"width": width,
"height": height,
"seed": seed,
}
payload.update({key: value for key, value in optional.items() if value is not None})

headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
try:
async with httpx.AsyncClient(proxy=self._proxy, timeout=180.0) as client:
response = await client.post(
f"{self.api_base}/image_generation",
headers=headers,
json=payload,
)
response.raise_for_status()
data = response.json()
base_resp = data.get("base_resp") or {}
if base_resp.get("status_code", 0) != 0:
return json.dumps(
{
"error": base_resp.get("status_msg") or "image generation failed",
"status_code": base_resp.get("status_code"),
"model": model,
},
ensure_ascii=False,
)
output = data.get("data") or {}
paths = await self._save_minimax_images(
client,
output.get("image_urls") or [],
output.get("image_base64") or [],
)
except httpx.HTTPStatusError as e:
return self._format_http_error(e)
except Exception as e:
logger.error("image_generate error: {}", e)
return json.dumps({"error": str(e)}, ensure_ascii=False)

if not paths:
return json.dumps(
{"error": "no image returned", "model": model, "metadata": data.get("metadata") or {}},
ensure_ascii=False,
)
logger.info("image_generate: {} image(s) via {} -> {}", len(paths), model, paths)
return json.dumps(
{
"success": True,
"model": model,
"paths": paths,
"metadata": data.get("metadata") or {},
},
ensure_ascii=False,
)

async def _save_minimax_images(
self,
client: httpx.AsyncClient,
urls: list[str],
encoded_images: list[str],
) -> list[str]:
paths: list[str] = []
for encoded in encoded_images:
path = self._output_path("png")
path.write_bytes(base64.b64decode(encoded))
paths.append(str(path))
for url in urls:
download = await client.get(url)
download.raise_for_status()
path = self._output_path("png")
path.write_bytes(download.content)
paths.append(str(path))
return paths


class SpeechGenerateTool(_OpenRouterMediaTool):
"""Synthesize speech from text via an OpenRouter audio model (gpt-audio)."""
Expand Down
34 changes: 25 additions & 9 deletions raven/config/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -745,14 +745,21 @@ class MediaToolConfig(Base):
model: str = ""


class ImageToolConfig(MediaToolConfig):
"""Image generation config with an explicit API protocol and region."""

provider: Literal["openrouter", "minimax"] = "openrouter"
region: Literal["global", "cn"] = "global"


class MediaGenConfig(Base):
"""Multimodal generation tools configuration.

OpenRouter is the only backend: image + speech via chat-completions output
modalities, and video via the async ``/videos`` endpoint (Kling).
Image generation can use MiniMax directly; the other configured protocols
use their existing media endpoints.
"""

image: MediaToolConfig = Field(default_factory=MediaToolConfig)
image: ImageToolConfig = Field(default_factory=ImageToolConfig)
speech: MediaToolConfig = Field(default_factory=MediaToolConfig)
video: MediaToolConfig = Field(default_factory=MediaToolConfig)
proxy: str | None = None # HTTP/SOCKS proxy for media API calls
Expand Down Expand Up @@ -847,11 +854,11 @@ def workspace_path(self) -> Path:
def effective_media_config(self) -> MediaGenConfig:
"""Media config resolved for registration and auth.

A media tool (image/speech/video) counts as configured only when the
user set its ``model`` or ``apiKey`` under ``tools.media.<tool>``. For
each configured tool we default a missing key to
``providers.openrouter.apiKey`` so the chat key can be reused without
re-declaring it. Tools the user did not configure are left untouched
A media tool (image/speech/video) counts as configured when the user
selects its provider or sets ``model`` or ``apiKey`` under
``tools.media.<tool>``. A configured image tool resolves the key for its
selected provider, while the other media tools retain their existing key
fallback. Tools the user did not configure are left untouched
(no key, no model) — ``AgentLoop`` registers a media tool only when it
has a key or model, so an OpenRouter key set for chat alone never
surfaces image/speech/video to the agent. Returns a copy so this
Expand All @@ -860,7 +867,16 @@ def effective_media_config(self) -> MediaGenConfig:
media = self.tools.media.model_copy(deep=True)
openrouter = self.providers.get("openrouter")
or_key = openrouter.api_key if openrouter else ""
for tool in (media.image, media.speech, media.video):
image_configured = bool(media.image.api_key or media.image.model or media.image.provider != "openrouter")
if image_configured and media.image.provider == "minimax" and not media.image.model:
media.image.model = "image-01"
if image_configured and not media.image.api_key:
if media.image.provider == "minimax":
minimax = self.providers.get("minimax")
media.image.api_key = minimax.api_key if minimax else ""
elif or_key:
media.image.api_key = or_key
for tool in (media.speech, media.video):
configured = bool(tool.api_key or tool.model)
if configured and or_key and not tool.api_key:
tool.api_key = or_key
Expand Down
Loading