Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 170 additions & 2 deletions raven/agent/tools/media_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
to download) or ``failed`` (with ``error``). Default model
``kwaivgi/kling-v3.0-std`` (Kling v3 Standard). Requires postpaid billing /
credits enabled on the OpenRouter account.
- MiniMax text-to-video models use the regional ``/v1/video_generation`` or
``/v2/video_generation`` API, with the matching query and file retrieval
flow. The global endpoint is the default; set ``apiBase`` to the China
endpoint when needed.

Generated files are written under ``<workspace>/<output_subdir>`` and the path
is returned so the agent can forward it with the ``message`` tool's ``media``
Expand Down Expand Up @@ -54,6 +58,18 @@
from raven.config.schema import MediaToolConfig

_DEFAULT_BASE = "https://openrouter.ai/api/v1"
_MINIMAX_GLOBAL_BASE = "https://api.minimax.io"
_MINIMAX_TEXT_VIDEO_MODELS = frozenset(
{
"MiniMax-H3",
"MiniMax-Hailuo-2.3",
"MiniMax-Hailuo-2.3-Fast",
"MiniMax-Hailuo-02",
"T2V-01-Director",
"T2V-01",
}
)
_MINIMAX_V2_MODEL = "MiniMax-H3"

_EXT_MIME = {
".png": "image/png",
Expand Down Expand Up @@ -493,8 +509,9 @@ class VideoGenerateTool(_OpenRouterMediaTool):
"params": {
"type": "object",
"description": (
"Optional extra provider params merged into the request, e.g. "
'{"duration": 5, "aspect_ratio": "16:9"} (3-15s; 16:9/9:16/1:1)'
"Optional provider-specific request fields. MiniMax supports "
"duration, resolution, ratio, callback_url, prompt_optimizer, "
"fast_pretreatment, and aigc_watermark where applicable."
),
},
},
Expand All @@ -517,6 +534,9 @@ async def execute(
return self._no_key_error()

model_id = self._model(model)
if model_id in _MINIMAX_TEXT_VIDEO_MODELS:
return await self._execute_minimax(prompt, model_id, params)

body: dict[str, Any] = {"model": model_id, "prompt": prompt}
if params:
body.update(params)
Expand Down Expand Up @@ -572,6 +592,154 @@ async def execute(
logger.info("video_generate: {} bytes via {} -> {}", len(data), model_id, path)
return json.dumps({"success": True, "model": model_id, "path": str(path)}, ensure_ascii=False)

def _minimax_api_base(self) -> str:
configured = getattr(self._config, "api_base", "") if self._config else ""
base = (configured or _MINIMAX_GLOBAL_BASE).rstrip("/")
for suffix in ("/v2/video_generation", "/v1/video_generation", "/v2", "/v1"):
if base.endswith(suffix):
return base[: -len(suffix)]
return base

@staticmethod
def _minimax_error(payload: dict[str, Any]) -> str:
response = payload.get("base_resp") or {}
status_code = response.get("status_code")
if status_code in (None, 0):
return ""
return response.get("status_msg") or f"MiniMax API status {status_code}"

async def _execute_minimax(
self,
prompt: str,
model_id: str,
params: dict[str, Any] | None,
) -> str:
api_base = self._minimax_api_base()
headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
options = params or {}
is_v2 = model_id == _MINIMAX_V2_MODEL
if is_v2:
allowed = {"resolution", "duration", "ratio", "callback_url", "aigc_watermark"}
body: dict[str, Any] = {
"model": model_id,
"content": [{"type": "text", "text": prompt}],
"resolution": options.get("resolution", "2K"),
"duration": options.get("duration", 5),
}
body.update({key: value for key, value in options.items() if key in allowed})
submit_url = f"{api_base}/v2/video_generation"
else:
allowed = {"prompt_optimizer", "fast_pretreatment", "duration", "resolution", "callback_url"}
body = {"model": model_id, "prompt": prompt}
body.update({key: value for key, value in options.items() if key in allowed})
submit_url = f"{api_base}/v1/video_generation"

try:
async with httpx.AsyncClient(proxy=self._proxy, timeout=120.0) as client:
response = await client.post(submit_url, headers=headers, json=body)
response.raise_for_status()
job = response.json()
if error := self._minimax_error(job):
return json.dumps({"error": error, "model": model_id}, ensure_ascii=False)
task_id = job.get("task_id")
if not task_id:
return json.dumps(
{"error": "video task response did not include task_id", "model": model_id},
ensure_ascii=False,
)

status = await self._poll_minimax(client, api_base, str(task_id), headers, is_v2=is_v2)
if status.get("status") == "timeout":
return json.dumps({"error": "video job status=timeout", "model": model_id}, ensure_ascii=False)
if error := self._minimax_error(status):
return json.dumps({"error": error, "model": model_id}, ensure_ascii=False)

if is_v2:
task = status.get("task") or {}
state = str(task.get("status", "")).lower()
video_url = (task.get("content") or {}).get("url")
detail = task.get("error") or status
succeeded = state == "succeeded"
else:
state = str(status.get("status", "")).lower()
detail = status
succeeded = state == "success"
video_url = None
if succeeded:
file_id = status.get("file_id")
file_response = await client.get(
f"{api_base}/v1/files/retrieve",
headers=headers,
params={"file_id": file_id},
)
file_response.raise_for_status()
file_payload = file_response.json()
if error := self._minimax_error(file_payload):
return json.dumps({"error": error, "model": model_id}, ensure_ascii=False)
video_url = (file_payload.get("file") or {}).get("download_url")

if not succeeded:
return json.dumps(
{"error": f"video job status={state}", "detail": detail, "model": model_id},
ensure_ascii=False,
)
if not video_url:
return json.dumps(
{"error": "completed but no video URL found", "model": model_id},
ensure_ascii=False,
)

video_origin = httpx.URL(video_url)
api_origin = httpx.URL(api_base)
same_origin = (video_origin.scheme, video_origin.host, video_origin.port) == (
api_origin.scheme,
api_origin.host,
api_origin.port,
)
download_headers = headers if same_origin else None
download = await client.get(video_url, headers=download_headers, timeout=180.0)
download.raise_for_status()
data = download.content
except httpx.HTTPStatusError as error:
return self._format_http_error(error)
except Exception as error:
logger.error("video_generate error: {}", error)
return json.dumps({"error": str(error)}, ensure_ascii=False)

path = self._output_path("mp4")
path.write_bytes(data)
logger.info("video_generate: {} bytes via {} -> {}", len(data), model_id, path)
return json.dumps({"success": True, "model": model_id, "path": str(path)}, ensure_ascii=False)

async def _poll_minimax(
self,
client: httpx.AsyncClient,
api_base: str,
task_id: str,
headers: dict[str, str],
*,
is_v2: bool,
) -> dict[str, Any]:
waited = 0.0
while waited < self._POLL_TIMEOUT_S:
if is_v2:
response = await client.get(f"{api_base}/v2/query/video_generation/{task_id}", headers=headers)
else:
response = await client.get(
f"{api_base}/v1/query/video_generation",
headers=headers,
params={"task_id": task_id},
)
response.raise_for_status()
job = response.json()
task = job.get("task") or {}
state = str(task.get("status") if is_v2 else job.get("status", "")).lower()
if state not in {"queued", "running", "preparing", "queueing", "processing"}:
return job
await asyncio.sleep(self._POLL_INTERVAL_S)
waited += self._POLL_INTERVAL_S
return {"status": "timeout"}

async def _poll(self, client: httpx.AsyncClient, poll_url: str, headers: dict[str, str]) -> dict[str, Any]:
"""Poll until the job leaves the pending/processing state or times out."""
waited = 0.0
Expand Down
33 changes: 20 additions & 13 deletions raven/config/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -735,9 +735,8 @@ class ExecToolConfig(Base):
class MediaToolConfig(Base):
"""Config for a media-generation tool (key + base + model).

Empty fields fall back at call time: ``api_key`` → ``providers.openrouter``
/ ``OPENROUTER_API_KEY``; ``api_base`` → OpenRouter; ``model`` → the tool's
default (Nano Banana for images).
Empty fields fall back at call time according to the configured model and
tool backend. A missing model uses the tool's default.
"""

api_key: str = ""
Expand All @@ -748,8 +747,8 @@ class MediaToolConfig(Base):
class MediaGenConfig(Base):
"""Multimodal generation tools configuration.

OpenRouter is the only backend: image + speech via chat-completions output
modalities, and video via the async ``/videos`` endpoint (Kling).
Image, speech, and video tools resolve their backend from the configured
model. Video generation supports both regional MiniMax API endpoints.
"""

image: MediaToolConfig = Field(default_factory=MediaToolConfig)
Expand Down Expand Up @@ -849,21 +848,29 @@ def effective_media_config(self) -> MediaGenConfig:

A media tool (image/speech/video) counts as configured only when the
user set its ``model`` or ``apiKey`` under ``tools.media.<tool>``. For
each configured tool we default a missing key to
``providers.openrouter.apiKey`` so the chat key can be reused without
re-declaring it. Tools the user did not configure are left untouched
(no key, no model) — ``AgentLoop`` registers a media tool only when it
has a key or model, so an OpenRouter key set for chat alone never
surfaces image/speech/video to the agent. Returns a copy so this
resolution never mutates the raw config.
each configured tool we default a missing key from its matching provider
section so the key can be reused without re-declaring it. Tools the user
did not configure are left untouched (no key, no model) — ``AgentLoop``
registers a media tool only when it has a key or model, so a provider key
set for chat alone never surfaces image/speech/video to the agent. Returns
a copy so this resolution never mutates the raw config.
"""
media = self.tools.media.model_copy(deep=True)
openrouter = self.providers.get("openrouter")
or_key = openrouter.api_key if openrouter else ""
for tool in (media.image, media.speech, media.video):
for tool in (media.image, media.speech):
configured = bool(tool.api_key or tool.model)
if configured and or_key and not tool.api_key:
tool.api_key = or_key

video_configured = bool(media.video.api_key or media.video.model)
uses_minimax = media.video.model.startswith(("MiniMax-", "T2V-"))
video_provider = self.providers.get("minimax" if uses_minimax else "openrouter")
if video_configured and video_provider:
if not media.video.api_key:
media.video.api_key = video_provider.effective_api_key
if uses_minimax and not media.video.api_base and video_provider.api_base:
media.video.api_base = video_provider.api_base
return media

def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | None", str | None]:
Expand Down
Loading