Merge pull request #466 from calesthio/codex/backlog-recovery-20260802

Recover bounded defects from the latest PR backlog
This commit is contained in:
Calesthio 2026-08-03 14:49:08 +05:30 committed by GitHub
commit 4eab34c5cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
68 changed files with 1913 additions and 376 deletions

View File

@ -2,72 +2,103 @@
# Copy this to .env and fill in your keys
# --- Image + video gateway ---
FAL_KEY= # FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images
# Get one at https://fal.ai/dashboard/keys
FAL_AI_API_KEY= # Alias for FAL_KEY (some SDKs/docs use this name); either one is read.
# FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images.
# Get one at https://fal.ai/dashboard/keys
FAL_KEY=
# Alias for FAL_KEY (some SDKs/docs use this name); either one is read.
FAL_AI_API_KEY=
# --- Replicate ---
REPLICATE_API_TOKEN= # Replicate-hosted video gen (seedance_replicate). Needed to make the
# Replicate-backed Seedance path selectable alongside the fal.ai one.
# Get one at https://replicate.com/account/api-tokens
# Replicate-hosted video gen (seedance_replicate). Needed to make the
# Replicate-backed Seedance path selectable alongside the fal.ai one.
# Get one at https://replicate.com/account/api-tokens
REPLICATE_API_TOKEN=
# --- Higgsfield ---
HIGGSFIELD_API_KEY= # Higgsfield Cloud key (higgsfield_video). Pair with the secret below,
HIGGSFIELD_API_SECRET= # or use the combined HIGGSFIELD_KEY="<key>:<secret>" form instead.
# Higgsfield Cloud key (higgsfield_video). Pair with the secret below,
# or use the combined HIGGSFIELD_KEY="<key>:<secret>" form instead.
HIGGSFIELD_API_KEY=
HIGGSFIELD_API_SECRET=
# HIGGSFIELD_KEY= # Combined key:secret — set this INSTEAD of the _KEY/_SECRET pair if you prefer.
# --- Kling official direct API ---
KLING_API_KEY= # Official Kling API key; enables video, image, TTS, avatar, lip sync
KLING_API_BASE_URL= # Optional endpoint override; leave blank for default https://api-singapore.klingai.com
# Mainland China accounts can use https://api-beijing.klingai.com
# Official Kling API key; enables video, image, TTS, avatar, lip sync.
KLING_API_KEY=
# Optional endpoint override; leave blank for default https://api-singapore.klingai.com
# Mainland China accounts can use https://api-beijing.klingai.com
KLING_API_BASE_URL=
# --- Google (one key unlocks image gen + TTS + video) ---
GOOGLE_API_KEY= # Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages),
# Gemini Omni video (generation + conversational editing, paid tier)
# Get one at https://aistudio.google.com/apikey
# Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages),
# Gemini Omni video (generation + conversational editing, paid tier).
# Get one at https://aistudio.google.com/apikey
GOOGLE_API_KEY=
# GEMINI_API_KEY= # Alias for GOOGLE_API_KEY (takes precedence when both are set)
# Alternative to the API key: service-account JSON auth.
# TTS uses Cloud Text-to-Speech; Imagen routes to Vertex AI.
GOOGLE_APPLICATION_CREDENTIALS= # path to a service-account JSON key file
GOOGLE_CLOUD_PROJECT= # GCP project id (required for Imagen via Vertex AI)
GOOGLE_CLOUD_LOCATION= # Vertex AI region, default us-central1
# Path to a service-account JSON key file.
GOOGLE_APPLICATION_CREDENTIALS=
# GCP project id (required for Imagen via Vertex AI).
GOOGLE_CLOUD_PROJECT=
# Vertex AI region, default us-central1.
GOOGLE_CLOUD_LOCATION=
# --- Voice ---
ELEVENLABS_API_KEY= # TTS narration, music generation, sound effects
OPENAI_API_KEY= # OpenAI TTS fallback and GPT Image 2 image generation
XAI_API_KEY= # Grok image generation/editing and Grok video generation
DOUBAO_SPEECH_API_KEY= # Volcengine Doubao Speech TTS (new console API Key)
DOUBAO_SPEECH_VOICE_TYPE= # Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts
# TTS narration, music generation, sound effects.
ELEVENLABS_API_KEY=
# OpenAI TTS fallback and GPT Image 2 image generation.
OPENAI_API_KEY=
# Grok image generation/editing and Grok video generation.
XAI_API_KEY=
# Volcengine Doubao Speech TTS (new console API Key).
DOUBAO_SPEECH_API_KEY=
# Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts.
DOUBAO_SPEECH_VOICE_TYPE=
# Piper local voices do not require env vars; install `piper-tts` via pip
# --- DashScope (Alibaba Cloud Bailian) ---
DASHSCOPE_API_KEY= # Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans)
# Get one at https://dashscope.aliyun.com/
# Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans).
# Get one at https://dashscope.aliyun.com/
DASHSCOPE_API_KEY=
# --- Music ---
SUNO_API_KEY= # Suno AI music generation (full songs, instrumentals, any genre)
# Suno AI music generation (full songs, instrumentals, any genre).
SUNO_API_KEY=
# --- Video Generation ---
HEYGEN_API_KEY= # HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key)
RUNWAY_API_KEY= # Runway Gen-4 (direct API, alternative to fal.ai routing)
VOLC_ACCESSKEY= # Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing)
VOLC_SECRETKEY= # Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage
VIDEO_GEN_LOCAL_ENABLED= # Set to "true" for local video gen (needs GPU + diffusers)
VIDEO_GEN_LOCAL_MODEL= # Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b
MODAL_LTX2_ENDPOINT_URL= # Modal self-hosted LTX-2 endpoint (optional)
# HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key).
HEYGEN_API_KEY=
# Runway Gen-4 (direct API, alternative to fal.ai routing).
RUNWAY_API_KEY=
# Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing).
VOLC_ACCESSKEY=
# Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage
VOLC_SECRETKEY=
# Set to "true" for local video gen (needs GPU + diffusers).
VIDEO_GEN_LOCAL_ENABLED=
# Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b.
VIDEO_GEN_LOCAL_MODEL=
# Modal self-hosted LTX-2 endpoint (optional).
MODAL_LTX2_ENDPOINT_URL=
# --- Stock Media ---
PEXELS_API_KEY= # Pexels stock footage/images (free)
PIXABAY_API_KEY= # Pixabay stock footage/images (free)
UNSPLASH_ACCESS_KEY= # Unsplash stock images (free developer key)
# Pexels stock footage/images (free).
PEXELS_API_KEY=
# Pixabay stock footage/images (free).
PIXABAY_API_KEY=
# Unsplash stock images (free developer key).
UNSPLASH_ACCESS_KEY=
# --- Analysis ---
HF_TOKEN= # HuggingFace token — enables speaker diarization in transcriber
# HuggingFace token — enables speaker diarization in transcriber.
HF_TOKEN=
# Speech-to-text: optional Azure AI Speech (Fast Transcription). When set, the
# agent prefers azure_stt for cloud STT; the local faster-whisper transcriber
# remains the default offline path.
AZURE_SPEECH_KEY= # Azure AI Speech resource key ('Keys and Endpoint' page)
AZURE_SPEECH_REGION= # Speech resource region, e.g. eastus
# Azure AI Speech resource key ('Keys and Endpoint' page).
AZURE_SPEECH_KEY=
# Speech resource region, e.g. eastus.
AZURE_SPEECH_REGION=
# AZURE_SPEECH_ENDPOINT= # Optional: full custom endpoint URL (overrides region)
# --- Avatar (local installs) ---

View File

@ -528,9 +528,10 @@ Music is a critical part of any video. **Surface the music situation to the user
Check music availability in this order and present the options:
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one.
2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly — include quota status if known.
3. **Royalty-free sources:** Note if the user can provide their own track (e.g., from YouTube Audio Library, Jamendo, or other free sources). Offer the `music_library/` drop path.
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. If tracks exist, list durations and let the user pick one.
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")` for configured search/download tools. Report licensing constraints and whether a key is required.
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly.
4. **Bring your own:** Note that the user can provide a track (for example from YouTube Audio Library or Jamendo) through the `music_library/` drop path.
**Always present the user with explicit choices:**
- Use a track from their library (which one?)

View File

@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import json
import time
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from typing import Optional
@ -147,18 +148,22 @@ async def _watch_projects() -> None:
hub.publish(pid)
@asynccontextmanager
async def _lifespan(app: FastAPI):
"""Own and cleanly stop the project watcher with FastAPI's lifespan API."""
task = asyncio.create_task(_watch_projects())
app.state.watch_task = task
try:
yield
finally:
task.cancel()
with suppress(asyncio.CancelledError):
await task
def create_app() -> FastAPI:
app = FastAPI(title="Backlot", docs_url=None, redoc_url=None)
@app.on_event("startup")
async def _startup() -> None:
app.state.watch_task = asyncio.create_task(_watch_projects())
@app.on_event("shutdown")
async def _shutdown() -> None:
task = getattr(app.state, "watch_task", None)
if task:
task.cancel()
app = FastAPI(title="Backlot", docs_url=None, redoc_url=None, lifespan=_lifespan)
# ---- API ----------------------------------------------------------

View File

@ -9,6 +9,7 @@ from __future__ import annotations
import json
import re
from functools import lru_cache
from pathlib import Path
from typing import Any, Optional
@ -59,6 +60,7 @@ def _rel(project_dir: Path, path: Path) -> str:
# Pipeline / stages
# ---------------------------------------------------------------------------
@lru_cache(maxsize=32)
def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]:
"""Stage order + gate flags from the manifest; graceful fallback."""
if pipeline_type and pipeline_type != "unknown":

View File

@ -95,9 +95,29 @@ class CheckpointValidationError(ValueError):
"""Raised when a checkpoint or its canonical artifacts are invalid."""
def _validate_style_playbook(style_playbook: str | None) -> None:
"""Fail closed when a checkpoint names a visual identity that cannot load."""
if style_playbook is None:
return
try:
from styles.playbook_loader import list_playbooks, load_playbook
load_playbook(style_playbook)
except Exception as exc:
try:
available = list_playbooks()
except Exception:
available = []
raise CheckpointValidationError(
f"Unknown or invalid style_playbook {style_playbook!r}. "
f"Available playbooks: {available}. Underlying error: {exc}"
) from exc
@lru_cache(maxsize=1)
def _load_checkpoint_schema() -> dict[str, Any]:
with open(CHECKPOINT_SCHEMA_PATH) as f:
with open(CHECKPOINT_SCHEMA_PATH, encoding="utf-8") as f:
return json.load(f)
@ -192,6 +212,7 @@ def init_project(
Idempotent: re-running preserves the original created_at and merges fields.
Returns the project directory.
"""
_validate_style_playbook(style_playbook)
base = pipeline_dir or PROJECTS_DIR
project_dir = base / project_id
for sub in (
@ -208,7 +229,7 @@ def init_project(
marker: dict[str, Any] = {}
if marker_path.exists():
try:
with open(marker_path) as f:
with open(marker_path, encoding="utf-8") as f:
marker = json.load(f)
except (json.JSONDecodeError, OSError):
marker = {}
@ -221,7 +242,7 @@ def init_project(
if style_playbook is not None:
marker["style_playbook"] = style_playbook
with open(marker_path, "w") as f:
with open(marker_path, "w", encoding="utf-8") as f:
json.dump(marker, f, indent=2)
return project_dir
@ -260,6 +281,71 @@ def _stage_requires_approval(pipeline_type: Optional[str], stage: str) -> Option
return get_stage_human_approval_default(manifest, stage)
def _enforce_stage_prerequisites(
pipeline_dir: Path,
project_id: str,
pipeline_type: str | None,
stage: str,
status: str,
) -> None:
"""Require completed, approved predecessors before advancing a stage.
``in_progress`` and failure heartbeats remain writable so an operator can
inspect or resume a broken run. Only lifecycle advancement
(``awaiting_human``/``completed``) is gated.
"""
if status not in {"awaiting_human", "completed"}:
return
if not pipeline_type or pipeline_type == "unknown":
return
stages = get_pipeline_stages(pipeline_type)
if stage not in stages:
return
incomplete: list[str] = []
unapproved: list[str] = []
for predecessor in stages[: stages.index(stage)]:
path = _checkpoint_path(pipeline_dir, project_id, predecessor)
if not path.exists():
incomplete.append(predecessor)
continue
try:
with open(path, encoding="utf-8") as handle:
checkpoint = json.load(handle)
validate_checkpoint(checkpoint)
except (OSError, json.JSONDecodeError, CheckpointValidationError):
incomplete.append(predecessor)
continue
if (
checkpoint.get("project_id") != project_id
or checkpoint.get("pipeline_type") != pipeline_type
or checkpoint.get("stage") != predecessor
):
incomplete.append(predecessor)
continue
if checkpoint.get("status") != "completed":
incomplete.append(predecessor)
continue
if _stage_requires_approval(pipeline_type, predecessor) and not checkpoint.get(
"human_approved"
):
unapproved.append(predecessor)
if incomplete or unapproved:
details = []
if incomplete:
details.append(f"incomplete or missing: {incomplete}")
if unapproved:
details.append(f"completed without required approval: {unapproved}")
raise CheckpointValidationError(
f"PREREQUISITE VIOLATION: stage {stage!r} cannot advance; "
+ "; ".join(details)
+ f". Pipeline order: {stages}."
)
def _archive_superseded_checkpoint(path: Path, stage: str) -> None:
"""Copy an existing checkpoint into history/ before it is overwritten.
@ -275,7 +361,7 @@ def _archive_superseded_checkpoint(path: Path, stage: str) -> None:
if not path.exists():
return
try:
with open(path) as f:
with open(path, encoding="utf-8") as f:
existing = json.load(f)
except (json.JSONDecodeError, OSError):
existing = {}
@ -314,7 +400,7 @@ def _merge_decision_log(
"""
path = _decision_log_path(pipeline_dir, project_id)
if path.exists():
with open(path) as f:
with open(path, encoding="utf-8") as f:
existing = json.load(f)
else:
existing = {
@ -329,7 +415,7 @@ def _merge_decision_log(
existing["decisions"].append(decision)
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f:
with open(path, "w", encoding="utf-8") as f:
json.dump(existing, f, indent=2)
@ -351,19 +437,22 @@ def write_checkpoint(
metadata: Optional[dict] = None,
) -> Path:
"""Write a checkpoint file for a pipeline stage."""
# Backfill a missing pipeline_type from the project marker so that
# omitting the kwarg doesn't quietly bypass gate enforcement.
if not pipeline_type:
marker = None
marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME
if marker_path.exists():
try:
with open(marker_path) as f:
marker = json.load(f)
except (json.JSONDecodeError, OSError):
marker = None
if isinstance(marker, dict) and marker.get("pipeline_type"):
# Backfill identity fields from the project marker so omitted kwargs
# cannot bypass either gate enforcement or style validation.
marker = None
marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME
if marker_path.exists() and (not pipeline_type or not style_playbook):
try:
with open(marker_path, encoding="utf-8") as f:
marker = json.load(f)
except (json.JSONDecodeError, OSError):
marker = None
if isinstance(marker, dict):
if not pipeline_type and marker.get("pipeline_type"):
pipeline_type = marker["pipeline_type"]
if not style_playbook and marker.get("style_playbook"):
style_playbook = marker["style_playbook"]
_validate_style_playbook(style_playbook)
valid_stages = (
set(get_pipeline_stages(pipeline_type)) if pipeline_type
@ -404,6 +493,14 @@ def write_checkpoint(
f"re-write with status='completed', human_approved=True."
)
_enforce_stage_prerequisites(
pipeline_dir,
project_id,
pipeline_type,
stage,
status,
)
checkpoint = {
"version": "1.0",
"project_id": project_id,
@ -456,7 +553,7 @@ def write_checkpoint(
# current checkpoint; then archive the superseded file and swap in the
# new one atomically.
tmp_path = path.with_suffix(".json.tmp")
with open(tmp_path, "w") as f:
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(checkpoint, f, indent=2)
# Preserve run history: a superseded completed/awaiting_human checkpoint
# is copied to history/ (stage versioning, gate audit trail, replay).
@ -474,7 +571,7 @@ def read_checkpoint(
path = _checkpoint_path(pipeline_dir, project_id, stage)
if not path.exists():
return None
with open(path) as f:
with open(path, encoding="utf-8") as f:
checkpoint = json.load(f)
validate_checkpoint(checkpoint)
return checkpoint
@ -496,7 +593,7 @@ def get_latest_checkpoint(
if not checkpoints:
return None
with open(checkpoints[0]) as f:
with open(checkpoints[0], encoding="utf-8") as f:
checkpoint = json.load(f)
validate_checkpoint(checkpoint)
return checkpoint

View File

@ -61,6 +61,20 @@ def model_info() -> dict:
}
def _as_feature_tensor(features):
"""Normalize CLIP feature return values across transformers versions.
Transformers 4 returned the projected tensor directly. Transformers 5 may
wrap that tensor in a model-output object whose ``pooler_output`` contains
the same shared-space embedding. Do not project it again: the vision
projection expects the pre-projection width, while ``pooler_output`` is
already the final CLIP width.
"""
pooled = getattr(features, "pooler_output", None)
return features if pooled is None else pooled
def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray:
"""Embed a list of image files into a (N, 512) float32 matrix.
@ -82,7 +96,7 @@ def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray:
inputs = _PROCESSOR(images=images, return_tensors="pt").to(_DEVICE)
with torch.no_grad():
features = _MODEL.get_image_features(**inputs)
features = _as_feature_tensor(_MODEL.get_image_features(**inputs))
features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8)
arr = features.cpu().numpy().astype(np.float32, copy=False)
# Close PIL handles to avoid leaking file handles on Windows
@ -116,7 +130,7 @@ def embed_texts(texts: Sequence[str]) -> np.ndarray:
max_length=77,
).to(_DEVICE)
with torch.no_grad():
features = _MODEL.get_text_features(**inputs)
features = _as_feature_tensor(_MODEL.get_text_features(**inputs))
features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8)
return features.cpu().numpy().astype(np.float32, copy=False)

View File

@ -26,7 +26,7 @@ from functools import lru_cache
@lru_cache(maxsize=1)
def _load_manifest_schema() -> dict:
with open(SCHEMA_PATH) as f:
with open(SCHEMA_PATH, encoding="utf-8") as f:
return json.load(f)
@ -61,7 +61,7 @@ def load_pipeline(name: str, defs_dir: Optional[Path] = None) -> dict[str, Any]:
if not path.exists():
raise FileNotFoundError(f"Pipeline manifest not found: {path}")
with open(path) as f:
with open(path, encoding="utf-8") as f:
manifest = yaml.safe_load(f)
schema = _load_manifest_schema()

View File

@ -15,6 +15,7 @@
"@remotion/player": "^4.0.484",
"@remotion/transitions": "^4.0.484",
"d3-geo": "^3.1.1",
"fast-uri": "^3.1.5",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"remotion": "^4.0.484",
@ -1951,9 +1952,9 @@
"license": "MIT"
},
"node_modules/fast-uri": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz",
"integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==",
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
"integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
"funding": [
{
"type": "github",
@ -2253,9 +2254,9 @@
"license": "MIT"
},
"node_modules/nanoid": {
"version": "3.3.15",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
"integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
"version": "3.3.16",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
"funding": [
{
"type": "github",
@ -2345,9 +2346,9 @@
"license": "ISC"
},
"node_modules/postcss": {
"version": "8.5.15",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
"version": "8.5.25",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz",
"integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==",
"funding": [
{
"type": "opencollective",
@ -2364,7 +2365,7 @@
],
"license": "MIT",
"dependencies": {
"nanoid": "^3.3.12",
"nanoid": "^3.3.16",
"picocolors": "^1.1.1",
"source-map-js": "^1.2.1"
},

View File

@ -15,6 +15,7 @@
"@remotion/player": "^4.0.484",
"@remotion/transitions": "^4.0.484",
"d3-geo": "^3.1.1",
"fast-uri": "^3.1.5",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"remotion": "^4.0.484",

View File

@ -8,31 +8,13 @@ import {
Sequence,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
function resolveAsset(src: string): string {
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
return src;
}
const clean = src.replace(/^file:\/\/\/?/, "");
if (clean.startsWith("/") || /^[A-Za-z]:[/\\]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
import { CinematicRendererProps, CinematicTone, CinematicVideoScene } from "./cinematic/types";
import { CaptionOverlay } from "./components/CaptionOverlay";
import { resolveAsset } from "./lib/resolveAsset";
const FPS = 30;
@ -57,10 +39,11 @@ const toneGradient = (tone: CinematicTone) => {
const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => {
const frame = useCurrentFrame();
const { durationInFrames, fps } = useVideoConfig();
const { fps } = useVideoConfig();
const durationInFrames = Math.max(1, Math.round(scene.durationSeconds * fps));
const fadeInFrames = scene.fadeInFrames ?? 10;
const fadeOutFrames = scene.fadeOutFrames ?? 10;
const fadeOutStart = Math.max(fadeInFrames, durationInFrames - fadeOutFrames);
const fadeOutStart = Math.max(0, durationInFrames - fadeOutFrames);
const fadeInOpacity =
fadeInFrames === 0
? 1
@ -98,6 +81,7 @@ const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => {
src={resolveAsset(scene.src)}
trimBefore={trimBefore}
trimAfter={trimAfter}
playbackRate={scene.playbackRate}
style={{
width: "100%",
height: "100%",

View File

@ -6,12 +6,12 @@ import {
interpolate,
random,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import React from "react";
import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay";
import { resolveAsset } from "./lib/resolveAsset";
const { fontFamily: playfairFamily } = loadPlayfair("normal", {
weights: ["400", "700"],
@ -22,23 +22,6 @@ const { fontFamily: playfairItalic } = loadPlayfair("italic", {
subsets: ["latin"],
});
function resolveAsset(src: string): string {
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src;
const clean = src.replace(/^file:\/\/\/?/, "");
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
export type CollageTransition =
| "pop"
| "slide-zoom"
@ -63,13 +46,13 @@ export interface CollageClip {
seed?: number;
}
export interface CollageBurstProps {
export type CollageBurstProps = {
backgroundSrc: string;
backgroundInSeconds?: number;
curtainStartSeconds: number;
curtainEndSeconds: number;
clips: CollageClip[];
}
};
// ----------------------------------------------------------------------------
// Opening text — elegant serif card that lives in the pre-reveal black, then

View File

@ -6,34 +6,10 @@ import {
Sequence,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import { loadFont } from "@remotion/google-fonts/SpaceGrotesk";
// Resolve asset path — handle URLs, absolute paths (Windows/Unix), and public/ relative paths
function resolveAsset(src: string): string {
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
return src;
}
// Strip any file:// prefix
const clean = src.replace(/^file:\/\/\/?/, "");
// Absolute paths (Unix: /foo, Windows: C:\foo or C:/foo) — convert to file:// URI
// staticFile() only accepts relative paths within public/, so absolute paths must bypass it
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
import { TextCard } from "./components/TextCard";
import { StatCard } from "./components/StatCard";
import { CalloutBox } from "./components/CalloutBox";
@ -54,6 +30,7 @@ import type { TerminalStep } from "./components/TerminalScene";
import { ScreenshotScene } from "./components/ScreenshotScene";
import type { ScreenshotStep } from "./components/ScreenshotScene";
import { ProviderChip } from "./components/ProviderChip";
import { resolveAsset } from "./lib/resolveAsset";
import type { ParticleType } from "./components/ParticleOverlay";
import { resolveTheme, type ThemeConfig, DEFAULT_THEME } from "./Root";
@ -254,6 +231,7 @@ interface Cut {
animation?: string;
transition_in?: string;
transition_out?: string;
transition_duration?: number;
transform?: {
animation?: string;
scale?: number;
@ -425,22 +403,49 @@ const ImageScene: React.FC<{ src: string; animation?: string }> = ({
// Enhanced Video Scene
// ---------------------------------------------------------------------------
const VideoScene: React.FC<{ src: string; startFrom?: number }> = ({
const VideoScene: React.FC<{
src: string;
startFrom?: number;
transitionIn?: string;
transitionOut?: string;
transitionDuration?: number;
sceneDurationSeconds: number;
backgroundColor?: string;
}> = ({
src,
startFrom = 0,
transitionIn,
transitionOut,
transitionDuration,
sceneDurationSeconds,
backgroundColor = "#0F172A",
}) => {
const frame = useCurrentFrame();
const { fps, durationInFrames } = useVideoConfig();
const { fps } = useVideoConfig();
const durationInFrames = Math.max(1, Math.round(sceneDurationSeconds * fps));
const fadeIn = spring({ frame, fps, config: { damping: 20 } });
const fadeOutStart = durationInFrames - 8;
const fadeOut = interpolate(frame, [fadeOutStart, durationInFrames], [1, 0.3], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const hardIn = ["cut", "none"].includes((transitionIn || "").toLowerCase());
const hardOut = ["cut", "none"].includes((transitionOut || "").toLowerCase());
const transitionFrames = Math.max(
1,
Math.round((transitionDuration ?? 8 / fps) * fps),
);
const fadeIn = hardIn
? 1
: interpolate(frame, [0, transitionFrames], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const fadeOutStart = Math.max(0, durationInFrames - transitionFrames);
const fadeOut = hardOut
? 1
: interpolate(frame, [fadeOutStart, durationInFrames], [1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill style={{ background: "#0F172A" }}>
<AbsoluteFill style={{ background: backgroundColor }}>
<OffthreadVideo
src={resolveAsset(src)}
startFrom={Math.round(startFrom * fps)}
@ -721,7 +726,17 @@ const SceneRenderer: React.FC<{ cut: Cut; theme: ThemeConfig }> = ({ cut, theme
}
if (cut.source && isVideo(cut.source)) {
return maybeWrapWithBg(<VideoScene src={cut.source} startFrom={cut.source_in_seconds ?? 0} />);
return maybeWrapWithBg(
<VideoScene
src={cut.source}
startFrom={cut.source_in_seconds ?? 0}
transitionIn={cut.transition_in}
transitionOut={cut.transition_out}
transitionDuration={cut.transition_duration}
sceneDurationSeconds={cut.out_seconds - cut.in_seconds}
backgroundColor={cut.backgroundColor}
/>,
);
}
// Final fallback — try as image if source exists, otherwise show text_card
@ -741,7 +756,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
if (overlay.type === "section_title") {
return (
<SectionTitle
title={overlay.text}
title={overlay.text ?? ""}
subtitle={overlay.subtitle}
accentColor={overlay.accentColor}
position={(overlay.position as any) || "top-left"}
@ -751,7 +766,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
if (overlay.type === "stat_reveal") {
return (
<StatReveal
stat={overlay.text}
stat={overlay.text ?? ""}
label={overlay.subtitle}
accentColor={overlay.accentColor}
position={(overlay.position as any) || "bottom-right"}
@ -759,7 +774,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
);
}
if (overlay.type === "hero_title") {
return <HeroTitle title={overlay.text} subtitle={overlay.subtitle} />;
return <HeroTitle title={overlay.text ?? ""} subtitle={overlay.subtitle} />;
}
if (overlay.type === "provider_chip" && overlay.providers) {
return (

View File

@ -3,46 +3,29 @@ import {
Audio,
OffthreadVideo,
interpolate,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import React from "react";
import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay";
import { resolveAsset } from "./lib/resolveAsset";
const { fontFamily: playfairItalic } = loadPlayfair("italic", {
weights: ["400", "700"],
subsets: ["latin"],
});
function resolveAsset(src: string): string {
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src;
const clean = src.replace(/^file:\/\/\/?/, "");
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
export interface Lyric {
text: string;
inSeconds: number;
outSeconds: number;
}
export interface LyricOverlayProps {
export type LyricOverlayProps = {
videoSrc: string;
lyrics: Lyric[];
bottomY?: number; // 0..1, vertical center of subtitle band
}
};
const LyricLine: React.FC<{ lyric: Lyric; bottomY: number }> = ({ lyric, bottomY }) => {
const frame = useCurrentFrame();

View File

@ -5,12 +5,12 @@ import {
Sequence,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import { getVideoMetadata } from "@remotion/media-utils";
import { loadFont } from "@remotion/google-fonts/PlayfairDisplay";
import { resolveAsset } from "./lib/resolveAsset";
// Editorial serif for the tagline — Playfair Display at its boldest weight.
// Loaded once at module scope so every render reuses the same font face.
@ -19,7 +19,7 @@ const { fontFamily } = loadFont("normal", {
subsets: ["latin"],
});
export interface TitledVideoProps {
export type TitledVideoProps = {
videoSrc: string;
tagline: string;
// When the tagline starts animating in, in seconds from the start of the video.
@ -32,33 +32,10 @@ export interface TitledVideoProps {
fontSize?: number;
// Accent color used for the underline and the glow halo.
accentColor?: string;
}
};
// Resolve asset path — handle URLs, absolute paths, and public/ relative paths.
// Mirrors the helper in Explainer.tsx so absolute Windows/Unix paths work.
function resolveAsset(src: string): string {
if (
src.startsWith("http://") ||
src.startsWith("https://") ||
src.startsWith("data:")
) {
return src;
}
const clean = src.replace(/^file:\/\/\/?/, "");
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
// ---------------------------------------------------------------------------
// EditorialTagline — big bold serif, upper-third, drawn underline, warm glow.
// Letter-by-letter spring entrance. Designed to feel like a printed headline,

View File

@ -12,6 +12,7 @@ export interface CinematicVideoScene extends CinematicBaseScene {
tone?: CinematicTone;
trimBeforeSeconds?: number;
trimAfterSeconds?: number;
playbackRate?: number;
filter?: string;
fadeInFrames?: number;
fadeOutFrames?: number;

View File

@ -3,27 +3,11 @@ import {
Img,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import { ParticleOverlay, type ParticleType } from "./ParticleOverlay";
/**
* Resolve asset path use staticFile() for local paths, passthrough URLs.
* Duplicated from Explainer.tsx to keep the component self-contained.
*/
function resolveAsset(src: string): string {
if (
src.startsWith("http://") ||
src.startsWith("https://") ||
src.startsWith("data:")
) {
return src;
}
const clean = src.replace(/^file:\/\/\/?/, "");
return staticFile(clean);
}
import { resolveAsset } from "../lib/resolveAsset";
// ---------------------------------------------------------------------------
// Types

View File

@ -14,7 +14,7 @@ export interface WordCaption {
endMs: number;
}
interface CaptionOverlayProps {
type CaptionOverlayProps = {
words: WordCaption[];
// How many words to show at once in a "page"
wordsPerPage?: number;
@ -23,7 +23,7 @@ interface CaptionOverlayProps {
highlightColor?: string;
backgroundColor?: string;
fontFamily?: string;
}
};
interface CaptionPage {
words: WordCaption[];

View File

@ -6,7 +6,7 @@ import {
useVideoConfig,
} from "remotion";
export interface EndTagProps {
export type EndTagProps = {
text: string;
palette?: "cool_offwhite_on_black" | "warm_ivory_on_black";
// Optional extra fade hold controls (all in seconds)
@ -19,7 +19,7 @@ export interface EndTagProps {
// AbsoluteFill drops its background fill — caller is responsible for
// rendering with an alpha-capable codec (VP9/WebM or ProRes 4444).
overlay?: boolean;
}
};
const PALETTES = {
cool_offwhite_on_black: {

View File

@ -6,10 +6,10 @@ import {
useVideoConfig,
} from "remotion";
interface HeroTitleProps {
type HeroTitleProps = {
title: string;
subtitle?: string;
}
};
export const HeroTitle: React.FC<HeroTitleProps> = ({ title, subtitle }) => {
const frame = useCurrentFrame();

View File

@ -9,14 +9,14 @@ import {
Easing,
} from "remotion";
export interface ProductRevealProps {
export type ProductRevealProps = {
productImage: string;
productName: string;
price: string;
tagline: string;
closer: string;
accentColor?: string;
}
};
export const ProductReveal: React.FC<ProductRevealProps> = ({
productImage,

View File

@ -53,7 +53,7 @@ export const ProviderChip: React.FC<ProviderChipProps> = ({
const translateY = interpolate(springIn, [0, 1], [12, 0]);
return (
<AbsoluteFill pointerEvents="none">
<AbsoluteFill style={{ pointerEvents: "none" }}>
<div
style={{
position: "absolute",

View File

@ -3,10 +3,10 @@ import {
Img,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import { resolveAsset } from "../lib/resolveAsset";
/**
* ScreenshotScene approach-1 synthetic UI demo.
@ -84,25 +84,6 @@ interface ScreenshotSceneProps {
// ---------- Helpers ----------
function resolveAsset(src: string): string {
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
return src;
}
const clean = src.replace(/^file:\/\/\/?/, "");
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
const posix = clean.replace(/\\/g, "/");
// POSIX absolute paths already have a leading "/" — file:// + posix
// gives exactly three slashes. Windows drive paths (C:/...) need the
// extra slash added explicitly. Do not merge these branches — adding
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
if (posix.startsWith("/")) {
return `file://${posix}`;
}
return `file:///${posix}`;
}
return staticFile(clean);
}
/** Compute the rendered bounding box of the backdrop inside a canvas,
* using object-fit: contain semantics. Returns pixel offsets/sizes. */
function containRect(

View File

@ -0,0 +1,28 @@
import { staticFile } from "remotion";
const isRemoteAsset = (src: string): boolean =>
src.startsWith("http://") ||
src.startsWith("https://") ||
src.startsWith("data:");
const isWindowsAbsolutePath = (src: string): boolean =>
/^[A-Za-z]:[\\/]/.test(src);
/** Resolve public assets and absolute filesystem paths consistently. */
export function resolveAsset(src: string): string {
if (isRemoteAsset(src)) {
return src;
}
const withoutScheme = src.replace(/^file:\/\//i, "");
const clean = /^\/[A-Za-z]:[\\/]/.test(withoutScheme)
? withoutScheme.slice(1)
: withoutScheme;
if (clean.startsWith("/") || isWindowsAbsolutePath(clean)) {
const posix = clean.replace(/\\/g, "/");
return posix.startsWith("/") ? `file://${posix}` : `file:///${posix}`;
}
return staticFile(clean);
}

View File

@ -39,7 +39,7 @@ def load_schema(name: str) -> dict:
path = SCHEMA_DIR / f"{name}.schema.json"
if not path.exists():
raise FileNotFoundError(f"Schema not found: {path}")
with open(path) as f:
with open(path, encoding="utf-8") as f:
return json.load(f)

View File

@ -57,6 +57,10 @@
"minimum": 0,
"description": "Duration of transition in seconds"
},
"backgroundColor": {
"type": "string",
"description": "Backing color used behind this cut by composition runtimes"
},
"reason": { "type": "string" }
},
"additionalProperties": false

View File

@ -11,7 +11,7 @@
"description": { "type": "string" },
"category": {
"type": "string",
"enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "custom"]
"enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "documentary", "custom"]
},
"stability": {
"type": "string",

View File

@ -184,6 +184,15 @@ def stage_project(pid: str, title: str, palette: str, scenes: list, *,
brief["topic"] = title
cp("research", "completed", {"research_brief": brief})
proposal = sample_artifact("proposal_packet")
cp("proposal", "awaiting_human", {"proposal_packet": proposal})
cp(
"proposal",
"completed",
{"proposal_packet": proposal},
human_approved=True,
)
script = script_artifact(title, scenes)
plan = scene_plan_artifact(scenes, hero)
(art_dir / "decision_log.json").write_text(json.dumps(decision_log(pid), indent=2))

View File

@ -60,7 +60,9 @@ Key capability families to look for in the output:
| `analysis` | — | Mixed providers |
| `character_animation` | — | Local character specs, SVG rigs, pose libraries, action timelines, previews, and QA |
| `graphics` | — | Local rendering tools |
| `music_generation` | — | Single-provider |
| `music_library` | — | Discovers user-provided local tracks |
| `music_search` | — | Discovers royalty-free search/download providers |
| `music_generation` | — | Discovers paid/local generation providers |
| `subtitle` | — | Pure Python |
| `avatar` | — | Local GPU models |
| `video_post` | — | FFmpeg-based local tools |

View File

@ -88,23 +88,26 @@ Cinematic videos live and die by their audio. **Surface the music situation befo
Check availability in this order:
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. List available tracks with durations and moods. Let the user choose.
2. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, and cost per track.
3. **Royalty-free sources:** Note that the user can provide a track from YouTube Audio Library, Jamendo, or other free sources by dropping it in `music_library/`.
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List available tracks with durations and moods.
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report available providers, licensing constraints, and any key requirement.
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs.
4. **Bring your own:** The user can drop a track from another licensed source into `music_library/`.
Present explicit options:
```
MUSIC PLAN
├── Your music library: [N tracks / empty]
├── Royalty-free search: [providers / unavailable]
├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost]
└── Bring your own: Drop a track in music_library/ before asset stage
Options:
(a) Use a library track (which one?)
(b) Provide your own track
(c) Generate via API (if available)
(d) Proceed without music (not recommended for cinematic)
(b) Search a royalty-free provider
(c) Provide your own track
(d) Generate via API (if available)
(e) Proceed without music (not recommended for cinematic)
```
Record the decision in `brief.metadata.music_strategy` with the chosen source and path/prompt.

View File

@ -222,14 +222,16 @@ Let the user select, combine, modify, or redirect entirely.
Cinematic videos live and die by their audio. Surface the music situation before the user approves.
Check availability in this order:
1. **User music library (`music_library/`)** — list available tracks
2. **Music generation APIs** — report status, cost, and quality honestly
3. **Bring-your-own path** — user can drop a track in `music_library/`
1. **User music library** — query `registry.get_by_capability("music_library")` and list available tracks
2. **Royalty-free search** — query `registry.get_by_capability("music_search")` and report providers/licensing
3. **Music generation APIs** — query `registry.get_by_capability("music_generation")` and report status, cost, and quality honestly
4. **Bring-your-own path** — user can drop a track in `music_library/`
Present explicit options:
```
MUSIC PLAN
├── Your music library: [N tracks / empty]
├── Royalty-free search: [providers / unavailable]
├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost]
└── Bring your own: Drop a track in music_library/ before asset stage

View File

@ -89,11 +89,14 @@ like abandoned footage at compose time. Do not assume silence will earn
itself. If the user has not mentioned music, ASSUME THEY WANT IT and pick:
- user-provided track (put path in `music_plan.source_path`),
- music library pick (list what's in `music_library/`),
- music library pick (query `registry.get_by_capability("music_library")` and list tracks),
- royalty-free search (query `registry.get_by_capability("music_search")`, report provider and license),
- generated (name the tool and prompt seed with register),
- explicit opt-out (`source: "none"` + `opt_out_reason`).
**Warn the user if no music source is available.** Do not silently
Before declaring no source available, also query
`registry.get_by_capability("music_generation")`. **Warn the user if no music
source is available.** Do not silently
defer this — it becomes an expensive surprise at the asset stage.
### 5. Note End-Tag Intent (MANDATORY)

View File

@ -129,7 +129,9 @@ If any dimension scores below 3, iterate before submitting. The reviewer will ch
### Step 7: Submit
Call `handle_explainer_idea(state, {"brief": brief_json})` to validate and persist.
Validate `brief_json` against the canonical brief schema, persist it through
the checkpoint protocol, and attach the stage review. There is no separate
explainer submit function.
## Playbook Selection Guide

View File

@ -369,9 +369,9 @@ Music is a critical part of the video's feel. **Surface the music situation to t
**Check music availability in this order:**
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one.
2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly.
3. **Stock music sources:** Note if stock music is available via any provider.
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List tracks with durations.
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report providers, licensing constraints, and key requirements.
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly.
**Present to the user:**
@ -381,15 +381,17 @@ MUSIC PLAN
│ ├── cosmic_interstellar_space.mp3 (3:13) — ambient, cosmic
│ ├── cinematic_epic.mp3 (2:45) — dramatic, building
│ └── lofi_beat.mp3 (4:00) — chill, electronic
├── Royalty-free search: [providers / unavailable]
├── AI generation: music_gen (ElevenLabs) — UNAVAILABLE (plan limit)
└── Recommendation: Use "cosmic_interstellar_space.mp3" from your library
OR provide a different track before asset generation
Would you like to:
(a) Use a track from your library (which one?)
(b) Provide a different track (drop it in music_library/)
(c) Generate one via API (if available)
(d) Proceed without music
(b) Search a royalty-free provider
(c) Provide a different track (drop it in music_library/)
(d) Generate one via API (if available)
(e) Proceed without music
```
**If no music source is available:** Tell the user explicitly. Do NOT let this surface as a surprise at the asset stage. Offer the `music_library/` path so they can add a track before production starts.

View File

@ -63,9 +63,17 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v
}
```
#### Scene Types and When to Use Them
#### Render Templates and Scene Types
| Type | Best For | Available Tools | Duration Guidance |
The canonical `scene_plan.scenes[].type` vocabulary is `talking_head`, `broll`,
`animation`, `character_scene`, `diagram`, `text_card`, `transition`,
`generated`, and `screen_recording`. Names such as `hero_title`, `stat_card`,
and chart/card variants below are downstream Remotion `cut.type` templates,
not valid scene-plan types. During scene planning, use `text_card` or
`animation` as appropriate and state the intended render template in the scene
description; the Edit stage converts that intent into `cut.type`.
| Render template or scene type | Best For | Available Tools | Duration Guidance |
|------|----------|-----------------|-------------------|
| `hero_title` | Opening titles, dramatic reveals | Remotion HeroTitle (theme-driven title treatment) | 3-5s |
| `stat_card` | Big dramatic numbers, impactful metrics | Remotion StatCard (large stat + subtitle) | 4-6s |
@ -84,7 +92,11 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v
| `broll` | Context, real-world examples | Stock or generated footage | 3-6s |
| `screen_recording` | Code demos, UI walkthroughs | Recorded or simulated | 5-15s |
**Zero-key scene selection:** When no image/video generation is available, prefer `hero_title`, `stat_card`, `bar_chart`, `line_chart`, `pie_chart`, `kpi_grid`, `comparison`, `callout`, `progress_bar`, and `text_card`. These render entirely from Remotion components with zero external dependencies and can still feel distinct if you derive color, typography, and pacing from the subject instead of defaulting to a generic dashboard aesthetic.
**Zero-key scene selection:** When no image/video generation is available,
plan `text_card`, `animation`, or `diagram` scenes and name an appropriate
downstream template (`hero_title`, `stat_card`, charts, `comparison`,
`callout`, or `progress_bar`) in each description. These render entirely from
Remotion components with zero external dependencies.
### Step 4: Apply the Visual Technique Library
@ -226,7 +238,9 @@ If any dimension scores below 3, revise.
### Step 8: Submit
Call `handle_explainer_scene_plan(state, {"scene_plan": scene_plan_json})` to validate and persist.
Validate `scene_plan_json` against the canonical scene-plan schema, persist it
through the checkpoint protocol, and attach the stage review. There is no
separate explainer submit function.
## Common Pitfalls

View File

@ -206,7 +206,9 @@ If any dimension scores below 3, revise before submitting.
### Step 7: Submit
Call `handle_explainer_script(state, {"script": script_json})` to validate and persist.
Validate `script_json` against the canonical script schema, persist it through
the checkpoint protocol, and attach the stage review. There is no separate
explainer submit function.
### Mid-Production Fact Verification

View File

@ -26,7 +26,7 @@ SCHEMA_PATH = (
def _load_playbook_schema() -> dict:
with open(SCHEMA_PATH) as f:
with open(SCHEMA_PATH, encoding="utf-8") as f:
return json.load(f)
@ -45,7 +45,7 @@ def load_playbook(name: str, styles_dir: Optional[Path] = None) -> dict[str, Any
if not path.exists():
raise FileNotFoundError(f"Playbook not found: {path}")
with open(path) as f:
with open(path, encoding="utf-8") as f:
playbook = yaml.safe_load(f)
validate_playbook(playbook)

View File

@ -7,7 +7,11 @@ import pytest
from backlot import state as state_mod
from backlot.state import load_board_state
from lib.checkpoint import CheckpointValidationError, write_checkpoint
from lib.checkpoint import (
CANONICAL_STAGE_ARTIFACTS,
CheckpointValidationError,
write_checkpoint,
)
def _script_artifact() -> dict:
@ -23,6 +27,22 @@ def _manifest_artifact() -> dict:
return {"version": "1.0", "assets": [], "total_cost_usd": 0.0}
def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None:
from tests.contracts.test_phase0_contracts import sample_artifact
for stage in stages:
artifact_name = CANONICAL_STAGE_ARTIFACTS[stage]
write_checkpoint(
tmp_path,
project_id,
stage,
"completed",
{artifact_name: sample_artifact(artifact_name)},
pipeline_type=pipeline_type,
human_approved=True,
)
def _write(path: Path, data: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data), encoding="utf-8")
@ -73,6 +93,15 @@ def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypat
def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path):
_approve_predecessors(
tmp_path,
"film",
"cinematic",
"research",
"proposal",
"script",
"scene_plan",
)
write_checkpoint(
tmp_path,
"film",

View File

@ -10,7 +10,8 @@ import urllib.request
import pytest
from lib.checkpoint import init_project, write_checkpoint
from lib.checkpoint import CANONICAL_STAGE_ARTIFACTS, init_project, write_checkpoint
from lib.pipeline_loader import get_stage_order, load_pipeline
from scripts import backlot_screenshot_stage
from tests.contracts.test_phase0_contracts import sample_artifact
@ -32,6 +33,28 @@ APPROVAL_CASES = [
]
def _complete_predecessors(root, project_id: str, pipeline_type: str, stage: str) -> None:
order = get_stage_order(load_pipeline(pipeline_type))
for predecessor in order[: order.index(stage)]:
artifact_name = CANONICAL_STAGE_ARTIFACTS.get(predecessor)
if artifact_name:
artifact = sample_artifact(artifact_name)
if artifact_name == "edit_decisions":
artifact["render_runtime"] = "ffmpeg"
artifacts = {artifact_name: artifact}
else:
artifacts = {}
write_checkpoint(
root,
project_id,
predecessor,
"completed",
artifacts,
pipeline_type=pipeline_type,
human_approved=True,
)
def _build_approval_projects() -> None:
root = backlot_screenshot_stage.STAGE_DIR
for project_id, pipeline_type, stage, artifact_name, _visible_text in APPROVAL_CASES:
@ -55,6 +78,7 @@ def _build_approval_projects() -> None:
pipeline_type=pipeline_type,
pipeline_dir=root,
)
_complete_predecessors(root, project_id, pipeline_type, stage)
write_checkpoint(
root,
project_id,
@ -80,6 +104,12 @@ def _build_approval_projects() -> None:
pipeline_type="character-animation",
pipeline_dir=root,
)
_complete_predecessors(
root,
"gate-character-design",
"character-animation",
"character_design",
)
write_checkpoint(
root,
"gate-character-design",

View File

@ -0,0 +1,31 @@
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
def _read(relative_path: str) -> str:
return (REPO_ROOT / relative_path).read_text(encoding="utf-8")
def test_music_plans_discover_all_music_capabilities() -> None:
instruction_files = [
"AGENT_GUIDE.md",
"skills/pipelines/cinematic/idea-director.md",
"skills/pipelines/cinematic/proposal-director.md",
"skills/pipelines/documentary-montage/idea-director.md",
"skills/pipelines/explainer/proposal-director.md",
]
for relative_path in instruction_files:
text = _read(relative_path)
for capability in ("music_library", "music_search", "music_generation"):
assert f'get_by_capability("{capability}")' in text, (
f"{relative_path} omits the {capability!r} music source"
)
def test_explainer_directors_do_not_reference_fictitious_submit_functions() -> None:
for stage in ("idea", "script", "scene"):
text = _read(f"skills/pipelines/explainer/{stage}-director.md")
assert "handle_explainer_" not in text

View File

@ -6,6 +6,7 @@ import json
import pytest
from lib.checkpoint import (
CANONICAL_STAGE_ARTIFACTS,
CheckpointValidationError,
HISTORY_DIRNAME,
PROJECT_MARKER_FILENAME,
@ -27,6 +28,22 @@ def _minimal_script() -> dict:
}
def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None:
from tests.contracts.test_phase0_contracts import sample_artifact
for stage in stages:
artifact_name = CANONICAL_STAGE_ARTIFACTS[stage]
write_checkpoint(
tmp_path,
project_id,
stage,
"completed",
artifacts={artifact_name: sample_artifact(artifact_name)},
pipeline_type=pipeline_type,
human_approved=True,
)
class TestGateEnforcement:
"""GI-4: gated stages cannot be completed without approval evidence."""
@ -39,6 +56,9 @@ class TestGateEnforcement:
)
def test_awaiting_human_is_the_correct_gate_state(self, tmp_path):
_approve_predecessors(
tmp_path, "proj", "animated-explainer", "research", "proposal"
)
path = write_checkpoint(
tmp_path, "proj", "script", "awaiting_human",
artifacts={"script": _minimal_script()},
@ -51,6 +71,9 @@ class TestGateEnforcement:
assert cp["human_approval_required"] is True
def test_completed_with_approval_passes(self, tmp_path):
_approve_predecessors(
tmp_path, "proj", "animated-explainer", "research", "proposal"
)
path = write_checkpoint(
tmp_path, "proj", "script", "completed",
artifacts={"script": _minimal_script()},
@ -84,6 +107,9 @@ class TestCheckpointHistory:
"""Superseded checkpoints are archived, not destroyed."""
def test_overwrite_archives_previous(self, tmp_path):
_approve_predecessors(
tmp_path, "proj", "animated-explainer", "research", "proposal"
)
write_checkpoint(
tmp_path, "proj", "script", "awaiting_human",
artifacts={"script": _minimal_script()},

View File

@ -0,0 +1,19 @@
from pathlib import Path
from dotenv import dotenv_values
REPO_ROOT = Path(__file__).resolve().parents[2]
def test_env_example_does_not_turn_comments_into_credentials() -> None:
"""A fresh copy of .env.example must leave every documented key unset."""
values = dotenv_values(REPO_ROOT / ".env.example")
false_credentials = {
key: value
for key, value in values.items()
if isinstance(value, str) and value.lstrip().startswith("#")
}
assert false_credentials == {}

View File

@ -174,10 +174,14 @@ class TestPhase2ErrorHandling:
# Either succeeds (provider available) or fails gracefully
assert isinstance(r, ToolResult)
def test_diagram_gen_empty_boxes(self):
def test_diagram_gen_empty_boxes(self, tmp_path):
tool = DiagramGen()
if tool.get_status() == ToolStatus.AVAILABLE:
r = tool.execute({"diagram_type": "boxes", "boxes": []})
r = tool.execute({
"diagram_type": "boxes",
"boxes": [],
"output_path": str(tmp_path / "empty-boxes.png"),
})
assert isinstance(r, ToolResult)

View File

@ -9,6 +9,7 @@ import builtins
import base64
import os
import shutil
from types import SimpleNamespace
from pathlib import Path
from unittest.mock import MagicMock, patch
@ -571,16 +572,26 @@ class TestVeoVideo:
called_kwargs = mock_client.models.generate_videos.call_args[1]
assert called_kwargs["image"] is not None
def test_vertex_ai_mode_rejection(self):
def test_vertex_ai_mode_requires_inline_video_bytes(self):
tool = VeoVideo()
mock_client = MagicMock()
mock_client.vertexai = True
if hasattr(mock_client, "_api_client"):
delattr(mock_client, "_api_client")
mock_client.models.generate_videos.return_value = SimpleNamespace(
done=True,
error=None,
response=SimpleNamespace(
generated_videos=[
SimpleNamespace(video=SimpleNamespace(video_bytes=None))
]
),
)
with (
patch.dict(os.environ, {"GEMINI_API_KEY": "test_key"}),
patch("google.genai.Client", return_value=mock_client),
patch(
"tools.google_credentials.get_genai_client",
return_value=mock_client,
),
):
inputs = {
"prompt": "cinematic shot",
@ -589,7 +600,7 @@ class TestVeoVideo:
res = tool.execute(inputs)
assert res.success is False
assert res.error is not None
assert "only supported using the Gemini Developer API" in res.error
assert "without inline bytes" in res.error
def test_missing_local_image_paths(self):
tool = VeoVideo()

View File

@ -0,0 +1,9 @@
"""Contracts for the category vocabulary used by shipped pipelines."""
from lib.pipeline_loader import load_pipeline
def test_documentary_pipeline_uses_a_schema_supported_category() -> None:
manifest = load_pipeline("documentary-montage")
assert manifest["category"] == "documentary"

View File

@ -0,0 +1,28 @@
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
def test_video_scene_honors_hard_cut_tokens_and_backing_color() -> None:
source = (REPO_ROOT / "remotion-composer/src/Explainer.tsx").read_text(
encoding="utf-8"
)
assert '["cut", "none"].includes((transitionIn || "").toLowerCase())' in source
assert '["cut", "none"].includes((transitionOut || "").toLowerCase())' in source
assert "transitionIn={cut.transition_in}" in source
assert "transitionOut={cut.transition_out}" in source
assert "sceneDurationSeconds={cut.out_seconds - cut.in_seconds}" in source
assert "Math.round(sceneDurationSeconds * fps)" in source
assert "durationInFrames - transitionFrames" in source
assert "backgroundColor={cut.backgroundColor}" in source
def test_cinematic_fades_are_bounded_by_each_scene_duration() -> None:
source = (REPO_ROOT / "remotion-composer/src/CinematicRenderer.tsx").read_text(
encoding="utf-8"
)
assert "Math.round(scene.durationSeconds * fps)" in source
assert "durationInFrames - fadeOutFrames" in source

View File

@ -0,0 +1,30 @@
import ast
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[2]
RUNTIME_FILES = [
"lib/checkpoint.py",
"lib/pipeline_loader.py",
"schemas/artifacts/__init__.py",
"styles/playbook_loader.py",
]
@pytest.mark.parametrize("relative_path", RUNTIME_FILES)
def test_pipeline_contract_files_use_explicit_utf8(relative_path: str) -> None:
path = REPO_ROOT / relative_path
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
bare_opens = []
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
if not isinstance(node.func, ast.Name) or node.func.id != "open":
continue
if not any(keyword.arg == "encoding" for keyword in node.keywords):
bare_opens.append(node.lineno)
assert bare_opens == [], f"bare open() calls at lines {bare_opens}"

View File

@ -0,0 +1,152 @@
import json
import pytest
from tests.contracts.test_phase0_contracts import sample_artifact
from lib.checkpoint import (
CheckpointValidationError,
init_project,
write_checkpoint,
)
def _script_artifact() -> dict:
return {
"version": "1.0",
"title": "Smoke",
"total_duration_seconds": 1,
"sections": [
{
"id": "s1",
"text": "One second.",
"start_seconds": 0,
"end_seconds": 1,
}
],
}
def test_later_stage_cannot_skip_a_missing_predecessor(tmp_path) -> None:
init_project(
"run",
title="Run",
pipeline_type="framework-smoke",
pipeline_dir=tmp_path,
)
with pytest.raises(CheckpointValidationError, match="PREREQUISITE VIOLATION"):
write_checkpoint(
tmp_path,
"run",
"script",
"completed",
{"script": _script_artifact()},
pipeline_type="framework-smoke",
human_approved=True,
)
def test_later_stage_rejects_unapproved_gated_predecessor(tmp_path) -> None:
project_dir = init_project(
"run",
title="Run",
pipeline_type="framework-smoke",
pipeline_dir=tmp_path,
)
predecessor_path = write_checkpoint(
tmp_path,
"run",
"research",
"awaiting_human",
{"research_brief": sample_artifact("research_brief")},
pipeline_type="framework-smoke",
)
predecessor = json.loads(predecessor_path.read_text(encoding="utf-8"))
predecessor["status"] = "completed"
predecessor["human_approved"] = False
predecessor_path.write_text(json.dumps(predecessor), encoding="utf-8")
with pytest.raises(CheckpointValidationError, match="completed without required approval"):
write_checkpoint(
tmp_path,
"run",
"script",
"completed",
{"script": _script_artifact()},
pipeline_type="framework-smoke",
human_approved=True,
)
def test_malformed_predecessor_cannot_forge_completion(tmp_path) -> None:
project_dir = init_project(
"run",
title="Run",
pipeline_type="framework-smoke",
pipeline_dir=tmp_path,
)
(project_dir / "checkpoint_research.json").write_text(
json.dumps({"status": "completed", "human_approved": True}),
encoding="utf-8",
)
with pytest.raises(CheckpointValidationError, match="incomplete or missing"):
write_checkpoint(
tmp_path,
"run",
"script",
"completed",
{"script": _script_artifact()},
pipeline_type="framework-smoke",
human_approved=True,
)
def test_in_progress_heartbeat_is_not_blocked_by_prerequisites(tmp_path) -> None:
init_project(
"run",
title="Run",
pipeline_type="framework-smoke",
pipeline_dir=tmp_path,
)
path = write_checkpoint(
tmp_path,
"run",
"script",
"in_progress",
{},
pipeline_type="framework-smoke",
)
assert path.exists()
def test_unknown_style_playbook_fails_before_project_creation(tmp_path) -> None:
with pytest.raises(CheckpointValidationError, match="style_playbook"):
init_project(
"run",
title="Run",
pipeline_type="framework-smoke",
pipeline_dir=tmp_path,
style_playbook="does-not-exist",
)
assert not (tmp_path / "run").exists()
def test_marker_derived_unknown_playbook_blocks_later_writes(tmp_path) -> None:
project_dir = tmp_path / "run"
project_dir.mkdir()
(project_dir / "project.json").write_text(
json.dumps({
"version": "1.0",
"project_id": "run",
"pipeline_type": "framework-smoke",
"style_playbook": "does-not-exist",
}),
encoding="utf-8",
)
with pytest.raises(CheckpointValidationError, match="style_playbook"):
write_checkpoint(tmp_path, "run", "research", "in_progress", {})

View File

@ -0,0 +1,26 @@
from lib.clip_embedder import _as_feature_tensor
class _Tensor:
pass
class _ModelOutput:
def __init__(self, pooler_output):
self.pooler_output = pooler_output
self.last_hidden_state = object()
def test_transformers_4_tensor_passes_through() -> None:
tensor = _Tensor()
assert _as_feature_tensor(tensor) is tensor
def test_transformers_5_output_unwraps_projected_pooler_output() -> None:
tensor = _Tensor()
assert _as_feature_tensor(_ModelOutput(tensor)) is tensor
def test_missing_pooler_output_does_not_replace_features_with_none() -> None:
output = _ModelOutput(None)
assert _as_feature_tensor(output) is output

View File

@ -0,0 +1,94 @@
import shutil
import subprocess
import math
import struct
import wave
from pathlib import Path
import pytest
from tools.audio.audio_mixer import AudioMixer
pytestmark = pytest.mark.skipif(
shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
reason="ffmpeg and ffprobe are required",
)
def _tone(path: Path, frequency: int, duration: float) -> None:
subprocess.run(
[
"ffmpeg", "-y", "-f", "lavfi", "-i",
f"sine=frequency={frequency}:duration={duration}", str(path),
],
capture_output=True,
check=True,
timeout=30,
)
def _duration(path: Path) -> float:
result = subprocess.run(
[
"ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "csv=p=0", str(path),
],
capture_output=True,
check=True,
text=True,
timeout=30,
)
return float(result.stdout.strip())
def _tail_rms(path: Path, start_seconds: float, end_seconds: float) -> float:
with wave.open(str(path), "rb") as handle:
sample_width = handle.getsampwidth()
assert sample_width == 2
frame_rate = handle.getframerate()
handle.setpos(int(start_seconds * frame_rate))
raw = handle.readframes(int((end_seconds - start_seconds) * frame_rate))
samples = struct.unpack(f"<{len(raw) // 2}h", raw)
return math.sqrt(sum(sample * sample for sample in samples) / len(samples))
def test_full_mix_can_pin_the_composition_length(tmp_path) -> None:
speech = tmp_path / "speech.wav"
music = tmp_path / "music.wav"
output = tmp_path / "mix.wav"
_tone(speech, 440, 1)
_tone(music, 220, 3)
result = AudioMixer().execute({
"operation": "full_mix",
"tracks": [
{"path": str(speech), "role": "speech"},
{"path": str(music), "role": "music"},
],
"ducking": {"enabled": True},
"normalize": False,
"target_duration": 3,
"output_path": str(output),
})
assert result.success, result.error
assert result.data["target_duration"] == 3
assert _duration(output) == pytest.approx(3.0, abs=0.15)
assert _tail_rms(output, 2.0, 2.5) > 100
@pytest.mark.parametrize("target", [0, -1, "not-a-number"])
def test_full_mix_rejects_invalid_target_duration(tmp_path, target) -> None:
tone = tmp_path / "tone.wav"
_tone(tone, 440, 0.25)
result = AudioMixer().execute({
"operation": "full_mix",
"tracks": [{"path": str(tone), "role": "speech"}],
"target_duration": target,
"output_path": str(tmp_path / "mix.wav"),
})
assert result.success is False
assert "target_duration" in result.error

View File

@ -0,0 +1,28 @@
import sys
from unittest.mock import MagicMock
from PIL import Image
def test_bg_remove_selects_model_through_rembg_session(monkeypatch, tmp_path) -> None:
fake_rembg = MagicMock()
fake_rembg.new_session.return_value = "selected-session"
fake_rembg.remove.side_effect = lambda image, **kwargs: image.convert("RGBA")
monkeypatch.setitem(sys.modules, "rembg", fake_rembg)
input_path = tmp_path / "input.png"
Image.new("RGB", (8, 8), (10, 20, 30)).save(input_path)
from tools.enhancement.bg_remove import BgRemove
result = BgRemove().execute({
"input_path": str(input_path),
"output_path": str(tmp_path / "output.png"),
"model": "isnet-general-use",
})
assert result.success, result.error
fake_rembg.new_session.assert_called_once_with("isnet-general-use")
kwargs = fake_rembg.remove.call_args.kwargs
assert kwargs["session"] == "selected-session"
assert "model_name" not in kwargs

View File

@ -0,0 +1,177 @@
import json
from pathlib import Path
import pytest
from tools.video.video_compose import VideoCompose
def test_cinematic_cut_adapter_builds_a_sequential_timeline() -> None:
scenes = VideoCompose._cuts_to_cinematic_scenes([
{
"id": "v1",
"source": "clip.mp4",
"in_seconds": 2,
"out_seconds": 6,
"transition_in": "cut",
"transition_out": "none",
},
{
"id": "title",
"source": "",
"type": "hero_title",
"text": "The signal arrives",
"in_seconds": 0,
"out_seconds": 3,
},
])
assert scenes[0] == {
"id": "v1",
"startSeconds": 0.0,
"durationSeconds": 4.0,
"kind": "video",
"src": "clip.mp4",
"trimBeforeSeconds": 2.0,
"trimAfterSeconds": 6.0,
"playbackRate": 1.0,
"fadeInFrames": 0,
"fadeOutFrames": 0,
}
assert scenes[1]["kind"] == "title"
assert scenes[1]["startSeconds"] == 4.0
assert scenes[1]["text"] == "The signal arrives"
def test_cinematic_cut_adapter_preserves_playback_speed() -> None:
scenes = VideoCompose._cuts_to_cinematic_scenes([
{
"id": "fast",
"source": "clip.mp4",
"in_seconds": 2,
"out_seconds": 6,
"speed": 2,
}
])
assert scenes[0]["durationSeconds"] == 2
assert scenes[0]["playbackRate"] == 2
@pytest.mark.parametrize("uri_style", ["standard", "legacy_windows"])
def test_remotion_media_staging_decodes_file_uris(tmp_path, uri_style) -> None:
source = tmp_path / "clip with space.mp4"
source.write_bytes(b"video")
public_dir = tmp_path / "public"
uri = source.as_uri()
if uri_style == "legacy_windows" and len(source.drive) == 2:
uri = f"file://{source.drive}{source.as_posix()[2:]}".replace(" ", "%20")
props = {"scenes": [{"src": uri}]}
staged_count = VideoCompose._stage_remotion_media(props, public_dir)
assert staged_count == 1
assert props["scenes"][0]["src"] != uri
assert (public_dir / props["scenes"][0]["src"]).read_bytes() == b"video"
def test_remotion_render_adapts_cuts_and_stages_local_video(monkeypatch, tmp_path) -> None:
source = tmp_path / "source.mp4"
source.write_bytes(b"not-a-real-video")
output = tmp_path / "render.mp4"
captured = {}
def fake_run_command(self, command, **kwargs):
captured["command"] = command
captured["timeout"] = kwargs["timeout"]
props_arg = next(arg for arg in command if arg.startswith("--props="))
captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text())
public_arg = next(arg for arg in command if arg.startswith("--public-dir="))
public_dir = Path(public_arg.split("=", 1)[1])
captured["staged_exists_during_render"] = (
public_dir / captured["props"]["scenes"][0]["src"]
).exists()
output.write_bytes(b"rendered")
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
result = VideoCompose()._remotion_render({
"edit_decisions": {
"renderer_family": "cinematic-trailer",
"cuts": [
{
"id": "v1",
"source": str(source),
"in_seconds": 0,
"out_seconds": 2,
}
],
},
"output_path": str(output),
})
assert result.success, result.error
assert "cuts" not in captured["props"]
assert captured["props"]["scenes"][0]["kind"] == "video"
assert captured["staged_exists_during_render"] is True
assert result.data["staged_media_count"] == 1
def test_remotion_timeout_scales_with_scene_count(monkeypatch, tmp_path) -> None:
output = tmp_path / "render.mp4"
captured = {}
def fake_run_command(self, command, **kwargs):
captured["timeout"] = kwargs["timeout"]
output.write_bytes(b"rendered")
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
cuts = [
{
"id": f"title-{index}",
"source": "",
"type": "hero_title",
"text": str(index),
"in_seconds": 0,
"out_seconds": 1,
}
for index in range(50)
]
result = VideoCompose()._remotion_render({
"edit_decisions": {"renderer_family": "cinematic-trailer", "cuts": cuts},
"output_path": str(output),
})
assert result.success, result.error
assert captured["timeout"] == 750
def test_remotion_render_preserves_direct_cinematic_scenes(monkeypatch, tmp_path) -> None:
output = tmp_path / "render.mp4"
captured = {}
def fake_run_command(self, command, **kwargs):
props_arg = next(arg for arg in command if arg.startswith("--props="))
captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text())
output.write_bytes(b"rendered")
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
scene = {
"id": "authored",
"kind": "title",
"text": "Keep me",
"startSeconds": 0,
"durationSeconds": 1,
}
result = VideoCompose()._remotion_render({
"composition_data": {
"renderer_family": "cinematic-trailer",
"scenes": [scene],
},
"output_path": str(output),
})
assert result.success, result.error
assert captured["props"]["scenes"] == [scene]

View File

@ -0,0 +1,59 @@
from dataclasses import dataclass
import pytest
import tools.video.stock_sources as stock_sources
from tools.video.corpus_builder import CorpusBuilder
@dataclass
class _Candidate:
clip_id: str
class _Source:
name = "fake"
def __init__(self, count: int) -> None:
self.count = count
def is_available(self) -> bool:
return True
def search(self, query, filters):
return [_Candidate(f"clip-{index}") for index in range(self.count)]
@pytest.fixture
def run_builder(monkeypatch, tmp_path):
def run(count: int, processor):
monkeypatch.setattr(stock_sources, "available_sources", lambda: [_Source(count)])
monkeypatch.setattr(stock_sources, "source_summary", lambda: {})
monkeypatch.setattr(CorpusBuilder, "_process_candidate", processor)
return CorpusBuilder().execute({
"corpus_dir": str(tmp_path / f"corpus-{count}"),
"queries": [{"query": "city at night"}],
"max_new_clips": 50,
})
return run
def test_all_candidate_failures_fail_closed_with_diagnostics(run_builder) -> None:
def broken_clip_stack(*args, **kwargs):
raise AttributeError("BaseModelOutput has no attribute norm")
result = run_builder(4, broken_clip_stack)
assert result.success is False
assert result.data["candidates_seen"] == 4
assert result.data["clips_failed"] == 4
assert "corpus index is empty" in result.error
assert "BaseModelOutput" in result.error
def test_no_candidates_is_a_valid_empty_search(run_builder) -> None:
result = run_builder(0, lambda *args, **kwargs: None)
assert result.success is True
assert result.data["candidates_seen"] == 0

View File

@ -0,0 +1,97 @@
from pathlib import Path
from types import SimpleNamespace
import pytest
def test_blank_google_location_uses_documented_default(monkeypatch) -> None:
monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "")
from tools.google_credentials import resolve_google_location
assert resolve_google_location() == "us-central1"
def test_google_music_requests_the_global_vertex_location(monkeypatch, tmp_path) -> None:
monkeypatch.setenv("GOOGLE_API_KEY", "test-key")
import tools.google_credentials as credentials
captured = {}
def stop_before_network(http_options=None, location=None):
captured["location"] = location
raise RuntimeError("stop before network")
monkeypatch.setattr(credentials, "get_genai_client", stop_before_network)
from tools.audio.google_music import GoogleMusic
result = GoogleMusic().execute({
"prompt": "solo piano",
"output_path": str(tmp_path / "music.mp3"),
})
assert result.success is False
assert captured["location"] == "global"
class _VideoAsset:
uri = None
def __init__(self, video_bytes):
self.video_bytes = video_bytes
def save(self, path):
Path(path).write_bytes(self.video_bytes or b"")
class _Models:
def __init__(self, video_bytes):
self.video_bytes = video_bytes
def generate_videos(self, **kwargs):
asset = _VideoAsset(self.video_bytes)
generated = SimpleNamespace(video=asset)
response = SimpleNamespace(generated_videos=[generated])
return SimpleNamespace(done=True, error=None, response=response)
class _VertexClient:
vertexai = True
def __init__(self, video_bytes):
self.models = _Models(video_bytes)
self.files = SimpleNamespace(
download=lambda **kwargs: pytest.fail("Vertex must not call files.download")
)
@pytest.mark.parametrize(
("video_bytes", "expected_success"),
[(b"VIDEO_BYTES", True), (None, False)],
)
def test_veo_accepts_vertex_and_requires_inline_bytes(
monkeypatch, tmp_path, video_bytes, expected_success
) -> None:
monkeypatch.setenv("GOOGLE_API_KEY", "test-key")
import tools.google_credentials as credentials
monkeypatch.setattr(
credentials,
"get_genai_client",
lambda http_options=None, location=None: _VertexClient(video_bytes),
)
from tools.video.veo_video import VeoVideo
output = tmp_path / "video.mp4"
result = VeoVideo().execute({
"backend": "google",
"prompt": "wind moving across grassland",
"output_path": str(output),
})
assert result.success is expected_success
if expected_success:
assert output.read_bytes() == b"VIDEO_BYTES"
else:
assert "without inline bytes" in result.error

View File

@ -250,6 +250,11 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch):
"_resolve_npm_package",
classmethod(lambda cls: {"version": "0.4.5"}),
)
monkeypatch.setattr(
HyperFramesCompose,
"_probe_cli",
classmethod(lambda cls: {"status": "ok"}),
)
rc = HyperFramesCompose()._runtime_check()
# Local binaries must still pass for this to go green.
if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]:
@ -259,6 +264,31 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch):
assert rc["reasons"] == []
def test_runtime_check_fails_when_published_cli_crashes(monkeypatch):
monkeypatch.setattr(
HyperFramesCompose,
"_resolve_npm_package",
classmethod(lambda cls: {"version": "0.7.89"}),
)
monkeypatch.setattr(
HyperFramesCompose,
"_probe_cli",
classmethod(
lambda cls: {
"error": 'doctor failed: The "file" argument must be of type string'
}
),
)
rc = HyperFramesCompose()._runtime_check()
if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]:
pytest.skip("Local runtime floor not met on this machine")
assert rc["runtime_available"] is False
assert rc["cli_probe_error"] is not None
assert any("not executable" in reason for reason in rc["reasons"])
def test_video_compose_render_engines_follow_hyperframes_runtime_check(monkeypatch):
"""Regression: `video_compose.get_info()['render_engines']['hyperframes']`
must track the true availability, not just the local-binary floor.

View File

@ -213,8 +213,9 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch):
# Build a fake RealESRGANer whose __init__ DOES accept device=
class FakeRealESRGANer:
def __init__(self, *, scale, model_path, model, dni_weight, half, device=None):
def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10, device=None):
self.called_with_device = device
self.called_with_tile = tile
fake_realesrganer_cls = FakeRealESRGANer
monkeypatch.setitem(sys.modules, "torch", fake_torch)
@ -239,6 +240,7 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch):
tool = upscale.Upscale()
result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False)
assert result.called_with_device == "device(mps)"
assert result.called_with_tile == 256
def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
@ -252,8 +254,9 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
# Build a fake RealESRGANer whose __init__ does NOT accept device=
class FakeRealESRGANerNoDevice:
def __init__(self, *, scale, model_path, model, dni_weight, half):
def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10):
self.called_with_device = None # no device param
self.called_with_tile = tile
fake_realesrganer_cls = FakeRealESRGANerNoDevice
monkeypatch.setitem(sys.modules, "torch", fake_torch)
@ -279,6 +282,7 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
# Should NOT raise TypeError about unexpected keyword argument 'device'
result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False)
assert result.called_with_device is None
assert result.called_with_tile == 256
# ------------------------------------------------------------------

View File

@ -0,0 +1,59 @@
import shutil
import subprocess
from pathlib import Path
import pytest
from tools.video.video_compose import VideoCompose
pytestmark = pytest.mark.skipif(
shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
reason="ffmpeg and ffprobe are required",
)
def _run(command: list[str]) -> None:
subprocess.run(command, capture_output=True, check=True, timeout=30)
def test_external_audio_mux_adds_audible_stream_without_changing_video_length(tmp_path) -> None:
video = tmp_path / "video.mp4"
audio = tmp_path / "audio.wav"
_run([
"ffmpeg", "-y", "-f", "lavfi", "-i",
"color=c=red:s=320x180:d=2:r=30",
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(video),
])
_run([
"ffmpeg", "-y", "-f", "lavfi", "-i",
"sine=frequency=440:duration=1", str(audio),
])
result = VideoCompose()._mux_external_audio(video, audio)
assert result.success, result.error
streams = subprocess.run(
[
"ffprobe", "-v", "error", "-show_entries",
"stream=codec_type", "-of", "csv=p=0", str(video),
],
capture_output=True,
check=True,
text=True,
timeout=30,
).stdout.splitlines()
duration = float(subprocess.run(
[
"ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "csv=p=0", str(video),
],
capture_output=True,
check=True,
text=True,
timeout=30,
).stdout.strip())
assert "video" in streams
assert "audio" in streams
assert duration == pytest.approx(2.0, abs=0.15)

View File

@ -0,0 +1,83 @@
import sys
from types import SimpleNamespace
from tools.analysis.transcriber import Transcriber
class _Info:
language = "en"
duration = 1.0
def test_transcriber_uses_ctranslate2_cuda_without_torch(monkeypatch, tmp_path) -> None:
devices = []
class FakeWhisperModel:
def __init__(self, model_size, *, device, compute_type):
devices.append((device, compute_type))
def transcribe(self, *args, **kwargs):
return iter(()), _Info()
monkeypatch.setitem(
sys.modules,
"faster_whisper",
SimpleNamespace(WhisperModel=FakeWhisperModel),
)
monkeypatch.setitem(
sys.modules,
"ctranslate2",
SimpleNamespace(
get_cuda_device_count=lambda: 1,
get_supported_compute_types=lambda device: {"float16", "float32"},
),
)
input_path = tmp_path / "audio.wav"
input_path.write_bytes(b"fake")
result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)})
assert result.success, result.error
assert devices == [("cuda", "float16")]
assert result.data["device"] == "cuda"
def test_transcriber_falls_back_when_cuda_fails_during_iteration(monkeypatch, tmp_path) -> None:
devices = []
class FakeWhisperModel:
def __init__(self, model_size, *, device, compute_type):
self.device = device
devices.append((device, compute_type))
def transcribe(self, *args, **kwargs):
if self.device == "cuda":
def broken_iterator():
raise RuntimeError("cublas64_12.dll not found")
yield
return broken_iterator(), _Info()
return iter(()), _Info()
monkeypatch.setitem(
sys.modules,
"faster_whisper",
SimpleNamespace(WhisperModel=FakeWhisperModel),
)
monkeypatch.setitem(
sys.modules,
"ctranslate2",
SimpleNamespace(
get_cuda_device_count=lambda: 1,
get_supported_compute_types=lambda device: {"float16"},
),
)
input_path = tmp_path / "audio.wav"
input_path.write_bytes(b"fake")
result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)})
assert result.success, result.error
assert devices == [("cuda", "float16"), ("cpu", "int8")]
assert result.data["device"] == "cpu"
assert "cublas64_12.dll" in result.data["gpu_fallback_reason"]

View File

@ -135,50 +135,76 @@ class Transcriber(BaseTool):
start = time.time()
# Load model (CPU by default, CUDA if available)
# faster-whisper executes through CTranslate2, so that runtime—not
# PyTorch—is authoritative for CUDA availability and compute types.
device = "cpu"
compute_type = "int8"
try:
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
compute_type = "float16" if device == "cuda" else "int8"
except ImportError:
import ctranslate2
if ctranslate2.get_cuda_device_count() > 0:
supported = ctranslate2.get_supported_compute_types("cuda")
for candidate in ("float16", "int8_float16", "float32"):
if candidate in supported:
device = "cuda"
compute_type = candidate
break
except Exception:
# Probing is advisory. CPU remains a safe deterministic baseline.
pass
def _transcribe_on(selected_device: str, selected_compute_type: str):
model = WhisperModel(
model_size,
device=selected_device,
compute_type=selected_compute_type,
)
segments_iter, transcription_info = model.transcribe(
str(input_path),
language=language,
word_timestamps=True,
vad_filter=True,
)
parsed_segments = []
parsed_words = []
# faster-whisper evaluates lazily. Draining the iterator here keeps
# missing CUDA runtime libraries inside the fallback boundary.
for seg in segments_iter:
seg_data = {
"id": seg.id,
"start": round(seg.start, 3),
"end": round(seg.end, 3),
"text": seg.text.strip(),
}
if seg.words:
words = []
for word in seg.words:
word_entry = {
"word": word.word,
"start": round(word.start, 3),
"end": round(word.end, 3),
"probability": round(word.probability, 3),
}
words.append(word_entry)
parsed_words.append(word_entry)
seg_data["words"] = words
parsed_segments.append(seg_data)
return parsed_segments, parsed_words, transcription_info
gpu_fallback_reason = None
try:
segments, word_timestamps, info = _transcribe_on(device, compute_type)
except Exception as exc:
if device == "cpu":
raise
gpu_fallback_reason = f"{type(exc).__name__}: {exc}"
device = "cpu"
compute_type = "int8"
model = WhisperModel(model_size, device=device, compute_type=compute_type)
# Transcribe
segments_iter, info = model.transcribe(
str(input_path),
language=language,
word_timestamps=True,
vad_filter=True,
)
segments = []
word_timestamps = []
for seg in segments_iter:
seg_data = {
"id": seg.id,
"start": round(seg.start, 3),
"end": round(seg.end, 3),
"text": seg.text.strip(),
}
if seg.words:
words = []
for w in seg.words:
word_entry = {
"word": w.word,
"start": round(w.start, 3),
"end": round(w.end, 3),
"probability": round(w.probability, 3),
}
words.append(word_entry)
word_timestamps.append(word_entry)
seg_data["words"] = words
segments.append(seg_data)
segments, word_timestamps, info = _transcribe_on(device, compute_type)
detected_language = language or info.language
duration = info.duration
@ -198,6 +224,8 @@ class Transcriber(BaseTool):
"duration_seconds": round(duration, 3),
"model_size": model_size,
"device": device,
"compute_type": compute_type,
"gpu_fallback_reason": gpu_fallback_reason,
}
# Write transcript JSON

View File

@ -184,6 +184,14 @@ class AudioMixer(BaseTool):
"default": 0.5,
"description": "Duration of fade in/out at segment boundaries (seconds).",
},
"target_duration": {
"type": "number",
"exclusiveMinimum": 0,
"description": (
"full_mix only. Exact output length in seconds. Pads a short "
"mix and trims a long mix so audio matches the composition."
),
},
},
}
@ -500,6 +508,15 @@ class AudioMixer(BaseTool):
output_path.parent.mkdir(parents=True, exist_ok=True)
normalize = inputs.get("normalize", True)
ducking = inputs.get("ducking", {"enabled": True})
target_duration = inputs.get("target_duration")
target: float | None = None
if target_duration is not None:
try:
target = float(target_duration)
except (TypeError, ValueError):
return ToolResult(success=False, error="target_duration must be a positive number")
if target <= 0:
return ToolResult(success=False, error="target_duration must be greater than zero")
speech_tracks = [t for t in tracks if t.get("role") in ("speech", "primary")]
music_tracks = [t for t in tracks if t.get("role") in ("music", "secondary")]
@ -547,7 +564,14 @@ class AudioMixer(BaseTool):
)
else:
filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_all]")
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
if target is not None:
filter_parts.append("[speech_all]asplit=2[speech_key_raw][speech_out]")
filter_parts.append(
f"[speech_key_raw]apad=whole_dur={target},"
f"atrim=duration={target},asetpts=PTS-STARTPTS[speech_key]"
)
else:
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
# Mix music tracks together
music_start = len(speech_tracks)
@ -596,19 +620,34 @@ class AudioMixer(BaseTool):
f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]"
)
# A ducked music stream is gated by the speech sidechain, so its tail
# can disappear when narration ends. If the caller knows the video
# duration, make that the authoritative mix length before loudness
# normalization: apad extends short audio and atrim caps long audio.
premix_label = "premix"
if target is not None:
filter_parts.append(
f"[premix]apad=whole_dur={target},atrim=duration={target},"
"asetpts=PTS-STARTPTS[premix_duration]"
)
premix_label = "premix_duration"
# Normalize
if normalize:
filter_parts.append(self._loudnorm_filter(inputs, "premix", "out"))
filter_parts.append(self._loudnorm_filter(inputs, premix_label, "out"))
out_label = "[out]"
else:
out_label = "[premix]"
out_label = f"[{premix_label}]"
filter_complex = ";".join(p for p in filter_parts if p)
cmd = ["ffmpeg", "-y"]
cmd.extend(input_args)
cmd.extend(["-filter_complex", filter_complex])
cmd.extend(["-map", out_label, str(output_path)])
cmd.extend(["-map", out_label])
if target is not None:
cmd.extend(["-t", str(target)])
cmd.append(str(output_path))
self.run_command(cmd)
@ -621,6 +660,7 @@ class AudioMixer(BaseTool):
"sfx_tracks": len(sfx_tracks),
"ducking_enabled": duck_enabled,
"normalized": normalize,
"target_duration": target_duration,
"output": str(output_path),
},
artifacts=[str(output_path)],

View File

@ -146,7 +146,8 @@ class GoogleMusic(BaseTool):
from tools.google_credentials import get_genai_client, GOOGLE_API_TIMEOUT_MS
http_options = types.HttpOptions(timeout=GOOGLE_API_TIMEOUT_MS)
client = get_genai_client(http_options=http_options)
# Lyria 3 is served only from Vertex's global location.
client = get_genai_client(http_options=http_options, location="global")
except ImportError as e:
return ToolResult(
success=False,

View File

@ -130,7 +130,7 @@ class BgRemove(BaseTool):
result_image = rembg.remove(
input_image,
model_name=model_name,
session=rembg.new_session(model_name),
alpha_matting=alpha_matting,
)

View File

@ -297,6 +297,11 @@ class Upscale(BaseTool):
"model": model,
"dni_weight": denoise_strength,
"half": half,
# Full-frame x4 inference can terminate the process on low-memory
# CPU/MPS hosts before Python can raise an exception. Bound the
# working set there; keep CUDA on the faster single-pass path.
"tile": 0 if _device == "cuda" else 256,
"tile_pad": 10,
}
# Guard: only pass device= if the installed version accepts it
if "device" in inspect.signature(RealESRGANer.__init__).parameters:

View File

@ -18,6 +18,12 @@ from typing import Any
# Broad scope that covers Cloud Text-to-Speech and Vertex AI prediction.
CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"
def resolve_google_location(location: str | None = None) -> str:
"""Return a Vertex location, treating blank env values as unset."""
return location or os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
# Shared constants for long-running Google/Vertex AI generation calls (e.g. music, video)
GOOGLE_API_TIMEOUT_SECONDS = 600
GOOGLE_API_TIMEOUT_MS = GOOGLE_API_TIMEOUT_SECONDS * 1000
@ -38,8 +44,15 @@ def has_google_credentials() -> bool:
)
def get_genai_client(http_options: Any | None = None) -> Any:
"""Lazily import and initialize the Google GenAI Client based on configured credentials."""
def get_genai_client(
http_options: Any | None = None,
location: str | None = None,
) -> Any:
"""Initialize Google GenAI using the configured credential mode.
``location`` overrides the Vertex region for globally hosted models. It is
deliberately ignored by the API-key backend, which has no region setting.
"""
from google import genai
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")
@ -51,7 +64,7 @@ def get_genai_client(http_options: Any | None = None) -> Any:
if use_vertex or (not api_key and service_account_configured()):
kwargs = {
"vertexai": True,
"location": os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1"),
"location": resolve_google_location(location),
"http_options": http_options,
}
project_id = resolve_project_id()

View File

@ -22,6 +22,7 @@ from tools.base_tool import (
)
from tools.google_credentials import (
get_access_token,
resolve_google_location,
resolve_project_id,
service_account_configured,
has_google_credentials,
@ -241,7 +242,7 @@ class GoogleImagen(BaseTool):
}
if bearer_token:
location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1")
location = resolve_google_location()
url = (
f"https://{location}-aiplatform.googleapis.com/v1/projects/"
f"{project_id}/locations/{location}/publishers/google/models/"

View File

@ -392,6 +392,38 @@ class CorpusBuilder(BaseTool):
except Exception as e:
cache_snapshot = {"error": f"{type(e).__name__}: {e}"}
# Per-candidate tolerance is useful only while at least one item
# survives. If every discovered candidate fails, reporting success
# persists an empty index and hides a systemic codec/CLIP failure
# until retrieval. A no-result or skip-only run remains valid.
total_failure = bool(candidates_seen) and not added_ids and not skipped
if total_failure:
first_errors = "; ".join(
item["error"]
for item in errors
if item.get("phase") == "process"
)[:400]
return ToolResult(
success=False,
error=(
f"All {failed} of {candidates_seen} candidates failed to "
"process; corpus index is empty. Check the media decoder "
"and the CLIP `transformers`/`torch` compatibility. "
f"First errors: {first_errors or '(none recorded)'}"
),
data={
"corpus_dir": str(corpus_dir),
"queries_run": len(queries),
"candidates_seen": candidates_seen,
"clips_added": 0,
"clips_skipped_existing": skipped,
"clips_failed": failed,
"total_corpus_size": len(corp),
"errors": errors[:25],
},
duration_seconds=round(elapsed, 2),
)
return ToolResult(
success=True,
data={

View File

@ -226,6 +226,7 @@ class HyperFramesCompose(BaseTool):
# We cache per-process so the first call pays ~2-5s and subsequent calls
# (get_info spam from the registry) are free.
_npm_resolve_cache: Optional[dict[str, str]] = None
_cli_probe_cache: Optional[dict[str, str]] = None
@classmethod
def _node_major_version(cls) -> Optional[int]:
@ -301,6 +302,45 @@ class HyperFramesCompose(BaseTool):
cls._npm_resolve_cache = {"version": version}
return cls._npm_resolve_cache
@classmethod
def _probe_cli(cls) -> dict[str, str]:
"""Run the published CLI's doctor command once per process.
Package resolution alone does not prove that the executable can start:
an upstream packaging regression can publish successfully while every
CLI command crashes during bootstrap. Provider preflight must not call
that state available.
"""
if cls._cli_probe_cache is not None:
return cls._cli_probe_cache
npx = shutil.which("npx")
if not npx:
cls._cli_probe_cache = {"error": "npx not on PATH"}
return cls._cli_probe_cache
try:
proc = subprocess.run(
[npx, "--yes", cls._NPM_PACKAGE, "doctor", "--json"],
capture_output=True,
text=True,
timeout=20,
)
except subprocess.TimeoutExpired:
cls._cli_probe_cache = {"error": "doctor timed out after 20s"}
return cls._cli_probe_cache
except (OSError, subprocess.SubprocessError) as exc:
cls._cli_probe_cache = {"error": f"doctor failed: {type(exc).__name__}"}
return cls._cli_probe_cache
if proc.returncode != 0:
output = "\n".join(filter(None, [proc.stderr, proc.stdout])).strip()
tail = output.splitlines()[-1][:200] if output else f"exit {proc.returncode}"
cls._cli_probe_cache = {"error": f"doctor failed: {tail}"}
else:
cls._cli_probe_cache = {"status": "ok"}
return cls._cli_probe_cache
def _runtime_check(self) -> dict[str, Any]:
"""Return availability state for the HyperFrames runtime.
@ -336,6 +376,12 @@ class HyperFramesCompose(BaseTool):
f"{npm_resolve['error']}"
)
cli_probe: dict[str, str] = {}
if not reasons:
cli_probe = self._probe_cli()
if "error" in cli_probe:
reasons.append(f"published CLI is not executable: {cli_probe['error']}")
return {
"runtime_available": not reasons,
"node_major": node_major,
@ -344,6 +390,8 @@ class HyperFramesCompose(BaseTool):
"npm_package": self._NPM_PACKAGE,
"npm_package_version": npm_resolve.get("version"),
"npm_resolve_error": npm_resolve.get("error"),
"cli_probe_status": cli_probe.get("status"),
"cli_probe_error": cli_probe.get("error"),
"reasons": reasons,
}

View File

@ -308,13 +308,6 @@ class VeoVideo(BaseTool):
client._api_client, "vertexai", False
)
if is_vertex:
return ToolResult(
success=False,
error="Google Veo video generation via google-genai is only supported using the Gemini Developer API (API key) backend. "
"Please configure GEMINI_API_KEY/GOOGLE_API_KEY or use the FAL.ai backend.",
)
prompt = inputs["prompt"]
operation = inputs.get("operation", "text_to_video")
model_variant = inputs.get("model_variant", "veo3.1")
@ -501,7 +494,19 @@ class VeoVideo(BaseTool):
success=False,
error="No video asset returned in the response.",
)
client.files.download(file=video_asset)
if not is_vertex:
# The Files service is a Gemini Developer API feature. Vertex
# returns bytes inline when no GCS output URI is requested.
client.files.download(file=video_asset)
elif not getattr(video_asset, "video_bytes", None):
return ToolResult(
success=False,
error=(
"Vertex AI returned a video without inline bytes "
f"(uri={getattr(video_asset, 'uri', None)!r}). Configure "
"the request without an output GCS URI so bytes are returned inline."
),
)
output_path = Path(inputs.get("output_path", "veo_output.mp4"))
output_path.parent.mkdir(parents=True, exist_ok=True)

View File

@ -31,11 +31,14 @@ the agent to re-ask the user rather than substituting a different engine.
from __future__ import annotations
import json
import hashlib
import logging
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any, Optional
from urllib.parse import unquote, urlsplit
from tools.base_tool import (
BaseTool,
@ -387,6 +390,49 @@ class VideoCompose(BaseTool):
except Exception:
return False
def _mux_external_audio(self, video_path: Path, audio_path: str | Path) -> ToolResult:
"""Atomically replace a rendered video's audio with the approved mix."""
audio = Path(audio_path).resolve()
if not audio.is_file():
return ToolResult(success=False, error=f"Mixed audio not found: {audio}")
temp_output = video_path.with_name(
f".{video_path.stem}.audio-mux-{time.time_ns()}{video_path.suffix}"
)
try:
self.run_command([
"ffmpeg", "-y",
"-i", str(video_path),
"-i", str(audio),
"-map", "0:v:0",
"-map", "1:a:0",
"-c:v", "copy",
"-c:a", "aac",
"-b:a", "192k",
"-af", "apad",
"-shortest",
"-movflags", "+faststart",
str(temp_output),
])
if not temp_output.is_file():
return ToolResult(
success=False,
error=f"Audio mux completed but output file is missing: {temp_output}",
)
temp_output.replace(video_path)
except Exception as exc:
return ToolResult(success=False, error=f"Could not mux mixed audio: {exc}")
finally:
if temp_output.exists():
temp_output.unlink()
return ToolResult(
success=True,
data={"output": str(video_path), "has_mixed_audio": True},
artifacts=[str(video_path)],
)
def _compose(self, inputs: dict[str, Any]) -> ToolResult:
"""FFmpeg composition: concat video cuts, add audio, burn subtitles.
@ -714,6 +760,124 @@ class VideoCompose(BaseTool):
)
return comp
@staticmethod
def _cuts_to_cinematic_scenes(cuts: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Adapt canonical sequential cuts to CinematicRenderer's scene contract."""
scenes: list[dict[str, Any]] = []
timeline_cursor = 0.0
hard_transitions = {"cut", "none"}
title_types = {"hero_title", "text_card", "title"}
for index, cut in enumerate(cuts):
try:
source_in = float(cut.get("in_seconds", 0))
source_out = float(cut.get("out_seconds", source_in))
speed = max(float(cut.get("speed", 1.0)), 0.1)
except (TypeError, ValueError):
continue
duration = max(0.0, (source_out - source_in) / speed)
if duration <= 0:
continue
scene_id = str(cut.get("id") or f"cut-{index + 1}")
source = str(cut.get("source") or "")
cut_type = str(cut.get("type") or "").lower()
common = {
"id": scene_id,
"startSeconds": timeline_cursor,
"durationSeconds": duration,
}
if cut_type in title_types or not source:
scene: dict[str, Any] = {
**common,
"kind": "title",
"text": str(
cut.get("text")
or cut.get("title")
or cut.get("reason")
or scene_id
),
}
if source:
scene["backgroundSrc"] = source
scene["backgroundTrimBeforeSeconds"] = source_in
scene["backgroundTrimAfterSeconds"] = source_out
else:
scene = {
**common,
"kind": "video",
"src": source,
"trimBeforeSeconds": source_in,
"trimAfterSeconds": source_out,
"playbackRate": speed,
}
if str(cut.get("transition_in") or "").lower() in hard_transitions:
scene["fadeInFrames"] = 0
if str(cut.get("transition_out") or "").lower() in hard_transitions:
scene["fadeOutFrames"] = 0
scenes.append(scene)
timeline_cursor += duration
return scenes
@staticmethod
def _stage_remotion_media(value: Any, public_dir: Path) -> int:
"""Copy local media references into a Remotion public dir in-place.
OffthreadVideo's compositor rejects ``file://`` sources. Rewriting
staged files to relative ``staticFile()`` paths works for video and
image components on every platform.
"""
staged_by_source: dict[Path, str] = {}
media_keys = {"source", "src", "backgroundSrc"}
def visit(node: Any, parent_key: str | None = None) -> Any:
if isinstance(node, dict):
for key, child in list(node.items()):
node[key] = visit(child, key)
return node
if isinstance(node, list):
for index, child in enumerate(node):
node[index] = visit(child, parent_key)
return node
if not isinstance(node, str) or parent_key not in media_keys:
return node
if node.startswith(("http://", "https://", "data:")):
return node
if node.lower().startswith("file://"):
parsed = urlsplit(node)
decoded_path = unquote(parsed.path)
if len(parsed.netloc) == 2 and parsed.netloc[1] == ":":
raw_path = f"{parsed.netloc}{decoded_path}"
elif parsed.netloc and parsed.netloc.lower() != "localhost":
raw_path = f"//{parsed.netloc}{decoded_path}"
else:
raw_path = decoded_path
# Standard Windows file URIs use file:///C:/...; pathlib on
# Windows needs the drive path without the URI's leading slash.
if len(raw_path) >= 3 and raw_path[0] == "/" and raw_path[2] == ":":
raw_path = raw_path[1:]
else:
raw_path = node
source = Path(raw_path).resolve()
if not source.is_file():
return node
if source not in staged_by_source:
digest = hashlib.sha256(str(source).encode("utf-8")).hexdigest()[:12]
name = f"{digest}-{source.name}"
public_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, public_dir / name)
staged_by_source[source] = name
return staged_by_source[source]
visit(value)
return len(staged_by_source)
def _render_via_atelier(
self,
inputs: dict[str, Any],
@ -840,6 +1004,11 @@ class VideoCompose(BaseTool):
error=f"Atelier render completed but output file missing: {output_path}",
)
if inputs.get("audio_path"):
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
if not mux_result.success:
return mux_result
# --- Atelier post-render review -------------------------------------
# The cut-schema paths run _run_final_review (technical/visual/audio
# probes + transcript-vs-script). Atelier MUST do the same so hero
@ -1063,8 +1232,12 @@ class VideoCompose(BaseTool):
try:
from styles.playbook_loader import load_playbook
playbook = load_playbook(playbook_name)
except Exception:
pass
except Exception as exc:
logging.getLogger(__name__).warning(
"Could not load style playbook %r for Remotion theme: %s",
playbook_name,
exc,
)
if playbook:
vl = playbook.get("visual_language", {})
@ -1418,6 +1591,8 @@ class VideoCompose(BaseTool):
# would only take effect on a direct _remotion_render() call.
if inputs.get("remotion_timeout_ms") is not None:
remotion_inputs["remotion_timeout_ms"] = inputs["remotion_timeout_ms"]
if inputs.get("public_dir") is not None:
remotion_inputs["public_dir"] = inputs["public_dir"]
render_result = self._remotion_render(remotion_inputs)
# Governance: NEVER silently fall back to FFmpeg when Remotion fails.
@ -1437,6 +1612,11 @@ class VideoCompose(BaseTool):
f"Per governance: renderer downgrade requires user approval."
),
)
if inputs.get("audio_path"):
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
if not mux_result.success:
return mux_result
render_result.data["has_mixed_audio"] = True
else:
# --- FFmpeg fallback: only when Remotion is unavailable ---
options = inputs.get("options", {})
@ -1546,7 +1726,12 @@ class VideoCompose(BaseTool):
try:
from styles.playbook_loader import load_playbook # type: ignore
playbook_data = load_playbook(playbook_name)
except Exception:
except Exception as exc:
logging.getLogger(__name__).warning(
"Could not load style playbook %r for HyperFrames bridge: %s",
playbook_name,
exc,
)
playbook_data = None
hf_inputs: dict[str, Any] = {
@ -1677,8 +1862,6 @@ class VideoCompose(BaseTool):
types, and transitions using React-based frame-accurate rendering.
Accepts edit_decisions (with resolved file paths) or raw composition_data.
"""
import shutil
if not shutil.which("npx"):
return ToolResult(
success=False,
@ -1700,16 +1883,6 @@ class VideoCompose(BaseTool):
# Deep-copy props so we don't mutate the original
props = json.loads(json.dumps(composition_data))
# Convert absolute file paths to file:// URIs for Remotion's
# Img and OffthreadVideo components
for cut in props.get("cuts", []):
source = cut.get("source", "")
if source and not source.startswith(("http://", "https://", "file://")):
resolved = Path(source).resolve()
if resolved.exists():
posix = resolved.as_posix()
cut["source"] = f"file:///{posix}" if not posix.startswith("/") else f"file://{posix}"
# Build a custom themeConfig from the playbook's actual colors.
# This ensures every video gets a unique visual identity derived
# from its production decisions — not picked from a preset menu.
@ -1723,11 +1896,6 @@ class VideoCompose(BaseTool):
if theme_config:
props["themeConfig"] = theme_config
# Write props to temp file for Remotion CLI
props_path = output_path.parent / ".remotion_props.json"
with open(props_path, "w", encoding="utf-8") as f:
json.dump(props, f)
# remotion-composer lives at project root
composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer"
if not composer_dir.exists():
@ -1741,6 +1909,39 @@ class VideoCompose(BaseTool):
renderer_family = (composition_data or {}).get("renderer_family", "explainer-data")
composition_id = self._get_composition_id(renderer_family)
if composition_id == "CinematicRenderer":
if not props.get("scenes") and props.get("cuts"):
props["scenes"] = self._cuts_to_cinematic_scenes(props["cuts"])
props.pop("cuts", None)
if not props.get("scenes"):
return ToolResult(
success=False,
error="CinematicRenderer received cuts but none could be adapted into scenes.",
)
requested_public_dir = inputs.get("public_dir")
cleanup_public_dir = False
public_dir: Path | None = None
if requested_public_dir:
public_dir = Path(requested_public_dir).resolve()
if not public_dir.is_dir():
return ToolResult(
success=False,
error=f"Remotion public_dir does not exist or is not a directory: {public_dir}",
)
else:
public_dir = output_path.parent / f".remotion-public-{output_path.stem}"
cleanup_public_dir = True
staged_count = self._stage_remotion_media(props, public_dir)
if not staged_count and cleanup_public_dir:
public_dir = None
# Write the fully adapted/staged props, never the original cut payload.
props_path = output_path.parent / ".remotion_props.json"
with open(props_path, "w", encoding="utf-8") as f:
json.dump(props, f)
cmd = [
"npx", "remotion", "render",
str(composer_dir / "src" / "index.tsx"),
@ -1753,6 +1954,8 @@ class VideoCompose(BaseTool):
# API Remotion recommends for file paths and is cross-platform safe.
f"--props={props_path}",
]
if public_dir is not None:
cmd.append(f"--public-dir={public_dir}")
# Apply media profile dimensions
profile_name = inputs.get("profile")
@ -1770,7 +1973,8 @@ class VideoCompose(BaseTool):
# opaque failure. Pass it through and give the subprocess enough headroom
# so run_command() does not kill Remotion before its own timeout fires.
remotion_timeout_ms = inputs.get("remotion_timeout_ms")
subprocess_timeout = 600
scene_count = len(props.get("scenes") or props.get("cuts") or [])
subprocess_timeout = max(600, scene_count * 15)
if remotion_timeout_ms:
try:
ms = int(remotion_timeout_ms)
@ -1808,6 +2012,8 @@ class VideoCompose(BaseTool):
finally:
if props_path.exists():
props_path.unlink()
if cleanup_public_dir and public_dir is not None and public_dir.exists():
shutil.rmtree(public_dir, ignore_errors=True)
if not output_path.exists():
return ToolResult(
@ -1821,6 +2027,7 @@ class VideoCompose(BaseTool):
"operation": "remotion_render",
"output": str(output_path),
"profile": profile_name,
"staged_media_count": staged_count,
},
artifacts=[str(output_path)],
)