Merge pull request #466 from calesthio/codex/backlog-recovery-20260802
Recover bounded defects from the latest PR backlog
This commit is contained in:
commit
4eab34c5cf
107
.env.example
107
.env.example
|
|
@ -2,72 +2,103 @@
|
|||
# Copy this to .env and fill in your keys
|
||||
|
||||
# --- Image + video gateway ---
|
||||
FAL_KEY= # FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images
|
||||
# Get one at https://fal.ai/dashboard/keys
|
||||
FAL_AI_API_KEY= # Alias for FAL_KEY (some SDKs/docs use this name); either one is read.
|
||||
# FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images.
|
||||
# Get one at https://fal.ai/dashboard/keys
|
||||
FAL_KEY=
|
||||
# Alias for FAL_KEY (some SDKs/docs use this name); either one is read.
|
||||
FAL_AI_API_KEY=
|
||||
|
||||
# --- Replicate ---
|
||||
REPLICATE_API_TOKEN= # Replicate-hosted video gen (seedance_replicate). Needed to make the
|
||||
# Replicate-backed Seedance path selectable alongside the fal.ai one.
|
||||
# Get one at https://replicate.com/account/api-tokens
|
||||
# Replicate-hosted video gen (seedance_replicate). Needed to make the
|
||||
# Replicate-backed Seedance path selectable alongside the fal.ai one.
|
||||
# Get one at https://replicate.com/account/api-tokens
|
||||
REPLICATE_API_TOKEN=
|
||||
|
||||
# --- Higgsfield ---
|
||||
HIGGSFIELD_API_KEY= # Higgsfield Cloud key (higgsfield_video). Pair with the secret below,
|
||||
HIGGSFIELD_API_SECRET= # or use the combined HIGGSFIELD_KEY="<key>:<secret>" form instead.
|
||||
# Higgsfield Cloud key (higgsfield_video). Pair with the secret below,
|
||||
# or use the combined HIGGSFIELD_KEY="<key>:<secret>" form instead.
|
||||
HIGGSFIELD_API_KEY=
|
||||
HIGGSFIELD_API_SECRET=
|
||||
# HIGGSFIELD_KEY= # Combined key:secret — set this INSTEAD of the _KEY/_SECRET pair if you prefer.
|
||||
|
||||
# --- Kling official direct API ---
|
||||
KLING_API_KEY= # Official Kling API key; enables video, image, TTS, avatar, lip sync
|
||||
KLING_API_BASE_URL= # Optional endpoint override; leave blank for default https://api-singapore.klingai.com
|
||||
# Mainland China accounts can use https://api-beijing.klingai.com
|
||||
# Official Kling API key; enables video, image, TTS, avatar, lip sync.
|
||||
KLING_API_KEY=
|
||||
# Optional endpoint override; leave blank for default https://api-singapore.klingai.com
|
||||
# Mainland China accounts can use https://api-beijing.klingai.com
|
||||
KLING_API_BASE_URL=
|
||||
|
||||
# --- Google (one key unlocks image gen + TTS + video) ---
|
||||
GOOGLE_API_KEY= # Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages),
|
||||
# Gemini Omni video (generation + conversational editing, paid tier)
|
||||
# Get one at https://aistudio.google.com/apikey
|
||||
# Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages),
|
||||
# Gemini Omni video (generation + conversational editing, paid tier).
|
||||
# Get one at https://aistudio.google.com/apikey
|
||||
GOOGLE_API_KEY=
|
||||
# GEMINI_API_KEY= # Alias for GOOGLE_API_KEY (takes precedence when both are set)
|
||||
# Alternative to the API key: service-account JSON auth.
|
||||
# TTS uses Cloud Text-to-Speech; Imagen routes to Vertex AI.
|
||||
GOOGLE_APPLICATION_CREDENTIALS= # path to a service-account JSON key file
|
||||
GOOGLE_CLOUD_PROJECT= # GCP project id (required for Imagen via Vertex AI)
|
||||
GOOGLE_CLOUD_LOCATION= # Vertex AI region, default us-central1
|
||||
# Path to a service-account JSON key file.
|
||||
GOOGLE_APPLICATION_CREDENTIALS=
|
||||
# GCP project id (required for Imagen via Vertex AI).
|
||||
GOOGLE_CLOUD_PROJECT=
|
||||
# Vertex AI region, default us-central1.
|
||||
GOOGLE_CLOUD_LOCATION=
|
||||
|
||||
# --- Voice ---
|
||||
ELEVENLABS_API_KEY= # TTS narration, music generation, sound effects
|
||||
OPENAI_API_KEY= # OpenAI TTS fallback and GPT Image 2 image generation
|
||||
XAI_API_KEY= # Grok image generation/editing and Grok video generation
|
||||
DOUBAO_SPEECH_API_KEY= # Volcengine Doubao Speech TTS (new console API Key)
|
||||
DOUBAO_SPEECH_VOICE_TYPE= # Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts
|
||||
# TTS narration, music generation, sound effects.
|
||||
ELEVENLABS_API_KEY=
|
||||
# OpenAI TTS fallback and GPT Image 2 image generation.
|
||||
OPENAI_API_KEY=
|
||||
# Grok image generation/editing and Grok video generation.
|
||||
XAI_API_KEY=
|
||||
# Volcengine Doubao Speech TTS (new console API Key).
|
||||
DOUBAO_SPEECH_API_KEY=
|
||||
# Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts.
|
||||
DOUBAO_SPEECH_VOICE_TYPE=
|
||||
# Piper local voices do not require env vars; install `piper-tts` via pip
|
||||
|
||||
# --- DashScope (Alibaba Cloud Bailian) ---
|
||||
DASHSCOPE_API_KEY= # Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans)
|
||||
# Get one at https://dashscope.aliyun.com/
|
||||
# Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans).
|
||||
# Get one at https://dashscope.aliyun.com/
|
||||
DASHSCOPE_API_KEY=
|
||||
|
||||
# --- Music ---
|
||||
SUNO_API_KEY= # Suno AI music generation (full songs, instrumentals, any genre)
|
||||
# Suno AI music generation (full songs, instrumentals, any genre).
|
||||
SUNO_API_KEY=
|
||||
|
||||
# --- Video Generation ---
|
||||
HEYGEN_API_KEY= # HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key)
|
||||
RUNWAY_API_KEY= # Runway Gen-4 (direct API, alternative to fal.ai routing)
|
||||
VOLC_ACCESSKEY= # Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing)
|
||||
VOLC_SECRETKEY= # Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage
|
||||
VIDEO_GEN_LOCAL_ENABLED= # Set to "true" for local video gen (needs GPU + diffusers)
|
||||
VIDEO_GEN_LOCAL_MODEL= # Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b
|
||||
MODAL_LTX2_ENDPOINT_URL= # Modal self-hosted LTX-2 endpoint (optional)
|
||||
# HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key).
|
||||
HEYGEN_API_KEY=
|
||||
# Runway Gen-4 (direct API, alternative to fal.ai routing).
|
||||
RUNWAY_API_KEY=
|
||||
# Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing).
|
||||
VOLC_ACCESSKEY=
|
||||
# Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage
|
||||
VOLC_SECRETKEY=
|
||||
# Set to "true" for local video gen (needs GPU + diffusers).
|
||||
VIDEO_GEN_LOCAL_ENABLED=
|
||||
# Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b.
|
||||
VIDEO_GEN_LOCAL_MODEL=
|
||||
# Modal self-hosted LTX-2 endpoint (optional).
|
||||
MODAL_LTX2_ENDPOINT_URL=
|
||||
|
||||
# --- Stock Media ---
|
||||
PEXELS_API_KEY= # Pexels stock footage/images (free)
|
||||
PIXABAY_API_KEY= # Pixabay stock footage/images (free)
|
||||
UNSPLASH_ACCESS_KEY= # Unsplash stock images (free developer key)
|
||||
# Pexels stock footage/images (free).
|
||||
PEXELS_API_KEY=
|
||||
# Pixabay stock footage/images (free).
|
||||
PIXABAY_API_KEY=
|
||||
# Unsplash stock images (free developer key).
|
||||
UNSPLASH_ACCESS_KEY=
|
||||
|
||||
# --- Analysis ---
|
||||
HF_TOKEN= # HuggingFace token — enables speaker diarization in transcriber
|
||||
# HuggingFace token — enables speaker diarization in transcriber.
|
||||
HF_TOKEN=
|
||||
# Speech-to-text: optional Azure AI Speech (Fast Transcription). When set, the
|
||||
# agent prefers azure_stt for cloud STT; the local faster-whisper transcriber
|
||||
# remains the default offline path.
|
||||
AZURE_SPEECH_KEY= # Azure AI Speech resource key ('Keys and Endpoint' page)
|
||||
AZURE_SPEECH_REGION= # Speech resource region, e.g. eastus
|
||||
# Azure AI Speech resource key ('Keys and Endpoint' page).
|
||||
AZURE_SPEECH_KEY=
|
||||
# Speech resource region, e.g. eastus.
|
||||
AZURE_SPEECH_REGION=
|
||||
# AZURE_SPEECH_ENDPOINT= # Optional: full custom endpoint URL (overrides region)
|
||||
|
||||
# --- Avatar (local installs) ---
|
||||
|
|
|
|||
|
|
@ -528,9 +528,10 @@ Music is a critical part of any video. **Surface the music situation to the user
|
|||
|
||||
Check music availability in this order and present the options:
|
||||
|
||||
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one.
|
||||
2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly — include quota status if known.
|
||||
3. **Royalty-free sources:** Note if the user can provide their own track (e.g., from YouTube Audio Library, Jamendo, or other free sources). Offer the `music_library/` drop path.
|
||||
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. If tracks exist, list durations and let the user pick one.
|
||||
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")` for configured search/download tools. Report licensing constraints and whether a key is required.
|
||||
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly.
|
||||
4. **Bring your own:** Note that the user can provide a track (for example from YouTube Audio Library or Jamendo) through the `music_library/` drop path.
|
||||
|
||||
**Always present the user with explicit choices:**
|
||||
- Use a track from their library (which one?)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -147,18 +148,22 @@ async def _watch_projects() -> None:
|
|||
hub.publish(pid)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: FastAPI):
|
||||
"""Own and cleanly stop the project watcher with FastAPI's lifespan API."""
|
||||
|
||||
task = asyncio.create_task(_watch_projects())
|
||||
app.state.watch_task = task
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(title="Backlot", docs_url=None, redoc_url=None)
|
||||
|
||||
@app.on_event("startup")
|
||||
async def _startup() -> None:
|
||||
app.state.watch_task = asyncio.create_task(_watch_projects())
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def _shutdown() -> None:
|
||||
task = getattr(app.state, "watch_task", None)
|
||||
if task:
|
||||
task.cancel()
|
||||
app = FastAPI(title="Backlot", docs_url=None, redoc_url=None, lifespan=_lifespan)
|
||||
|
||||
# ---- API ----------------------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
|
|
@ -59,6 +60,7 @@ def _rel(project_dir: Path, path: Path) -> str:
|
|||
# Pipeline / stages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]:
|
||||
"""Stage order + gate flags from the manifest; graceful fallback."""
|
||||
if pipeline_type and pipeline_type != "unknown":
|
||||
|
|
|
|||
|
|
@ -95,9 +95,29 @@ class CheckpointValidationError(ValueError):
|
|||
"""Raised when a checkpoint or its canonical artifacts are invalid."""
|
||||
|
||||
|
||||
def _validate_style_playbook(style_playbook: str | None) -> None:
|
||||
"""Fail closed when a checkpoint names a visual identity that cannot load."""
|
||||
|
||||
if style_playbook is None:
|
||||
return
|
||||
try:
|
||||
from styles.playbook_loader import list_playbooks, load_playbook
|
||||
|
||||
load_playbook(style_playbook)
|
||||
except Exception as exc:
|
||||
try:
|
||||
available = list_playbooks()
|
||||
except Exception:
|
||||
available = []
|
||||
raise CheckpointValidationError(
|
||||
f"Unknown or invalid style_playbook {style_playbook!r}. "
|
||||
f"Available playbooks: {available}. Underlying error: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_checkpoint_schema() -> dict[str, Any]:
|
||||
with open(CHECKPOINT_SCHEMA_PATH) as f:
|
||||
with open(CHECKPOINT_SCHEMA_PATH, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
|
|
@ -192,6 +212,7 @@ def init_project(
|
|||
Idempotent: re-running preserves the original created_at and merges fields.
|
||||
Returns the project directory.
|
||||
"""
|
||||
_validate_style_playbook(style_playbook)
|
||||
base = pipeline_dir or PROJECTS_DIR
|
||||
project_dir = base / project_id
|
||||
for sub in (
|
||||
|
|
@ -208,7 +229,7 @@ def init_project(
|
|||
marker: dict[str, Any] = {}
|
||||
if marker_path.exists():
|
||||
try:
|
||||
with open(marker_path) as f:
|
||||
with open(marker_path, encoding="utf-8") as f:
|
||||
marker = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
marker = {}
|
||||
|
|
@ -221,7 +242,7 @@ def init_project(
|
|||
if style_playbook is not None:
|
||||
marker["style_playbook"] = style_playbook
|
||||
|
||||
with open(marker_path, "w") as f:
|
||||
with open(marker_path, "w", encoding="utf-8") as f:
|
||||
json.dump(marker, f, indent=2)
|
||||
|
||||
return project_dir
|
||||
|
|
@ -260,6 +281,71 @@ def _stage_requires_approval(pipeline_type: Optional[str], stage: str) -> Option
|
|||
return get_stage_human_approval_default(manifest, stage)
|
||||
|
||||
|
||||
def _enforce_stage_prerequisites(
|
||||
pipeline_dir: Path,
|
||||
project_id: str,
|
||||
pipeline_type: str | None,
|
||||
stage: str,
|
||||
status: str,
|
||||
) -> None:
|
||||
"""Require completed, approved predecessors before advancing a stage.
|
||||
|
||||
``in_progress`` and failure heartbeats remain writable so an operator can
|
||||
inspect or resume a broken run. Only lifecycle advancement
|
||||
(``awaiting_human``/``completed``) is gated.
|
||||
"""
|
||||
|
||||
if status not in {"awaiting_human", "completed"}:
|
||||
return
|
||||
if not pipeline_type or pipeline_type == "unknown":
|
||||
return
|
||||
|
||||
stages = get_pipeline_stages(pipeline_type)
|
||||
if stage not in stages:
|
||||
return
|
||||
|
||||
incomplete: list[str] = []
|
||||
unapproved: list[str] = []
|
||||
for predecessor in stages[: stages.index(stage)]:
|
||||
path = _checkpoint_path(pipeline_dir, project_id, predecessor)
|
||||
if not path.exists():
|
||||
incomplete.append(predecessor)
|
||||
continue
|
||||
try:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
checkpoint = json.load(handle)
|
||||
validate_checkpoint(checkpoint)
|
||||
except (OSError, json.JSONDecodeError, CheckpointValidationError):
|
||||
incomplete.append(predecessor)
|
||||
continue
|
||||
if (
|
||||
checkpoint.get("project_id") != project_id
|
||||
or checkpoint.get("pipeline_type") != pipeline_type
|
||||
or checkpoint.get("stage") != predecessor
|
||||
):
|
||||
incomplete.append(predecessor)
|
||||
continue
|
||||
if checkpoint.get("status") != "completed":
|
||||
incomplete.append(predecessor)
|
||||
continue
|
||||
if _stage_requires_approval(pipeline_type, predecessor) and not checkpoint.get(
|
||||
"human_approved"
|
||||
):
|
||||
unapproved.append(predecessor)
|
||||
|
||||
if incomplete or unapproved:
|
||||
details = []
|
||||
if incomplete:
|
||||
details.append(f"incomplete or missing: {incomplete}")
|
||||
if unapproved:
|
||||
details.append(f"completed without required approval: {unapproved}")
|
||||
raise CheckpointValidationError(
|
||||
f"PREREQUISITE VIOLATION: stage {stage!r} cannot advance; "
|
||||
+ "; ".join(details)
|
||||
+ f". Pipeline order: {stages}."
|
||||
)
|
||||
|
||||
|
||||
def _archive_superseded_checkpoint(path: Path, stage: str) -> None:
|
||||
"""Copy an existing checkpoint into history/ before it is overwritten.
|
||||
|
||||
|
|
@ -275,7 +361,7 @@ def _archive_superseded_checkpoint(path: Path, stage: str) -> None:
|
|||
if not path.exists():
|
||||
return
|
||||
try:
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
existing = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
existing = {}
|
||||
|
|
@ -314,7 +400,7 @@ def _merge_decision_log(
|
|||
"""
|
||||
path = _decision_log_path(pipeline_dir, project_id)
|
||||
if path.exists():
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
existing = json.load(f)
|
||||
else:
|
||||
existing = {
|
||||
|
|
@ -329,7 +415,7 @@ def _merge_decision_log(
|
|||
existing["decisions"].append(decision)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(existing, f, indent=2)
|
||||
|
||||
|
||||
|
|
@ -351,19 +437,22 @@ def write_checkpoint(
|
|||
metadata: Optional[dict] = None,
|
||||
) -> Path:
|
||||
"""Write a checkpoint file for a pipeline stage."""
|
||||
# Backfill a missing pipeline_type from the project marker so that
|
||||
# omitting the kwarg doesn't quietly bypass gate enforcement.
|
||||
if not pipeline_type:
|
||||
marker = None
|
||||
marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME
|
||||
if marker_path.exists():
|
||||
try:
|
||||
with open(marker_path) as f:
|
||||
marker = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
marker = None
|
||||
if isinstance(marker, dict) and marker.get("pipeline_type"):
|
||||
# Backfill identity fields from the project marker so omitted kwargs
|
||||
# cannot bypass either gate enforcement or style validation.
|
||||
marker = None
|
||||
marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME
|
||||
if marker_path.exists() and (not pipeline_type or not style_playbook):
|
||||
try:
|
||||
with open(marker_path, encoding="utf-8") as f:
|
||||
marker = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
marker = None
|
||||
if isinstance(marker, dict):
|
||||
if not pipeline_type and marker.get("pipeline_type"):
|
||||
pipeline_type = marker["pipeline_type"]
|
||||
if not style_playbook and marker.get("style_playbook"):
|
||||
style_playbook = marker["style_playbook"]
|
||||
_validate_style_playbook(style_playbook)
|
||||
|
||||
valid_stages = (
|
||||
set(get_pipeline_stages(pipeline_type)) if pipeline_type
|
||||
|
|
@ -404,6 +493,14 @@ def write_checkpoint(
|
|||
f"re-write with status='completed', human_approved=True."
|
||||
)
|
||||
|
||||
_enforce_stage_prerequisites(
|
||||
pipeline_dir,
|
||||
project_id,
|
||||
pipeline_type,
|
||||
stage,
|
||||
status,
|
||||
)
|
||||
|
||||
checkpoint = {
|
||||
"version": "1.0",
|
||||
"project_id": project_id,
|
||||
|
|
@ -456,7 +553,7 @@ def write_checkpoint(
|
|||
# current checkpoint; then archive the superseded file and swap in the
|
||||
# new one atomically.
|
||||
tmp_path = path.with_suffix(".json.tmp")
|
||||
with open(tmp_path, "w") as f:
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(checkpoint, f, indent=2)
|
||||
# Preserve run history: a superseded completed/awaiting_human checkpoint
|
||||
# is copied to history/ (stage versioning, gate audit trail, replay).
|
||||
|
|
@ -474,7 +571,7 @@ def read_checkpoint(
|
|||
path = _checkpoint_path(pipeline_dir, project_id, stage)
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
checkpoint = json.load(f)
|
||||
validate_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
|
@ -496,7 +593,7 @@ def get_latest_checkpoint(
|
|||
if not checkpoints:
|
||||
return None
|
||||
|
||||
with open(checkpoints[0]) as f:
|
||||
with open(checkpoints[0], encoding="utf-8") as f:
|
||||
checkpoint = json.load(f)
|
||||
validate_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
|
|
|||
|
|
@ -61,6 +61,20 @@ def model_info() -> dict:
|
|||
}
|
||||
|
||||
|
||||
def _as_feature_tensor(features):
|
||||
"""Normalize CLIP feature return values across transformers versions.
|
||||
|
||||
Transformers 4 returned the projected tensor directly. Transformers 5 may
|
||||
wrap that tensor in a model-output object whose ``pooler_output`` contains
|
||||
the same shared-space embedding. Do not project it again: the vision
|
||||
projection expects the pre-projection width, while ``pooler_output`` is
|
||||
already the final CLIP width.
|
||||
"""
|
||||
|
||||
pooled = getattr(features, "pooler_output", None)
|
||||
return features if pooled is None else pooled
|
||||
|
||||
|
||||
def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray:
|
||||
"""Embed a list of image files into a (N, 512) float32 matrix.
|
||||
|
||||
|
|
@ -82,7 +96,7 @@ def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray:
|
|||
|
||||
inputs = _PROCESSOR(images=images, return_tensors="pt").to(_DEVICE)
|
||||
with torch.no_grad():
|
||||
features = _MODEL.get_image_features(**inputs)
|
||||
features = _as_feature_tensor(_MODEL.get_image_features(**inputs))
|
||||
features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8)
|
||||
arr = features.cpu().numpy().astype(np.float32, copy=False)
|
||||
# Close PIL handles to avoid leaking file handles on Windows
|
||||
|
|
@ -116,7 +130,7 @@ def embed_texts(texts: Sequence[str]) -> np.ndarray:
|
|||
max_length=77,
|
||||
).to(_DEVICE)
|
||||
with torch.no_grad():
|
||||
features = _MODEL.get_text_features(**inputs)
|
||||
features = _as_feature_tensor(_MODEL.get_text_features(**inputs))
|
||||
features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8)
|
||||
return features.cpu().numpy().astype(np.float32, copy=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from functools import lru_cache
|
|||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_manifest_schema() -> dict:
|
||||
with open(SCHEMA_PATH) as f:
|
||||
with open(SCHEMA_PATH, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ def load_pipeline(name: str, defs_dir: Optional[Path] = None) -> dict[str, Any]:
|
|||
if not path.exists():
|
||||
raise FileNotFoundError(f"Pipeline manifest not found: {path}")
|
||||
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
manifest = yaml.safe_load(f)
|
||||
|
||||
schema = _load_manifest_schema()
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
"@remotion/player": "^4.0.484",
|
||||
"@remotion/transitions": "^4.0.484",
|
||||
"d3-geo": "^3.1.1",
|
||||
"fast-uri": "^3.1.5",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"remotion": "^4.0.484",
|
||||
|
|
@ -1951,9 +1952,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz",
|
||||
"integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==",
|
||||
"version": "3.1.5",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
|
||||
"integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
|
@ -2253,9 +2254,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.15",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
|
||||
"integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
|
||||
"version": "3.3.16",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
|
@ -2345,9 +2346,9 @@
|
|||
"license": "ISC"
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.15",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
|
||||
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
|
||||
"version": "8.5.25",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz",
|
||||
"integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "opencollective",
|
||||
|
|
@ -2364,7 +2365,7 @@
|
|||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.12",
|
||||
"nanoid": "^3.3.16",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
"@remotion/player": "^4.0.484",
|
||||
"@remotion/transitions": "^4.0.484",
|
||||
"d3-geo": "^3.1.1",
|
||||
"fast-uri": "^3.1.5",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"remotion": "^4.0.484",
|
||||
|
|
|
|||
|
|
@ -8,31 +8,13 @@ import {
|
|||
Sequence,
|
||||
interpolate,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
|
||||
function resolveAsset(src: string): string {
|
||||
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
|
||||
return src;
|
||||
}
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[/\\]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
import { CinematicRendererProps, CinematicTone, CinematicVideoScene } from "./cinematic/types";
|
||||
import { CaptionOverlay } from "./components/CaptionOverlay";
|
||||
import { resolveAsset } from "./lib/resolveAsset";
|
||||
|
||||
const FPS = 30;
|
||||
|
||||
|
|
@ -57,10 +39,11 @@ const toneGradient = (tone: CinematicTone) => {
|
|||
|
||||
const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { durationInFrames, fps } = useVideoConfig();
|
||||
const { fps } = useVideoConfig();
|
||||
const durationInFrames = Math.max(1, Math.round(scene.durationSeconds * fps));
|
||||
const fadeInFrames = scene.fadeInFrames ?? 10;
|
||||
const fadeOutFrames = scene.fadeOutFrames ?? 10;
|
||||
const fadeOutStart = Math.max(fadeInFrames, durationInFrames - fadeOutFrames);
|
||||
const fadeOutStart = Math.max(0, durationInFrames - fadeOutFrames);
|
||||
const fadeInOpacity =
|
||||
fadeInFrames === 0
|
||||
? 1
|
||||
|
|
@ -98,6 +81,7 @@ const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => {
|
|||
src={resolveAsset(scene.src)}
|
||||
trimBefore={trimBefore}
|
||||
trimAfter={trimAfter}
|
||||
playbackRate={scene.playbackRate}
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@ import {
|
|||
interpolate,
|
||||
random,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import React from "react";
|
||||
import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay";
|
||||
import { resolveAsset } from "./lib/resolveAsset";
|
||||
|
||||
const { fontFamily: playfairFamily } = loadPlayfair("normal", {
|
||||
weights: ["400", "700"],
|
||||
|
|
@ -22,23 +22,6 @@ const { fontFamily: playfairItalic } = loadPlayfair("italic", {
|
|||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
function resolveAsset(src: string): string {
|
||||
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src;
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
|
||||
export type CollageTransition =
|
||||
| "pop"
|
||||
| "slide-zoom"
|
||||
|
|
@ -63,13 +46,13 @@ export interface CollageClip {
|
|||
seed?: number;
|
||||
}
|
||||
|
||||
export interface CollageBurstProps {
|
||||
export type CollageBurstProps = {
|
||||
backgroundSrc: string;
|
||||
backgroundInSeconds?: number;
|
||||
curtainStartSeconds: number;
|
||||
curtainEndSeconds: number;
|
||||
clips: CollageClip[];
|
||||
}
|
||||
};
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Opening text — elegant serif card that lives in the pre-reveal black, then
|
||||
|
|
|
|||
|
|
@ -6,34 +6,10 @@ import {
|
|||
Sequence,
|
||||
interpolate,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import { loadFont } from "@remotion/google-fonts/SpaceGrotesk";
|
||||
|
||||
// Resolve asset path — handle URLs, absolute paths (Windows/Unix), and public/ relative paths
|
||||
function resolveAsset(src: string): string {
|
||||
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
|
||||
return src;
|
||||
}
|
||||
// Strip any file:// prefix
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
// Absolute paths (Unix: /foo, Windows: C:\foo or C:/foo) — convert to file:// URI
|
||||
// staticFile() only accepts relative paths within public/, so absolute paths must bypass it
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
import { TextCard } from "./components/TextCard";
|
||||
import { StatCard } from "./components/StatCard";
|
||||
import { CalloutBox } from "./components/CalloutBox";
|
||||
|
|
@ -54,6 +30,7 @@ import type { TerminalStep } from "./components/TerminalScene";
|
|||
import { ScreenshotScene } from "./components/ScreenshotScene";
|
||||
import type { ScreenshotStep } from "./components/ScreenshotScene";
|
||||
import { ProviderChip } from "./components/ProviderChip";
|
||||
import { resolveAsset } from "./lib/resolveAsset";
|
||||
import type { ParticleType } from "./components/ParticleOverlay";
|
||||
import { resolveTheme, type ThemeConfig, DEFAULT_THEME } from "./Root";
|
||||
|
||||
|
|
@ -254,6 +231,7 @@ interface Cut {
|
|||
animation?: string;
|
||||
transition_in?: string;
|
||||
transition_out?: string;
|
||||
transition_duration?: number;
|
||||
transform?: {
|
||||
animation?: string;
|
||||
scale?: number;
|
||||
|
|
@ -425,22 +403,49 @@ const ImageScene: React.FC<{ src: string; animation?: string }> = ({
|
|||
// Enhanced Video Scene
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VideoScene: React.FC<{ src: string; startFrom?: number }> = ({
|
||||
const VideoScene: React.FC<{
|
||||
src: string;
|
||||
startFrom?: number;
|
||||
transitionIn?: string;
|
||||
transitionOut?: string;
|
||||
transitionDuration?: number;
|
||||
sceneDurationSeconds: number;
|
||||
backgroundColor?: string;
|
||||
}> = ({
|
||||
src,
|
||||
startFrom = 0,
|
||||
transitionIn,
|
||||
transitionOut,
|
||||
transitionDuration,
|
||||
sceneDurationSeconds,
|
||||
backgroundColor = "#0F172A",
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps, durationInFrames } = useVideoConfig();
|
||||
const { fps } = useVideoConfig();
|
||||
const durationInFrames = Math.max(1, Math.round(sceneDurationSeconds * fps));
|
||||
|
||||
const fadeIn = spring({ frame, fps, config: { damping: 20 } });
|
||||
const fadeOutStart = durationInFrames - 8;
|
||||
const fadeOut = interpolate(frame, [fadeOutStart, durationInFrames], [1, 0.3], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const hardIn = ["cut", "none"].includes((transitionIn || "").toLowerCase());
|
||||
const hardOut = ["cut", "none"].includes((transitionOut || "").toLowerCase());
|
||||
const transitionFrames = Math.max(
|
||||
1,
|
||||
Math.round((transitionDuration ?? 8 / fps) * fps),
|
||||
);
|
||||
const fadeIn = hardIn
|
||||
? 1
|
||||
: interpolate(frame, [0, transitionFrames], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const fadeOutStart = Math.max(0, durationInFrames - transitionFrames);
|
||||
const fadeOut = hardOut
|
||||
? 1
|
||||
: interpolate(frame, [fadeOutStart, durationInFrames], [1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: "#0F172A" }}>
|
||||
<AbsoluteFill style={{ background: backgroundColor }}>
|
||||
<OffthreadVideo
|
||||
src={resolveAsset(src)}
|
||||
startFrom={Math.round(startFrom * fps)}
|
||||
|
|
@ -721,7 +726,17 @@ const SceneRenderer: React.FC<{ cut: Cut; theme: ThemeConfig }> = ({ cut, theme
|
|||
}
|
||||
|
||||
if (cut.source && isVideo(cut.source)) {
|
||||
return maybeWrapWithBg(<VideoScene src={cut.source} startFrom={cut.source_in_seconds ?? 0} />);
|
||||
return maybeWrapWithBg(
|
||||
<VideoScene
|
||||
src={cut.source}
|
||||
startFrom={cut.source_in_seconds ?? 0}
|
||||
transitionIn={cut.transition_in}
|
||||
transitionOut={cut.transition_out}
|
||||
transitionDuration={cut.transition_duration}
|
||||
sceneDurationSeconds={cut.out_seconds - cut.in_seconds}
|
||||
backgroundColor={cut.backgroundColor}
|
||||
/>,
|
||||
);
|
||||
}
|
||||
|
||||
// Final fallback — try as image if source exists, otherwise show text_card
|
||||
|
|
@ -741,7 +756,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
|
|||
if (overlay.type === "section_title") {
|
||||
return (
|
||||
<SectionTitle
|
||||
title={overlay.text}
|
||||
title={overlay.text ?? ""}
|
||||
subtitle={overlay.subtitle}
|
||||
accentColor={overlay.accentColor}
|
||||
position={(overlay.position as any) || "top-left"}
|
||||
|
|
@ -751,7 +766,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
|
|||
if (overlay.type === "stat_reveal") {
|
||||
return (
|
||||
<StatReveal
|
||||
stat={overlay.text}
|
||||
stat={overlay.text ?? ""}
|
||||
label={overlay.subtitle}
|
||||
accentColor={overlay.accentColor}
|
||||
position={(overlay.position as any) || "bottom-right"}
|
||||
|
|
@ -759,7 +774,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => {
|
|||
);
|
||||
}
|
||||
if (overlay.type === "hero_title") {
|
||||
return <HeroTitle title={overlay.text} subtitle={overlay.subtitle} />;
|
||||
return <HeroTitle title={overlay.text ?? ""} subtitle={overlay.subtitle} />;
|
||||
}
|
||||
if (overlay.type === "provider_chip" && overlay.providers) {
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -3,46 +3,29 @@ import {
|
|||
Audio,
|
||||
OffthreadVideo,
|
||||
interpolate,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import React from "react";
|
||||
import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay";
|
||||
import { resolveAsset } from "./lib/resolveAsset";
|
||||
|
||||
const { fontFamily: playfairItalic } = loadPlayfair("italic", {
|
||||
weights: ["400", "700"],
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
function resolveAsset(src: string): string {
|
||||
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src;
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
|
||||
export interface Lyric {
|
||||
text: string;
|
||||
inSeconds: number;
|
||||
outSeconds: number;
|
||||
}
|
||||
|
||||
export interface LyricOverlayProps {
|
||||
export type LyricOverlayProps = {
|
||||
videoSrc: string;
|
||||
lyrics: Lyric[];
|
||||
bottomY?: number; // 0..1, vertical center of subtitle band
|
||||
}
|
||||
};
|
||||
|
||||
const LyricLine: React.FC<{ lyric: Lyric; bottomY: number }> = ({ lyric, bottomY }) => {
|
||||
const frame = useCurrentFrame();
|
||||
|
|
|
|||
|
|
@ -5,12 +5,12 @@ import {
|
|||
Sequence,
|
||||
interpolate,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import { getVideoMetadata } from "@remotion/media-utils";
|
||||
import { loadFont } from "@remotion/google-fonts/PlayfairDisplay";
|
||||
import { resolveAsset } from "./lib/resolveAsset";
|
||||
|
||||
// Editorial serif for the tagline — Playfair Display at its boldest weight.
|
||||
// Loaded once at module scope so every render reuses the same font face.
|
||||
|
|
@ -19,7 +19,7 @@ const { fontFamily } = loadFont("normal", {
|
|||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
export interface TitledVideoProps {
|
||||
export type TitledVideoProps = {
|
||||
videoSrc: string;
|
||||
tagline: string;
|
||||
// When the tagline starts animating in, in seconds from the start of the video.
|
||||
|
|
@ -32,33 +32,10 @@ export interface TitledVideoProps {
|
|||
fontSize?: number;
|
||||
// Accent color used for the underline and the glow halo.
|
||||
accentColor?: string;
|
||||
}
|
||||
};
|
||||
|
||||
// Resolve asset path — handle URLs, absolute paths, and public/ relative paths.
|
||||
// Mirrors the helper in Explainer.tsx so absolute Windows/Unix paths work.
|
||||
function resolveAsset(src: string): string {
|
||||
if (
|
||||
src.startsWith("http://") ||
|
||||
src.startsWith("https://") ||
|
||||
src.startsWith("data:")
|
||||
) {
|
||||
return src;
|
||||
}
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// EditorialTagline — big bold serif, upper-third, drawn underline, warm glow.
|
||||
// Letter-by-letter spring entrance. Designed to feel like a printed headline,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export interface CinematicVideoScene extends CinematicBaseScene {
|
|||
tone?: CinematicTone;
|
||||
trimBeforeSeconds?: number;
|
||||
trimAfterSeconds?: number;
|
||||
playbackRate?: number;
|
||||
filter?: string;
|
||||
fadeInFrames?: number;
|
||||
fadeOutFrames?: number;
|
||||
|
|
|
|||
|
|
@ -3,27 +3,11 @@ import {
|
|||
Img,
|
||||
interpolate,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import { ParticleOverlay, type ParticleType } from "./ParticleOverlay";
|
||||
|
||||
/**
|
||||
* Resolve asset path — use staticFile() for local paths, passthrough URLs.
|
||||
* Duplicated from Explainer.tsx to keep the component self-contained.
|
||||
*/
|
||||
function resolveAsset(src: string): string {
|
||||
if (
|
||||
src.startsWith("http://") ||
|
||||
src.startsWith("https://") ||
|
||||
src.startsWith("data:")
|
||||
) {
|
||||
return src;
|
||||
}
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
return staticFile(clean);
|
||||
}
|
||||
import { resolveAsset } from "../lib/resolveAsset";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export interface WordCaption {
|
|||
endMs: number;
|
||||
}
|
||||
|
||||
interface CaptionOverlayProps {
|
||||
type CaptionOverlayProps = {
|
||||
words: WordCaption[];
|
||||
// How many words to show at once in a "page"
|
||||
wordsPerPage?: number;
|
||||
|
|
@ -23,7 +23,7 @@ interface CaptionOverlayProps {
|
|||
highlightColor?: string;
|
||||
backgroundColor?: string;
|
||||
fontFamily?: string;
|
||||
}
|
||||
};
|
||||
|
||||
interface CaptionPage {
|
||||
words: WordCaption[];
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
useVideoConfig,
|
||||
} from "remotion";
|
||||
|
||||
export interface EndTagProps {
|
||||
export type EndTagProps = {
|
||||
text: string;
|
||||
palette?: "cool_offwhite_on_black" | "warm_ivory_on_black";
|
||||
// Optional extra fade hold controls (all in seconds)
|
||||
|
|
@ -19,7 +19,7 @@ export interface EndTagProps {
|
|||
// AbsoluteFill drops its background fill — caller is responsible for
|
||||
// rendering with an alpha-capable codec (VP9/WebM or ProRes 4444).
|
||||
overlay?: boolean;
|
||||
}
|
||||
};
|
||||
|
||||
const PALETTES = {
|
||||
cool_offwhite_on_black: {
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ import {
|
|||
useVideoConfig,
|
||||
} from "remotion";
|
||||
|
||||
interface HeroTitleProps {
|
||||
type HeroTitleProps = {
|
||||
title: string;
|
||||
subtitle?: string;
|
||||
}
|
||||
};
|
||||
|
||||
export const HeroTitle: React.FC<HeroTitleProps> = ({ title, subtitle }) => {
|
||||
const frame = useCurrentFrame();
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@ import {
|
|||
Easing,
|
||||
} from "remotion";
|
||||
|
||||
export interface ProductRevealProps {
|
||||
export type ProductRevealProps = {
|
||||
productImage: string;
|
||||
productName: string;
|
||||
price: string;
|
||||
tagline: string;
|
||||
closer: string;
|
||||
accentColor?: string;
|
||||
}
|
||||
};
|
||||
|
||||
export const ProductReveal: React.FC<ProductRevealProps> = ({
|
||||
productImage,
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ export const ProviderChip: React.FC<ProviderChipProps> = ({
|
|||
const translateY = interpolate(springIn, [0, 1], [12, 0]);
|
||||
|
||||
return (
|
||||
<AbsoluteFill pointerEvents="none">
|
||||
<AbsoluteFill style={{ pointerEvents: "none" }}>
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import {
|
|||
Img,
|
||||
interpolate,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from "remotion";
|
||||
import { resolveAsset } from "../lib/resolveAsset";
|
||||
|
||||
/**
|
||||
* ScreenshotScene — approach-1 synthetic UI demo.
|
||||
|
|
@ -84,25 +84,6 @@ interface ScreenshotSceneProps {
|
|||
|
||||
// ---------- Helpers ----------
|
||||
|
||||
function resolveAsset(src: string): string {
|
||||
if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) {
|
||||
return src;
|
||||
}
|
||||
const clean = src.replace(/^file:\/\/\/?/, "");
|
||||
if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
// POSIX absolute paths already have a leading "/" — file:// + posix
|
||||
// gives exactly three slashes. Windows drive paths (C:/...) need the
|
||||
// extra slash added explicitly. Do not merge these branches — adding
|
||||
// "file:///" unconditionally double-slashes POSIX paths (file:////...).
|
||||
if (posix.startsWith("/")) {
|
||||
return `file://${posix}`;
|
||||
}
|
||||
return `file:///${posix}`;
|
||||
}
|
||||
return staticFile(clean);
|
||||
}
|
||||
|
||||
/** Compute the rendered bounding box of the backdrop inside a canvas,
|
||||
* using object-fit: contain semantics. Returns pixel offsets/sizes. */
|
||||
function containRect(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
import { staticFile } from "remotion";
|
||||
|
||||
const isRemoteAsset = (src: string): boolean =>
|
||||
src.startsWith("http://") ||
|
||||
src.startsWith("https://") ||
|
||||
src.startsWith("data:");
|
||||
|
||||
const isWindowsAbsolutePath = (src: string): boolean =>
|
||||
/^[A-Za-z]:[\\/]/.test(src);
|
||||
|
||||
/** Resolve public assets and absolute filesystem paths consistently. */
|
||||
export function resolveAsset(src: string): string {
|
||||
if (isRemoteAsset(src)) {
|
||||
return src;
|
||||
}
|
||||
|
||||
const withoutScheme = src.replace(/^file:\/\//i, "");
|
||||
const clean = /^\/[A-Za-z]:[\\/]/.test(withoutScheme)
|
||||
? withoutScheme.slice(1)
|
||||
: withoutScheme;
|
||||
|
||||
if (clean.startsWith("/") || isWindowsAbsolutePath(clean)) {
|
||||
const posix = clean.replace(/\\/g, "/");
|
||||
return posix.startsWith("/") ? `file://${posix}` : `file:///${posix}`;
|
||||
}
|
||||
|
||||
return staticFile(clean);
|
||||
}
|
||||
|
|
@ -39,7 +39,7 @@ def load_schema(name: str) -> dict:
|
|||
path = SCHEMA_DIR / f"{name}.schema.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Schema not found: {path}")
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,10 @@
|
|||
"minimum": 0,
|
||||
"description": "Duration of transition in seconds"
|
||||
},
|
||||
"backgroundColor": {
|
||||
"type": "string",
|
||||
"description": "Backing color used behind this cut by composition runtimes"
|
||||
},
|
||||
"reason": { "type": "string" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
"description": { "type": "string" },
|
||||
"category": {
|
||||
"type": "string",
|
||||
"enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "custom"]
|
||||
"enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "documentary", "custom"]
|
||||
},
|
||||
"stability": {
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -184,6 +184,15 @@ def stage_project(pid: str, title: str, palette: str, scenes: list, *,
|
|||
brief["topic"] = title
|
||||
cp("research", "completed", {"research_brief": brief})
|
||||
|
||||
proposal = sample_artifact("proposal_packet")
|
||||
cp("proposal", "awaiting_human", {"proposal_packet": proposal})
|
||||
cp(
|
||||
"proposal",
|
||||
"completed",
|
||||
{"proposal_packet": proposal},
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
script = script_artifact(title, scenes)
|
||||
plan = scene_plan_artifact(scenes, hero)
|
||||
(art_dir / "decision_log.json").write_text(json.dumps(decision_log(pid), indent=2))
|
||||
|
|
|
|||
|
|
@ -60,7 +60,9 @@ Key capability families to look for in the output:
|
|||
| `analysis` | — | Mixed providers |
|
||||
| `character_animation` | — | Local character specs, SVG rigs, pose libraries, action timelines, previews, and QA |
|
||||
| `graphics` | — | Local rendering tools |
|
||||
| `music_generation` | — | Single-provider |
|
||||
| `music_library` | — | Discovers user-provided local tracks |
|
||||
| `music_search` | — | Discovers royalty-free search/download providers |
|
||||
| `music_generation` | — | Discovers paid/local generation providers |
|
||||
| `subtitle` | — | Pure Python |
|
||||
| `avatar` | — | Local GPU models |
|
||||
| `video_post` | — | FFmpeg-based local tools |
|
||||
|
|
|
|||
|
|
@ -88,23 +88,26 @@ Cinematic videos live and die by their audio. **Surface the music situation befo
|
|||
|
||||
Check availability in this order:
|
||||
|
||||
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. List available tracks with durations and moods. Let the user choose.
|
||||
2. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, and cost per track.
|
||||
3. **Royalty-free sources:** Note that the user can provide a track from YouTube Audio Library, Jamendo, or other free sources by dropping it in `music_library/`.
|
||||
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List available tracks with durations and moods.
|
||||
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report available providers, licensing constraints, and any key requirement.
|
||||
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs.
|
||||
4. **Bring your own:** The user can drop a track from another licensed source into `music_library/`.
|
||||
|
||||
Present explicit options:
|
||||
|
||||
```
|
||||
MUSIC PLAN
|
||||
├── Your music library: [N tracks / empty]
|
||||
├── Royalty-free search: [providers / unavailable]
|
||||
├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost]
|
||||
└── Bring your own: Drop a track in music_library/ before asset stage
|
||||
|
||||
Options:
|
||||
(a) Use a library track (which one?)
|
||||
(b) Provide your own track
|
||||
(c) Generate via API (if available)
|
||||
(d) Proceed without music (not recommended for cinematic)
|
||||
(b) Search a royalty-free provider
|
||||
(c) Provide your own track
|
||||
(d) Generate via API (if available)
|
||||
(e) Proceed without music (not recommended for cinematic)
|
||||
```
|
||||
|
||||
Record the decision in `brief.metadata.music_strategy` with the chosen source and path/prompt.
|
||||
|
|
|
|||
|
|
@ -222,14 +222,16 @@ Let the user select, combine, modify, or redirect entirely.
|
|||
Cinematic videos live and die by their audio. Surface the music situation before the user approves.
|
||||
|
||||
Check availability in this order:
|
||||
1. **User music library (`music_library/`)** — list available tracks
|
||||
2. **Music generation APIs** — report status, cost, and quality honestly
|
||||
3. **Bring-your-own path** — user can drop a track in `music_library/`
|
||||
1. **User music library** — query `registry.get_by_capability("music_library")` and list available tracks
|
||||
2. **Royalty-free search** — query `registry.get_by_capability("music_search")` and report providers/licensing
|
||||
3. **Music generation APIs** — query `registry.get_by_capability("music_generation")` and report status, cost, and quality honestly
|
||||
4. **Bring-your-own path** — user can drop a track in `music_library/`
|
||||
|
||||
Present explicit options:
|
||||
```
|
||||
MUSIC PLAN
|
||||
├── Your music library: [N tracks / empty]
|
||||
├── Royalty-free search: [providers / unavailable]
|
||||
├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost]
|
||||
└── Bring your own: Drop a track in music_library/ before asset stage
|
||||
|
||||
|
|
|
|||
|
|
@ -89,11 +89,14 @@ like abandoned footage at compose time. Do not assume silence will earn
|
|||
itself. If the user has not mentioned music, ASSUME THEY WANT IT and pick:
|
||||
|
||||
- user-provided track (put path in `music_plan.source_path`),
|
||||
- music library pick (list what's in `music_library/`),
|
||||
- music library pick (query `registry.get_by_capability("music_library")` and list tracks),
|
||||
- royalty-free search (query `registry.get_by_capability("music_search")`, report provider and license),
|
||||
- generated (name the tool and prompt seed with register),
|
||||
- explicit opt-out (`source: "none"` + `opt_out_reason`).
|
||||
|
||||
**Warn the user if no music source is available.** Do not silently
|
||||
Before declaring no source available, also query
|
||||
`registry.get_by_capability("music_generation")`. **Warn the user if no music
|
||||
source is available.** Do not silently
|
||||
defer this — it becomes an expensive surprise at the asset stage.
|
||||
|
||||
### 5. Note End-Tag Intent (MANDATORY)
|
||||
|
|
|
|||
|
|
@ -129,7 +129,9 @@ If any dimension scores below 3, iterate before submitting. The reviewer will ch
|
|||
|
||||
### Step 7: Submit
|
||||
|
||||
Call `handle_explainer_idea(state, {"brief": brief_json})` to validate and persist.
|
||||
Validate `brief_json` against the canonical brief schema, persist it through
|
||||
the checkpoint protocol, and attach the stage review. There is no separate
|
||||
explainer submit function.
|
||||
|
||||
## Playbook Selection Guide
|
||||
|
||||
|
|
|
|||
|
|
@ -369,9 +369,9 @@ Music is a critical part of the video's feel. **Surface the music situation to t
|
|||
|
||||
**Check music availability in this order:**
|
||||
|
||||
1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one.
|
||||
2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly.
|
||||
3. **Stock music sources:** Note if stock music is available via any provider.
|
||||
1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List tracks with durations.
|
||||
2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report providers, licensing constraints, and key requirements.
|
||||
3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly.
|
||||
|
||||
**Present to the user:**
|
||||
|
||||
|
|
@ -381,15 +381,17 @@ MUSIC PLAN
|
|||
│ ├── cosmic_interstellar_space.mp3 (3:13) — ambient, cosmic
|
||||
│ ├── cinematic_epic.mp3 (2:45) — dramatic, building
|
||||
│ └── lofi_beat.mp3 (4:00) — chill, electronic
|
||||
├── Royalty-free search: [providers / unavailable]
|
||||
├── AI generation: music_gen (ElevenLabs) — UNAVAILABLE (plan limit)
|
||||
└── Recommendation: Use "cosmic_interstellar_space.mp3" from your library
|
||||
OR provide a different track before asset generation
|
||||
|
||||
Would you like to:
|
||||
(a) Use a track from your library (which one?)
|
||||
(b) Provide a different track (drop it in music_library/)
|
||||
(c) Generate one via API (if available)
|
||||
(d) Proceed without music
|
||||
(b) Search a royalty-free provider
|
||||
(c) Provide a different track (drop it in music_library/)
|
||||
(d) Generate one via API (if available)
|
||||
(e) Proceed without music
|
||||
```
|
||||
|
||||
**If no music source is available:** Tell the user explicitly. Do NOT let this surface as a surprise at the asset stage. Offer the `music_library/` path so they can add a track before production starts.
|
||||
|
|
|
|||
|
|
@ -63,9 +63,17 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v
|
|||
}
|
||||
```
|
||||
|
||||
#### Scene Types and When to Use Them
|
||||
#### Render Templates and Scene Types
|
||||
|
||||
| Type | Best For | Available Tools | Duration Guidance |
|
||||
The canonical `scene_plan.scenes[].type` vocabulary is `talking_head`, `broll`,
|
||||
`animation`, `character_scene`, `diagram`, `text_card`, `transition`,
|
||||
`generated`, and `screen_recording`. Names such as `hero_title`, `stat_card`,
|
||||
and chart/card variants below are downstream Remotion `cut.type` templates,
|
||||
not valid scene-plan types. During scene planning, use `text_card` or
|
||||
`animation` as appropriate and state the intended render template in the scene
|
||||
description; the Edit stage converts that intent into `cut.type`.
|
||||
|
||||
| Render template or scene type | Best For | Available Tools | Duration Guidance |
|
||||
|------|----------|-----------------|-------------------|
|
||||
| `hero_title` | Opening titles, dramatic reveals | Remotion HeroTitle (theme-driven title treatment) | 3-5s |
|
||||
| `stat_card` | Big dramatic numbers, impactful metrics | Remotion StatCard (large stat + subtitle) | 4-6s |
|
||||
|
|
@ -84,7 +92,11 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v
|
|||
| `broll` | Context, real-world examples | Stock or generated footage | 3-6s |
|
||||
| `screen_recording` | Code demos, UI walkthroughs | Recorded or simulated | 5-15s |
|
||||
|
||||
**Zero-key scene selection:** When no image/video generation is available, prefer `hero_title`, `stat_card`, `bar_chart`, `line_chart`, `pie_chart`, `kpi_grid`, `comparison`, `callout`, `progress_bar`, and `text_card`. These render entirely from Remotion components with zero external dependencies and can still feel distinct if you derive color, typography, and pacing from the subject instead of defaulting to a generic dashboard aesthetic.
|
||||
**Zero-key scene selection:** When no image/video generation is available,
|
||||
plan `text_card`, `animation`, or `diagram` scenes and name an appropriate
|
||||
downstream template (`hero_title`, `stat_card`, charts, `comparison`,
|
||||
`callout`, or `progress_bar`) in each description. These render entirely from
|
||||
Remotion components with zero external dependencies.
|
||||
|
||||
### Step 4: Apply the Visual Technique Library
|
||||
|
||||
|
|
@ -226,7 +238,9 @@ If any dimension scores below 3, revise.
|
|||
|
||||
### Step 8: Submit
|
||||
|
||||
Call `handle_explainer_scene_plan(state, {"scene_plan": scene_plan_json})` to validate and persist.
|
||||
Validate `scene_plan_json` against the canonical scene-plan schema, persist it
|
||||
through the checkpoint protocol, and attach the stage review. There is no
|
||||
separate explainer submit function.
|
||||
|
||||
## Common Pitfalls
|
||||
|
||||
|
|
|
|||
|
|
@ -206,7 +206,9 @@ If any dimension scores below 3, revise before submitting.
|
|||
|
||||
### Step 7: Submit
|
||||
|
||||
Call `handle_explainer_script(state, {"script": script_json})` to validate and persist.
|
||||
Validate `script_json` against the canonical script schema, persist it through
|
||||
the checkpoint protocol, and attach the stage review. There is no separate
|
||||
explainer submit function.
|
||||
|
||||
### Mid-Production Fact Verification
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ SCHEMA_PATH = (
|
|||
|
||||
|
||||
def _load_playbook_schema() -> dict:
|
||||
with open(SCHEMA_PATH) as f:
|
||||
with open(SCHEMA_PATH, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ def load_playbook(name: str, styles_dir: Optional[Path] = None) -> dict[str, Any
|
|||
if not path.exists():
|
||||
raise FileNotFoundError(f"Playbook not found: {path}")
|
||||
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
playbook = yaml.safe_load(f)
|
||||
|
||||
validate_playbook(playbook)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,11 @@ import pytest
|
|||
|
||||
from backlot import state as state_mod
|
||||
from backlot.state import load_board_state
|
||||
from lib.checkpoint import CheckpointValidationError, write_checkpoint
|
||||
from lib.checkpoint import (
|
||||
CANONICAL_STAGE_ARTIFACTS,
|
||||
CheckpointValidationError,
|
||||
write_checkpoint,
|
||||
)
|
||||
|
||||
|
||||
def _script_artifact() -> dict:
|
||||
|
|
@ -23,6 +27,22 @@ def _manifest_artifact() -> dict:
|
|||
return {"version": "1.0", "assets": [], "total_cost_usd": 0.0}
|
||||
|
||||
|
||||
def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None:
|
||||
from tests.contracts.test_phase0_contracts import sample_artifact
|
||||
|
||||
for stage in stages:
|
||||
artifact_name = CANONICAL_STAGE_ARTIFACTS[stage]
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
project_id,
|
||||
stage,
|
||||
"completed",
|
||||
{artifact_name: sample_artifact(artifact_name)},
|
||||
pipeline_type=pipeline_type,
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
def _write(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
|
@ -73,6 +93,15 @@ def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypat
|
|||
|
||||
|
||||
def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path):
|
||||
_approve_predecessors(
|
||||
tmp_path,
|
||||
"film",
|
||||
"cinematic",
|
||||
"research",
|
||||
"proposal",
|
||||
"script",
|
||||
"scene_plan",
|
||||
)
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
"film",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import urllib.request
|
|||
|
||||
import pytest
|
||||
|
||||
from lib.checkpoint import init_project, write_checkpoint
|
||||
from lib.checkpoint import CANONICAL_STAGE_ARTIFACTS, init_project, write_checkpoint
|
||||
from lib.pipeline_loader import get_stage_order, load_pipeline
|
||||
from scripts import backlot_screenshot_stage
|
||||
from tests.contracts.test_phase0_contracts import sample_artifact
|
||||
|
||||
|
|
@ -32,6 +33,28 @@ APPROVAL_CASES = [
|
|||
]
|
||||
|
||||
|
||||
def _complete_predecessors(root, project_id: str, pipeline_type: str, stage: str) -> None:
|
||||
order = get_stage_order(load_pipeline(pipeline_type))
|
||||
for predecessor in order[: order.index(stage)]:
|
||||
artifact_name = CANONICAL_STAGE_ARTIFACTS.get(predecessor)
|
||||
if artifact_name:
|
||||
artifact = sample_artifact(artifact_name)
|
||||
if artifact_name == "edit_decisions":
|
||||
artifact["render_runtime"] = "ffmpeg"
|
||||
artifacts = {artifact_name: artifact}
|
||||
else:
|
||||
artifacts = {}
|
||||
write_checkpoint(
|
||||
root,
|
||||
project_id,
|
||||
predecessor,
|
||||
"completed",
|
||||
artifacts,
|
||||
pipeline_type=pipeline_type,
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_approval_projects() -> None:
|
||||
root = backlot_screenshot_stage.STAGE_DIR
|
||||
for project_id, pipeline_type, stage, artifact_name, _visible_text in APPROVAL_CASES:
|
||||
|
|
@ -55,6 +78,7 @@ def _build_approval_projects() -> None:
|
|||
pipeline_type=pipeline_type,
|
||||
pipeline_dir=root,
|
||||
)
|
||||
_complete_predecessors(root, project_id, pipeline_type, stage)
|
||||
write_checkpoint(
|
||||
root,
|
||||
project_id,
|
||||
|
|
@ -80,6 +104,12 @@ def _build_approval_projects() -> None:
|
|||
pipeline_type="character-animation",
|
||||
pipeline_dir=root,
|
||||
)
|
||||
_complete_predecessors(
|
||||
root,
|
||||
"gate-character-design",
|
||||
"character-animation",
|
||||
"character_design",
|
||||
)
|
||||
write_checkpoint(
|
||||
root,
|
||||
"gate-character-design",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _read(relative_path: str) -> str:
|
||||
return (REPO_ROOT / relative_path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_music_plans_discover_all_music_capabilities() -> None:
|
||||
instruction_files = [
|
||||
"AGENT_GUIDE.md",
|
||||
"skills/pipelines/cinematic/idea-director.md",
|
||||
"skills/pipelines/cinematic/proposal-director.md",
|
||||
"skills/pipelines/documentary-montage/idea-director.md",
|
||||
"skills/pipelines/explainer/proposal-director.md",
|
||||
]
|
||||
|
||||
for relative_path in instruction_files:
|
||||
text = _read(relative_path)
|
||||
for capability in ("music_library", "music_search", "music_generation"):
|
||||
assert f'get_by_capability("{capability}")' in text, (
|
||||
f"{relative_path} omits the {capability!r} music source"
|
||||
)
|
||||
|
||||
|
||||
def test_explainer_directors_do_not_reference_fictitious_submit_functions() -> None:
|
||||
for stage in ("idea", "script", "scene"):
|
||||
text = _read(f"skills/pipelines/explainer/{stage}-director.md")
|
||||
assert "handle_explainer_" not in text
|
||||
|
|
@ -6,6 +6,7 @@ import json
|
|||
import pytest
|
||||
|
||||
from lib.checkpoint import (
|
||||
CANONICAL_STAGE_ARTIFACTS,
|
||||
CheckpointValidationError,
|
||||
HISTORY_DIRNAME,
|
||||
PROJECT_MARKER_FILENAME,
|
||||
|
|
@ -27,6 +28,22 @@ def _minimal_script() -> dict:
|
|||
}
|
||||
|
||||
|
||||
def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None:
|
||||
from tests.contracts.test_phase0_contracts import sample_artifact
|
||||
|
||||
for stage in stages:
|
||||
artifact_name = CANONICAL_STAGE_ARTIFACTS[stage]
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
project_id,
|
||||
stage,
|
||||
"completed",
|
||||
artifacts={artifact_name: sample_artifact(artifact_name)},
|
||||
pipeline_type=pipeline_type,
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
class TestGateEnforcement:
|
||||
"""GI-4: gated stages cannot be completed without approval evidence."""
|
||||
|
||||
|
|
@ -39,6 +56,9 @@ class TestGateEnforcement:
|
|||
)
|
||||
|
||||
def test_awaiting_human_is_the_correct_gate_state(self, tmp_path):
|
||||
_approve_predecessors(
|
||||
tmp_path, "proj", "animated-explainer", "research", "proposal"
|
||||
)
|
||||
path = write_checkpoint(
|
||||
tmp_path, "proj", "script", "awaiting_human",
|
||||
artifacts={"script": _minimal_script()},
|
||||
|
|
@ -51,6 +71,9 @@ class TestGateEnforcement:
|
|||
assert cp["human_approval_required"] is True
|
||||
|
||||
def test_completed_with_approval_passes(self, tmp_path):
|
||||
_approve_predecessors(
|
||||
tmp_path, "proj", "animated-explainer", "research", "proposal"
|
||||
)
|
||||
path = write_checkpoint(
|
||||
tmp_path, "proj", "script", "completed",
|
||||
artifacts={"script": _minimal_script()},
|
||||
|
|
@ -84,6 +107,9 @@ class TestCheckpointHistory:
|
|||
"""Superseded checkpoints are archived, not destroyed."""
|
||||
|
||||
def test_overwrite_archives_previous(self, tmp_path):
|
||||
_approve_predecessors(
|
||||
tmp_path, "proj", "animated-explainer", "research", "proposal"
|
||||
)
|
||||
write_checkpoint(
|
||||
tmp_path, "proj", "script", "awaiting_human",
|
||||
artifacts={"script": _minimal_script()},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
from pathlib import Path
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_env_example_does_not_turn_comments_into_credentials() -> None:
|
||||
"""A fresh copy of .env.example must leave every documented key unset."""
|
||||
|
||||
values = dotenv_values(REPO_ROOT / ".env.example")
|
||||
false_credentials = {
|
||||
key: value
|
||||
for key, value in values.items()
|
||||
if isinstance(value, str) and value.lstrip().startswith("#")
|
||||
}
|
||||
|
||||
assert false_credentials == {}
|
||||
|
|
@ -174,10 +174,14 @@ class TestPhase2ErrorHandling:
|
|||
# Either succeeds (provider available) or fails gracefully
|
||||
assert isinstance(r, ToolResult)
|
||||
|
||||
def test_diagram_gen_empty_boxes(self):
|
||||
def test_diagram_gen_empty_boxes(self, tmp_path):
|
||||
tool = DiagramGen()
|
||||
if tool.get_status() == ToolStatus.AVAILABLE:
|
||||
r = tool.execute({"diagram_type": "boxes", "boxes": []})
|
||||
r = tool.execute({
|
||||
"diagram_type": "boxes",
|
||||
"boxes": [],
|
||||
"output_path": str(tmp_path / "empty-boxes.png"),
|
||||
})
|
||||
assert isinstance(r, ToolResult)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import builtins
|
|||
import base64
|
||||
import os
|
||||
import shutil
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -571,16 +572,26 @@ class TestVeoVideo:
|
|||
called_kwargs = mock_client.models.generate_videos.call_args[1]
|
||||
assert called_kwargs["image"] is not None
|
||||
|
||||
def test_vertex_ai_mode_rejection(self):
|
||||
def test_vertex_ai_mode_requires_inline_video_bytes(self):
|
||||
tool = VeoVideo()
|
||||
mock_client = MagicMock()
|
||||
mock_client.vertexai = True
|
||||
if hasattr(mock_client, "_api_client"):
|
||||
delattr(mock_client, "_api_client")
|
||||
mock_client.models.generate_videos.return_value = SimpleNamespace(
|
||||
done=True,
|
||||
error=None,
|
||||
response=SimpleNamespace(
|
||||
generated_videos=[
|
||||
SimpleNamespace(video=SimpleNamespace(video_bytes=None))
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
patch.dict(os.environ, {"GEMINI_API_KEY": "test_key"}),
|
||||
patch("google.genai.Client", return_value=mock_client),
|
||||
patch(
|
||||
"tools.google_credentials.get_genai_client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
):
|
||||
inputs = {
|
||||
"prompt": "cinematic shot",
|
||||
|
|
@ -589,7 +600,7 @@ class TestVeoVideo:
|
|||
res = tool.execute(inputs)
|
||||
assert res.success is False
|
||||
assert res.error is not None
|
||||
assert "only supported using the Gemini Developer API" in res.error
|
||||
assert "without inline bytes" in res.error
|
||||
|
||||
def test_missing_local_image_paths(self):
|
||||
tool = VeoVideo()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
"""Contracts for the category vocabulary used by shipped pipelines."""
|
||||
|
||||
from lib.pipeline_loader import load_pipeline
|
||||
|
||||
|
||||
def test_documentary_pipeline_uses_a_schema_supported_category() -> None:
|
||||
manifest = load_pipeline("documentary-montage")
|
||||
|
||||
assert manifest["category"] == "documentary"
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_video_scene_honors_hard_cut_tokens_and_backing_color() -> None:
|
||||
source = (REPO_ROOT / "remotion-composer/src/Explainer.tsx").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert '["cut", "none"].includes((transitionIn || "").toLowerCase())' in source
|
||||
assert '["cut", "none"].includes((transitionOut || "").toLowerCase())' in source
|
||||
assert "transitionIn={cut.transition_in}" in source
|
||||
assert "transitionOut={cut.transition_out}" in source
|
||||
assert "sceneDurationSeconds={cut.out_seconds - cut.in_seconds}" in source
|
||||
assert "Math.round(sceneDurationSeconds * fps)" in source
|
||||
assert "durationInFrames - transitionFrames" in source
|
||||
assert "backgroundColor={cut.backgroundColor}" in source
|
||||
|
||||
|
||||
def test_cinematic_fades_are_bounded_by_each_scene_duration() -> None:
|
||||
source = (REPO_ROOT / "remotion-composer/src/CinematicRenderer.tsx").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "Math.round(scene.durationSeconds * fps)" in source
|
||||
assert "durationInFrames - fadeOutFrames" in source
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME_FILES = [
|
||||
"lib/checkpoint.py",
|
||||
"lib/pipeline_loader.py",
|
||||
"schemas/artifacts/__init__.py",
|
||||
"styles/playbook_loader.py",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("relative_path", RUNTIME_FILES)
|
||||
def test_pipeline_contract_files_use_explicit_utf8(relative_path: str) -> None:
|
||||
path = REPO_ROOT / relative_path
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
|
||||
bare_opens = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
if not isinstance(node.func, ast.Name) or node.func.id != "open":
|
||||
continue
|
||||
if not any(keyword.arg == "encoding" for keyword in node.keywords):
|
||||
bare_opens.append(node.lineno)
|
||||
|
||||
assert bare_opens == [], f"bare open() calls at lines {bare_opens}"
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
from tests.contracts.test_phase0_contracts import sample_artifact
|
||||
|
||||
from lib.checkpoint import (
|
||||
CheckpointValidationError,
|
||||
init_project,
|
||||
write_checkpoint,
|
||||
)
|
||||
|
||||
|
||||
def _script_artifact() -> dict:
|
||||
return {
|
||||
"version": "1.0",
|
||||
"title": "Smoke",
|
||||
"total_duration_seconds": 1,
|
||||
"sections": [
|
||||
{
|
||||
"id": "s1",
|
||||
"text": "One second.",
|
||||
"start_seconds": 0,
|
||||
"end_seconds": 1,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_later_stage_cannot_skip_a_missing_predecessor(tmp_path) -> None:
|
||||
init_project(
|
||||
"run",
|
||||
title="Run",
|
||||
pipeline_type="framework-smoke",
|
||||
pipeline_dir=tmp_path,
|
||||
)
|
||||
|
||||
with pytest.raises(CheckpointValidationError, match="PREREQUISITE VIOLATION"):
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
"run",
|
||||
"script",
|
||||
"completed",
|
||||
{"script": _script_artifact()},
|
||||
pipeline_type="framework-smoke",
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
def test_later_stage_rejects_unapproved_gated_predecessor(tmp_path) -> None:
|
||||
project_dir = init_project(
|
||||
"run",
|
||||
title="Run",
|
||||
pipeline_type="framework-smoke",
|
||||
pipeline_dir=tmp_path,
|
||||
)
|
||||
predecessor_path = write_checkpoint(
|
||||
tmp_path,
|
||||
"run",
|
||||
"research",
|
||||
"awaiting_human",
|
||||
{"research_brief": sample_artifact("research_brief")},
|
||||
pipeline_type="framework-smoke",
|
||||
)
|
||||
predecessor = json.loads(predecessor_path.read_text(encoding="utf-8"))
|
||||
predecessor["status"] = "completed"
|
||||
predecessor["human_approved"] = False
|
||||
predecessor_path.write_text(json.dumps(predecessor), encoding="utf-8")
|
||||
|
||||
with pytest.raises(CheckpointValidationError, match="completed without required approval"):
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
"run",
|
||||
"script",
|
||||
"completed",
|
||||
{"script": _script_artifact()},
|
||||
pipeline_type="framework-smoke",
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
def test_malformed_predecessor_cannot_forge_completion(tmp_path) -> None:
|
||||
project_dir = init_project(
|
||||
"run",
|
||||
title="Run",
|
||||
pipeline_type="framework-smoke",
|
||||
pipeline_dir=tmp_path,
|
||||
)
|
||||
(project_dir / "checkpoint_research.json").write_text(
|
||||
json.dumps({"status": "completed", "human_approved": True}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(CheckpointValidationError, match="incomplete or missing"):
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
"run",
|
||||
"script",
|
||||
"completed",
|
||||
{"script": _script_artifact()},
|
||||
pipeline_type="framework-smoke",
|
||||
human_approved=True,
|
||||
)
|
||||
|
||||
|
||||
def test_in_progress_heartbeat_is_not_blocked_by_prerequisites(tmp_path) -> None:
|
||||
init_project(
|
||||
"run",
|
||||
title="Run",
|
||||
pipeline_type="framework-smoke",
|
||||
pipeline_dir=tmp_path,
|
||||
)
|
||||
|
||||
path = write_checkpoint(
|
||||
tmp_path,
|
||||
"run",
|
||||
"script",
|
||||
"in_progress",
|
||||
{},
|
||||
pipeline_type="framework-smoke",
|
||||
)
|
||||
|
||||
assert path.exists()
|
||||
|
||||
|
||||
def test_unknown_style_playbook_fails_before_project_creation(tmp_path) -> None:
|
||||
with pytest.raises(CheckpointValidationError, match="style_playbook"):
|
||||
init_project(
|
||||
"run",
|
||||
title="Run",
|
||||
pipeline_type="framework-smoke",
|
||||
pipeline_dir=tmp_path,
|
||||
style_playbook="does-not-exist",
|
||||
)
|
||||
|
||||
assert not (tmp_path / "run").exists()
|
||||
|
||||
|
||||
def test_marker_derived_unknown_playbook_blocks_later_writes(tmp_path) -> None:
|
||||
project_dir = tmp_path / "run"
|
||||
project_dir.mkdir()
|
||||
(project_dir / "project.json").write_text(
|
||||
json.dumps({
|
||||
"version": "1.0",
|
||||
"project_id": "run",
|
||||
"pipeline_type": "framework-smoke",
|
||||
"style_playbook": "does-not-exist",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(CheckpointValidationError, match="style_playbook"):
|
||||
write_checkpoint(tmp_path, "run", "research", "in_progress", {})
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
from lib.clip_embedder import _as_feature_tensor
|
||||
|
||||
|
||||
class _Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class _ModelOutput:
|
||||
def __init__(self, pooler_output):
|
||||
self.pooler_output = pooler_output
|
||||
self.last_hidden_state = object()
|
||||
|
||||
|
||||
def test_transformers_4_tensor_passes_through() -> None:
|
||||
tensor = _Tensor()
|
||||
assert _as_feature_tensor(tensor) is tensor
|
||||
|
||||
|
||||
def test_transformers_5_output_unwraps_projected_pooler_output() -> None:
|
||||
tensor = _Tensor()
|
||||
assert _as_feature_tensor(_ModelOutput(tensor)) is tensor
|
||||
|
||||
|
||||
def test_missing_pooler_output_does_not_replace_features_with_none() -> None:
|
||||
output = _ModelOutput(None)
|
||||
assert _as_feature_tensor(output) is output
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
import math
|
||||
import struct
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.audio.audio_mixer import AudioMixer
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
|
||||
reason="ffmpeg and ffprobe are required",
|
||||
)
|
||||
|
||||
|
||||
def _tone(path: Path, frequency: int, duration: float) -> None:
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-y", "-f", "lavfi", "-i",
|
||||
f"sine=frequency={frequency}:duration={duration}", str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def _duration(path: Path) -> float:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "csv=p=0", str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
return float(result.stdout.strip())
|
||||
|
||||
|
||||
def _tail_rms(path: Path, start_seconds: float, end_seconds: float) -> float:
|
||||
with wave.open(str(path), "rb") as handle:
|
||||
sample_width = handle.getsampwidth()
|
||||
assert sample_width == 2
|
||||
frame_rate = handle.getframerate()
|
||||
handle.setpos(int(start_seconds * frame_rate))
|
||||
raw = handle.readframes(int((end_seconds - start_seconds) * frame_rate))
|
||||
samples = struct.unpack(f"<{len(raw) // 2}h", raw)
|
||||
return math.sqrt(sum(sample * sample for sample in samples) / len(samples))
|
||||
|
||||
|
||||
def test_full_mix_can_pin_the_composition_length(tmp_path) -> None:
|
||||
speech = tmp_path / "speech.wav"
|
||||
music = tmp_path / "music.wav"
|
||||
output = tmp_path / "mix.wav"
|
||||
_tone(speech, 440, 1)
|
||||
_tone(music, 220, 3)
|
||||
|
||||
result = AudioMixer().execute({
|
||||
"operation": "full_mix",
|
||||
"tracks": [
|
||||
{"path": str(speech), "role": "speech"},
|
||||
{"path": str(music), "role": "music"},
|
||||
],
|
||||
"ducking": {"enabled": True},
|
||||
"normalize": False,
|
||||
"target_duration": 3,
|
||||
"output_path": str(output),
|
||||
})
|
||||
|
||||
assert result.success, result.error
|
||||
assert result.data["target_duration"] == 3
|
||||
assert _duration(output) == pytest.approx(3.0, abs=0.15)
|
||||
assert _tail_rms(output, 2.0, 2.5) > 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", [0, -1, "not-a-number"])
|
||||
def test_full_mix_rejects_invalid_target_duration(tmp_path, target) -> None:
|
||||
tone = tmp_path / "tone.wav"
|
||||
_tone(tone, 440, 0.25)
|
||||
|
||||
result = AudioMixer().execute({
|
||||
"operation": "full_mix",
|
||||
"tracks": [{"path": str(tone), "role": "speech"}],
|
||||
"target_duration": target,
|
||||
"output_path": str(tmp_path / "mix.wav"),
|
||||
})
|
||||
|
||||
assert result.success is False
|
||||
assert "target_duration" in result.error
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def test_bg_remove_selects_model_through_rembg_session(monkeypatch, tmp_path) -> None:
|
||||
fake_rembg = MagicMock()
|
||||
fake_rembg.new_session.return_value = "selected-session"
|
||||
fake_rembg.remove.side_effect = lambda image, **kwargs: image.convert("RGBA")
|
||||
monkeypatch.setitem(sys.modules, "rembg", fake_rembg)
|
||||
|
||||
input_path = tmp_path / "input.png"
|
||||
Image.new("RGB", (8, 8), (10, 20, 30)).save(input_path)
|
||||
|
||||
from tools.enhancement.bg_remove import BgRemove
|
||||
|
||||
result = BgRemove().execute({
|
||||
"input_path": str(input_path),
|
||||
"output_path": str(tmp_path / "output.png"),
|
||||
"model": "isnet-general-use",
|
||||
})
|
||||
|
||||
assert result.success, result.error
|
||||
fake_rembg.new_session.assert_called_once_with("isnet-general-use")
|
||||
kwargs = fake_rembg.remove.call_args.kwargs
|
||||
assert kwargs["session"] == "selected-session"
|
||||
assert "model_name" not in kwargs
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.video.video_compose import VideoCompose
|
||||
|
||||
|
||||
def test_cinematic_cut_adapter_builds_a_sequential_timeline() -> None:
|
||||
scenes = VideoCompose._cuts_to_cinematic_scenes([
|
||||
{
|
||||
"id": "v1",
|
||||
"source": "clip.mp4",
|
||||
"in_seconds": 2,
|
||||
"out_seconds": 6,
|
||||
"transition_in": "cut",
|
||||
"transition_out": "none",
|
||||
},
|
||||
{
|
||||
"id": "title",
|
||||
"source": "",
|
||||
"type": "hero_title",
|
||||
"text": "The signal arrives",
|
||||
"in_seconds": 0,
|
||||
"out_seconds": 3,
|
||||
},
|
||||
])
|
||||
|
||||
assert scenes[0] == {
|
||||
"id": "v1",
|
||||
"startSeconds": 0.0,
|
||||
"durationSeconds": 4.0,
|
||||
"kind": "video",
|
||||
"src": "clip.mp4",
|
||||
"trimBeforeSeconds": 2.0,
|
||||
"trimAfterSeconds": 6.0,
|
||||
"playbackRate": 1.0,
|
||||
"fadeInFrames": 0,
|
||||
"fadeOutFrames": 0,
|
||||
}
|
||||
assert scenes[1]["kind"] == "title"
|
||||
assert scenes[1]["startSeconds"] == 4.0
|
||||
assert scenes[1]["text"] == "The signal arrives"
|
||||
|
||||
|
||||
def test_cinematic_cut_adapter_preserves_playback_speed() -> None:
|
||||
scenes = VideoCompose._cuts_to_cinematic_scenes([
|
||||
{
|
||||
"id": "fast",
|
||||
"source": "clip.mp4",
|
||||
"in_seconds": 2,
|
||||
"out_seconds": 6,
|
||||
"speed": 2,
|
||||
}
|
||||
])
|
||||
|
||||
assert scenes[0]["durationSeconds"] == 2
|
||||
assert scenes[0]["playbackRate"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uri_style", ["standard", "legacy_windows"])
|
||||
def test_remotion_media_staging_decodes_file_uris(tmp_path, uri_style) -> None:
|
||||
source = tmp_path / "clip with space.mp4"
|
||||
source.write_bytes(b"video")
|
||||
public_dir = tmp_path / "public"
|
||||
uri = source.as_uri()
|
||||
if uri_style == "legacy_windows" and len(source.drive) == 2:
|
||||
uri = f"file://{source.drive}{source.as_posix()[2:]}".replace(" ", "%20")
|
||||
props = {"scenes": [{"src": uri}]}
|
||||
|
||||
staged_count = VideoCompose._stage_remotion_media(props, public_dir)
|
||||
|
||||
assert staged_count == 1
|
||||
assert props["scenes"][0]["src"] != uri
|
||||
assert (public_dir / props["scenes"][0]["src"]).read_bytes() == b"video"
|
||||
|
||||
|
||||
def test_remotion_render_adapts_cuts_and_stages_local_video(monkeypatch, tmp_path) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"not-a-real-video")
|
||||
output = tmp_path / "render.mp4"
|
||||
captured = {}
|
||||
|
||||
def fake_run_command(self, command, **kwargs):
|
||||
captured["command"] = command
|
||||
captured["timeout"] = kwargs["timeout"]
|
||||
props_arg = next(arg for arg in command if arg.startswith("--props="))
|
||||
captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text())
|
||||
public_arg = next(arg for arg in command if arg.startswith("--public-dir="))
|
||||
public_dir = Path(public_arg.split("=", 1)[1])
|
||||
captured["staged_exists_during_render"] = (
|
||||
public_dir / captured["props"]["scenes"][0]["src"]
|
||||
).exists()
|
||||
output.write_bytes(b"rendered")
|
||||
|
||||
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
|
||||
|
||||
result = VideoCompose()._remotion_render({
|
||||
"edit_decisions": {
|
||||
"renderer_family": "cinematic-trailer",
|
||||
"cuts": [
|
||||
{
|
||||
"id": "v1",
|
||||
"source": str(source),
|
||||
"in_seconds": 0,
|
||||
"out_seconds": 2,
|
||||
}
|
||||
],
|
||||
},
|
||||
"output_path": str(output),
|
||||
})
|
||||
|
||||
assert result.success, result.error
|
||||
assert "cuts" not in captured["props"]
|
||||
assert captured["props"]["scenes"][0]["kind"] == "video"
|
||||
assert captured["staged_exists_during_render"] is True
|
||||
assert result.data["staged_media_count"] == 1
|
||||
|
||||
|
||||
def test_remotion_timeout_scales_with_scene_count(monkeypatch, tmp_path) -> None:
|
||||
output = tmp_path / "render.mp4"
|
||||
captured = {}
|
||||
|
||||
def fake_run_command(self, command, **kwargs):
|
||||
captured["timeout"] = kwargs["timeout"]
|
||||
output.write_bytes(b"rendered")
|
||||
|
||||
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
|
||||
cuts = [
|
||||
{
|
||||
"id": f"title-{index}",
|
||||
"source": "",
|
||||
"type": "hero_title",
|
||||
"text": str(index),
|
||||
"in_seconds": 0,
|
||||
"out_seconds": 1,
|
||||
}
|
||||
for index in range(50)
|
||||
]
|
||||
|
||||
result = VideoCompose()._remotion_render({
|
||||
"edit_decisions": {"renderer_family": "cinematic-trailer", "cuts": cuts},
|
||||
"output_path": str(output),
|
||||
})
|
||||
|
||||
assert result.success, result.error
|
||||
assert captured["timeout"] == 750
|
||||
|
||||
|
||||
def test_remotion_render_preserves_direct_cinematic_scenes(monkeypatch, tmp_path) -> None:
|
||||
output = tmp_path / "render.mp4"
|
||||
captured = {}
|
||||
|
||||
def fake_run_command(self, command, **kwargs):
|
||||
props_arg = next(arg for arg in command if arg.startswith("--props="))
|
||||
captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text())
|
||||
output.write_bytes(b"rendered")
|
||||
|
||||
monkeypatch.setattr(VideoCompose, "run_command", fake_run_command)
|
||||
scene = {
|
||||
"id": "authored",
|
||||
"kind": "title",
|
||||
"text": "Keep me",
|
||||
"startSeconds": 0,
|
||||
"durationSeconds": 1,
|
||||
}
|
||||
|
||||
result = VideoCompose()._remotion_render({
|
||||
"composition_data": {
|
||||
"renderer_family": "cinematic-trailer",
|
||||
"scenes": [scene],
|
||||
},
|
||||
"output_path": str(output),
|
||||
})
|
||||
|
||||
assert result.success, result.error
|
||||
assert captured["props"]["scenes"] == [scene]
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.video.stock_sources as stock_sources
|
||||
from tools.video.corpus_builder import CorpusBuilder
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Candidate:
|
||||
clip_id: str
|
||||
|
||||
|
||||
class _Source:
|
||||
name = "fake"
|
||||
|
||||
def __init__(self, count: int) -> None:
|
||||
self.count = count
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def search(self, query, filters):
|
||||
return [_Candidate(f"clip-{index}") for index in range(self.count)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_builder(monkeypatch, tmp_path):
|
||||
def run(count: int, processor):
|
||||
monkeypatch.setattr(stock_sources, "available_sources", lambda: [_Source(count)])
|
||||
monkeypatch.setattr(stock_sources, "source_summary", lambda: {})
|
||||
monkeypatch.setattr(CorpusBuilder, "_process_candidate", processor)
|
||||
return CorpusBuilder().execute({
|
||||
"corpus_dir": str(tmp_path / f"corpus-{count}"),
|
||||
"queries": [{"query": "city at night"}],
|
||||
"max_new_clips": 50,
|
||||
})
|
||||
|
||||
return run
|
||||
|
||||
|
||||
def test_all_candidate_failures_fail_closed_with_diagnostics(run_builder) -> None:
|
||||
def broken_clip_stack(*args, **kwargs):
|
||||
raise AttributeError("BaseModelOutput has no attribute norm")
|
||||
|
||||
result = run_builder(4, broken_clip_stack)
|
||||
|
||||
assert result.success is False
|
||||
assert result.data["candidates_seen"] == 4
|
||||
assert result.data["clips_failed"] == 4
|
||||
assert "corpus index is empty" in result.error
|
||||
assert "BaseModelOutput" in result.error
|
||||
|
||||
|
||||
def test_no_candidates_is_a_valid_empty_search(run_builder) -> None:
|
||||
result = run_builder(0, lambda *args, **kwargs: None)
|
||||
|
||||
assert result.success is True
|
||||
assert result.data["candidates_seen"] == 0
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_blank_google_location_uses_documented_default(monkeypatch) -> None:
|
||||
monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "")
|
||||
from tools.google_credentials import resolve_google_location
|
||||
|
||||
assert resolve_google_location() == "us-central1"
|
||||
|
||||
|
||||
def test_google_music_requests_the_global_vertex_location(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setenv("GOOGLE_API_KEY", "test-key")
|
||||
import tools.google_credentials as credentials
|
||||
|
||||
captured = {}
|
||||
|
||||
def stop_before_network(http_options=None, location=None):
|
||||
captured["location"] = location
|
||||
raise RuntimeError("stop before network")
|
||||
|
||||
monkeypatch.setattr(credentials, "get_genai_client", stop_before_network)
|
||||
|
||||
from tools.audio.google_music import GoogleMusic
|
||||
|
||||
result = GoogleMusic().execute({
|
||||
"prompt": "solo piano",
|
||||
"output_path": str(tmp_path / "music.mp3"),
|
||||
})
|
||||
|
||||
assert result.success is False
|
||||
assert captured["location"] == "global"
|
||||
|
||||
|
||||
class _VideoAsset:
|
||||
uri = None
|
||||
|
||||
def __init__(self, video_bytes):
|
||||
self.video_bytes = video_bytes
|
||||
|
||||
def save(self, path):
|
||||
Path(path).write_bytes(self.video_bytes or b"")
|
||||
|
||||
|
||||
class _Models:
|
||||
def __init__(self, video_bytes):
|
||||
self.video_bytes = video_bytes
|
||||
|
||||
def generate_videos(self, **kwargs):
|
||||
asset = _VideoAsset(self.video_bytes)
|
||||
generated = SimpleNamespace(video=asset)
|
||||
response = SimpleNamespace(generated_videos=[generated])
|
||||
return SimpleNamespace(done=True, error=None, response=response)
|
||||
|
||||
|
||||
class _VertexClient:
|
||||
vertexai = True
|
||||
|
||||
def __init__(self, video_bytes):
|
||||
self.models = _Models(video_bytes)
|
||||
self.files = SimpleNamespace(
|
||||
download=lambda **kwargs: pytest.fail("Vertex must not call files.download")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("video_bytes", "expected_success"),
|
||||
[(b"VIDEO_BYTES", True), (None, False)],
|
||||
)
|
||||
def test_veo_accepts_vertex_and_requires_inline_bytes(
|
||||
monkeypatch, tmp_path, video_bytes, expected_success
|
||||
) -> None:
|
||||
monkeypatch.setenv("GOOGLE_API_KEY", "test-key")
|
||||
import tools.google_credentials as credentials
|
||||
|
||||
monkeypatch.setattr(
|
||||
credentials,
|
||||
"get_genai_client",
|
||||
lambda http_options=None, location=None: _VertexClient(video_bytes),
|
||||
)
|
||||
|
||||
from tools.video.veo_video import VeoVideo
|
||||
|
||||
output = tmp_path / "video.mp4"
|
||||
result = VeoVideo().execute({
|
||||
"backend": "google",
|
||||
"prompt": "wind moving across grassland",
|
||||
"output_path": str(output),
|
||||
})
|
||||
|
||||
assert result.success is expected_success
|
||||
if expected_success:
|
||||
assert output.read_bytes() == b"VIDEO_BYTES"
|
||||
else:
|
||||
assert "without inline bytes" in result.error
|
||||
|
|
@ -250,6 +250,11 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch):
|
|||
"_resolve_npm_package",
|
||||
classmethod(lambda cls: {"version": "0.4.5"}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
HyperFramesCompose,
|
||||
"_probe_cli",
|
||||
classmethod(lambda cls: {"status": "ok"}),
|
||||
)
|
||||
rc = HyperFramesCompose()._runtime_check()
|
||||
# Local binaries must still pass for this to go green.
|
||||
if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]:
|
||||
|
|
@ -259,6 +264,31 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch):
|
|||
assert rc["reasons"] == []
|
||||
|
||||
|
||||
def test_runtime_check_fails_when_published_cli_crashes(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
HyperFramesCompose,
|
||||
"_resolve_npm_package",
|
||||
classmethod(lambda cls: {"version": "0.7.89"}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
HyperFramesCompose,
|
||||
"_probe_cli",
|
||||
classmethod(
|
||||
lambda cls: {
|
||||
"error": 'doctor failed: The "file" argument must be of type string'
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
rc = HyperFramesCompose()._runtime_check()
|
||||
|
||||
if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]:
|
||||
pytest.skip("Local runtime floor not met on this machine")
|
||||
assert rc["runtime_available"] is False
|
||||
assert rc["cli_probe_error"] is not None
|
||||
assert any("not executable" in reason for reason in rc["reasons"])
|
||||
|
||||
|
||||
def test_video_compose_render_engines_follow_hyperframes_runtime_check(monkeypatch):
|
||||
"""Regression: `video_compose.get_info()['render_engines']['hyperframes']`
|
||||
must track the true availability, not just the local-binary floor.
|
||||
|
|
|
|||
|
|
@ -213,8 +213,9 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch):
|
|||
|
||||
# Build a fake RealESRGANer whose __init__ DOES accept device=
|
||||
class FakeRealESRGANer:
|
||||
def __init__(self, *, scale, model_path, model, dni_weight, half, device=None):
|
||||
def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10, device=None):
|
||||
self.called_with_device = device
|
||||
self.called_with_tile = tile
|
||||
fake_realesrganer_cls = FakeRealESRGANer
|
||||
|
||||
monkeypatch.setitem(sys.modules, "torch", fake_torch)
|
||||
|
|
@ -239,6 +240,7 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch):
|
|||
tool = upscale.Upscale()
|
||||
result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False)
|
||||
assert result.called_with_device == "device(mps)"
|
||||
assert result.called_with_tile == 256
|
||||
|
||||
|
||||
def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
|
||||
|
|
@ -252,8 +254,9 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
|
|||
|
||||
# Build a fake RealESRGANer whose __init__ does NOT accept device=
|
||||
class FakeRealESRGANerNoDevice:
|
||||
def __init__(self, *, scale, model_path, model, dni_weight, half):
|
||||
def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10):
|
||||
self.called_with_device = None # no device param
|
||||
self.called_with_tile = tile
|
||||
fake_realesrganer_cls = FakeRealESRGANerNoDevice
|
||||
|
||||
monkeypatch.setitem(sys.modules, "torch", fake_torch)
|
||||
|
|
@ -279,6 +282,7 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch):
|
|||
# Should NOT raise TypeError about unexpected keyword argument 'device'
|
||||
result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False)
|
||||
assert result.called_with_device is None
|
||||
assert result.called_with_tile == 256
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.video.video_compose import VideoCompose
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
|
||||
reason="ffmpeg and ffprobe are required",
|
||||
)
|
||||
|
||||
|
||||
def _run(command: list[str]) -> None:
|
||||
subprocess.run(command, capture_output=True, check=True, timeout=30)
|
||||
|
||||
|
||||
def test_external_audio_mux_adds_audible_stream_without_changing_video_length(tmp_path) -> None:
|
||||
video = tmp_path / "video.mp4"
|
||||
audio = tmp_path / "audio.wav"
|
||||
_run([
|
||||
"ffmpeg", "-y", "-f", "lavfi", "-i",
|
||||
"color=c=red:s=320x180:d=2:r=30",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(video),
|
||||
])
|
||||
_run([
|
||||
"ffmpeg", "-y", "-f", "lavfi", "-i",
|
||||
"sine=frequency=440:duration=1", str(audio),
|
||||
])
|
||||
|
||||
result = VideoCompose()._mux_external_audio(video, audio)
|
||||
|
||||
assert result.success, result.error
|
||||
streams = subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error", "-show_entries",
|
||||
"stream=codec_type", "-of", "csv=p=0", str(video),
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
).stdout.splitlines()
|
||||
duration = float(subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "csv=p=0", str(video),
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
).stdout.strip())
|
||||
|
||||
assert "video" in streams
|
||||
assert "audio" in streams
|
||||
assert duration == pytest.approx(2.0, abs=0.15)
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tools.analysis.transcriber import Transcriber
|
||||
|
||||
|
||||
class _Info:
|
||||
language = "en"
|
||||
duration = 1.0
|
||||
|
||||
|
||||
def test_transcriber_uses_ctranslate2_cuda_without_torch(monkeypatch, tmp_path) -> None:
|
||||
devices = []
|
||||
|
||||
class FakeWhisperModel:
|
||||
def __init__(self, model_size, *, device, compute_type):
|
||||
devices.append((device, compute_type))
|
||||
|
||||
def transcribe(self, *args, **kwargs):
|
||||
return iter(()), _Info()
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"faster_whisper",
|
||||
SimpleNamespace(WhisperModel=FakeWhisperModel),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"ctranslate2",
|
||||
SimpleNamespace(
|
||||
get_cuda_device_count=lambda: 1,
|
||||
get_supported_compute_types=lambda device: {"float16", "float32"},
|
||||
),
|
||||
)
|
||||
input_path = tmp_path / "audio.wav"
|
||||
input_path.write_bytes(b"fake")
|
||||
|
||||
result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)})
|
||||
|
||||
assert result.success, result.error
|
||||
assert devices == [("cuda", "float16")]
|
||||
assert result.data["device"] == "cuda"
|
||||
|
||||
|
||||
def test_transcriber_falls_back_when_cuda_fails_during_iteration(monkeypatch, tmp_path) -> None:
|
||||
devices = []
|
||||
|
||||
class FakeWhisperModel:
|
||||
def __init__(self, model_size, *, device, compute_type):
|
||||
self.device = device
|
||||
devices.append((device, compute_type))
|
||||
|
||||
def transcribe(self, *args, **kwargs):
|
||||
if self.device == "cuda":
|
||||
def broken_iterator():
|
||||
raise RuntimeError("cublas64_12.dll not found")
|
||||
yield
|
||||
|
||||
return broken_iterator(), _Info()
|
||||
return iter(()), _Info()
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"faster_whisper",
|
||||
SimpleNamespace(WhisperModel=FakeWhisperModel),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"ctranslate2",
|
||||
SimpleNamespace(
|
||||
get_cuda_device_count=lambda: 1,
|
||||
get_supported_compute_types=lambda device: {"float16"},
|
||||
),
|
||||
)
|
||||
input_path = tmp_path / "audio.wav"
|
||||
input_path.write_bytes(b"fake")
|
||||
|
||||
result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)})
|
||||
|
||||
assert result.success, result.error
|
||||
assert devices == [("cuda", "float16"), ("cpu", "int8")]
|
||||
assert result.data["device"] == "cpu"
|
||||
assert "cublas64_12.dll" in result.data["gpu_fallback_reason"]
|
||||
|
|
@ -135,50 +135,76 @@ class Transcriber(BaseTool):
|
|||
|
||||
start = time.time()
|
||||
|
||||
# Load model (CPU by default, CUDA if available)
|
||||
# faster-whisper executes through CTranslate2, so that runtime—not
|
||||
# PyTorch—is authoritative for CUDA availability and compute types.
|
||||
device = "cpu"
|
||||
compute_type = "int8"
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
except ImportError:
|
||||
import ctranslate2
|
||||
|
||||
if ctranslate2.get_cuda_device_count() > 0:
|
||||
supported = ctranslate2.get_supported_compute_types("cuda")
|
||||
for candidate in ("float16", "int8_float16", "float32"):
|
||||
if candidate in supported:
|
||||
device = "cuda"
|
||||
compute_type = candidate
|
||||
break
|
||||
except Exception:
|
||||
# Probing is advisory. CPU remains a safe deterministic baseline.
|
||||
pass
|
||||
|
||||
def _transcribe_on(selected_device: str, selected_compute_type: str):
|
||||
model = WhisperModel(
|
||||
model_size,
|
||||
device=selected_device,
|
||||
compute_type=selected_compute_type,
|
||||
)
|
||||
segments_iter, transcription_info = model.transcribe(
|
||||
str(input_path),
|
||||
language=language,
|
||||
word_timestamps=True,
|
||||
vad_filter=True,
|
||||
)
|
||||
|
||||
parsed_segments = []
|
||||
parsed_words = []
|
||||
# faster-whisper evaluates lazily. Draining the iterator here keeps
|
||||
# missing CUDA runtime libraries inside the fallback boundary.
|
||||
for seg in segments_iter:
|
||||
seg_data = {
|
||||
"id": seg.id,
|
||||
"start": round(seg.start, 3),
|
||||
"end": round(seg.end, 3),
|
||||
"text": seg.text.strip(),
|
||||
}
|
||||
|
||||
if seg.words:
|
||||
words = []
|
||||
for word in seg.words:
|
||||
word_entry = {
|
||||
"word": word.word,
|
||||
"start": round(word.start, 3),
|
||||
"end": round(word.end, 3),
|
||||
"probability": round(word.probability, 3),
|
||||
}
|
||||
words.append(word_entry)
|
||||
parsed_words.append(word_entry)
|
||||
seg_data["words"] = words
|
||||
|
||||
parsed_segments.append(seg_data)
|
||||
|
||||
return parsed_segments, parsed_words, transcription_info
|
||||
|
||||
gpu_fallback_reason = None
|
||||
try:
|
||||
segments, word_timestamps, info = _transcribe_on(device, compute_type)
|
||||
except Exception as exc:
|
||||
if device == "cpu":
|
||||
raise
|
||||
gpu_fallback_reason = f"{type(exc).__name__}: {exc}"
|
||||
device = "cpu"
|
||||
compute_type = "int8"
|
||||
|
||||
model = WhisperModel(model_size, device=device, compute_type=compute_type)
|
||||
|
||||
# Transcribe
|
||||
segments_iter, info = model.transcribe(
|
||||
str(input_path),
|
||||
language=language,
|
||||
word_timestamps=True,
|
||||
vad_filter=True,
|
||||
)
|
||||
|
||||
segments = []
|
||||
word_timestamps = []
|
||||
|
||||
for seg in segments_iter:
|
||||
seg_data = {
|
||||
"id": seg.id,
|
||||
"start": round(seg.start, 3),
|
||||
"end": round(seg.end, 3),
|
||||
"text": seg.text.strip(),
|
||||
}
|
||||
|
||||
if seg.words:
|
||||
words = []
|
||||
for w in seg.words:
|
||||
word_entry = {
|
||||
"word": w.word,
|
||||
"start": round(w.start, 3),
|
||||
"end": round(w.end, 3),
|
||||
"probability": round(w.probability, 3),
|
||||
}
|
||||
words.append(word_entry)
|
||||
word_timestamps.append(word_entry)
|
||||
seg_data["words"] = words
|
||||
|
||||
segments.append(seg_data)
|
||||
segments, word_timestamps, info = _transcribe_on(device, compute_type)
|
||||
|
||||
detected_language = language or info.language
|
||||
duration = info.duration
|
||||
|
|
@ -198,6 +224,8 @@ class Transcriber(BaseTool):
|
|||
"duration_seconds": round(duration, 3),
|
||||
"model_size": model_size,
|
||||
"device": device,
|
||||
"compute_type": compute_type,
|
||||
"gpu_fallback_reason": gpu_fallback_reason,
|
||||
}
|
||||
|
||||
# Write transcript JSON
|
||||
|
|
|
|||
|
|
@ -184,6 +184,14 @@ class AudioMixer(BaseTool):
|
|||
"default": 0.5,
|
||||
"description": "Duration of fade in/out at segment boundaries (seconds).",
|
||||
},
|
||||
"target_duration": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"description": (
|
||||
"full_mix only. Exact output length in seconds. Pads a short "
|
||||
"mix and trims a long mix so audio matches the composition."
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -500,6 +508,15 @@ class AudioMixer(BaseTool):
|
|||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
normalize = inputs.get("normalize", True)
|
||||
ducking = inputs.get("ducking", {"enabled": True})
|
||||
target_duration = inputs.get("target_duration")
|
||||
target: float | None = None
|
||||
if target_duration is not None:
|
||||
try:
|
||||
target = float(target_duration)
|
||||
except (TypeError, ValueError):
|
||||
return ToolResult(success=False, error="target_duration must be a positive number")
|
||||
if target <= 0:
|
||||
return ToolResult(success=False, error="target_duration must be greater than zero")
|
||||
|
||||
speech_tracks = [t for t in tracks if t.get("role") in ("speech", "primary")]
|
||||
music_tracks = [t for t in tracks if t.get("role") in ("music", "secondary")]
|
||||
|
|
@ -547,7 +564,14 @@ class AudioMixer(BaseTool):
|
|||
)
|
||||
else:
|
||||
filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_all]")
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
|
||||
if target is not None:
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key_raw][speech_out]")
|
||||
filter_parts.append(
|
||||
f"[speech_key_raw]apad=whole_dur={target},"
|
||||
f"atrim=duration={target},asetpts=PTS-STARTPTS[speech_key]"
|
||||
)
|
||||
else:
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
|
||||
|
||||
# Mix music tracks together
|
||||
music_start = len(speech_tracks)
|
||||
|
|
@ -596,19 +620,34 @@ class AudioMixer(BaseTool):
|
|||
f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]"
|
||||
)
|
||||
|
||||
# A ducked music stream is gated by the speech sidechain, so its tail
|
||||
# can disappear when narration ends. If the caller knows the video
|
||||
# duration, make that the authoritative mix length before loudness
|
||||
# normalization: apad extends short audio and atrim caps long audio.
|
||||
premix_label = "premix"
|
||||
if target is not None:
|
||||
filter_parts.append(
|
||||
f"[premix]apad=whole_dur={target},atrim=duration={target},"
|
||||
"asetpts=PTS-STARTPTS[premix_duration]"
|
||||
)
|
||||
premix_label = "premix_duration"
|
||||
|
||||
# Normalize
|
||||
if normalize:
|
||||
filter_parts.append(self._loudnorm_filter(inputs, "premix", "out"))
|
||||
filter_parts.append(self._loudnorm_filter(inputs, premix_label, "out"))
|
||||
out_label = "[out]"
|
||||
else:
|
||||
out_label = "[premix]"
|
||||
out_label = f"[{premix_label}]"
|
||||
|
||||
filter_complex = ";".join(p for p in filter_parts if p)
|
||||
|
||||
cmd = ["ffmpeg", "-y"]
|
||||
cmd.extend(input_args)
|
||||
cmd.extend(["-filter_complex", filter_complex])
|
||||
cmd.extend(["-map", out_label, str(output_path)])
|
||||
cmd.extend(["-map", out_label])
|
||||
if target is not None:
|
||||
cmd.extend(["-t", str(target)])
|
||||
cmd.append(str(output_path))
|
||||
|
||||
self.run_command(cmd)
|
||||
|
||||
|
|
@ -621,6 +660,7 @@ class AudioMixer(BaseTool):
|
|||
"sfx_tracks": len(sfx_tracks),
|
||||
"ducking_enabled": duck_enabled,
|
||||
"normalized": normalize,
|
||||
"target_duration": target_duration,
|
||||
"output": str(output_path),
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
|
|
|
|||
|
|
@ -146,7 +146,8 @@ class GoogleMusic(BaseTool):
|
|||
from tools.google_credentials import get_genai_client, GOOGLE_API_TIMEOUT_MS
|
||||
|
||||
http_options = types.HttpOptions(timeout=GOOGLE_API_TIMEOUT_MS)
|
||||
client = get_genai_client(http_options=http_options)
|
||||
# Lyria 3 is served only from Vertex's global location.
|
||||
client = get_genai_client(http_options=http_options, location="global")
|
||||
except ImportError as e:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ class BgRemove(BaseTool):
|
|||
|
||||
result_image = rembg.remove(
|
||||
input_image,
|
||||
model_name=model_name,
|
||||
session=rembg.new_session(model_name),
|
||||
alpha_matting=alpha_matting,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -297,6 +297,11 @@ class Upscale(BaseTool):
|
|||
"model": model,
|
||||
"dni_weight": denoise_strength,
|
||||
"half": half,
|
||||
# Full-frame x4 inference can terminate the process on low-memory
|
||||
# CPU/MPS hosts before Python can raise an exception. Bound the
|
||||
# working set there; keep CUDA on the faster single-pass path.
|
||||
"tile": 0 if _device == "cuda" else 256,
|
||||
"tile_pad": 10,
|
||||
}
|
||||
# Guard: only pass device= if the installed version accepts it
|
||||
if "device" in inspect.signature(RealESRGANer.__init__).parameters:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ from typing import Any
|
|||
# Broad scope that covers Cloud Text-to-Speech and Vertex AI prediction.
|
||||
CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"
|
||||
|
||||
|
||||
def resolve_google_location(location: str | None = None) -> str:
|
||||
"""Return a Vertex location, treating blank env values as unset."""
|
||||
|
||||
return location or os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
|
||||
|
||||
# Shared constants for long-running Google/Vertex AI generation calls (e.g. music, video)
|
||||
GOOGLE_API_TIMEOUT_SECONDS = 600
|
||||
GOOGLE_API_TIMEOUT_MS = GOOGLE_API_TIMEOUT_SECONDS * 1000
|
||||
|
|
@ -38,8 +44,15 @@ def has_google_credentials() -> bool:
|
|||
)
|
||||
|
||||
|
||||
def get_genai_client(http_options: Any | None = None) -> Any:
|
||||
"""Lazily import and initialize the Google GenAI Client based on configured credentials."""
|
||||
def get_genai_client(
|
||||
http_options: Any | None = None,
|
||||
location: str | None = None,
|
||||
) -> Any:
|
||||
"""Initialize Google GenAI using the configured credential mode.
|
||||
|
||||
``location`` overrides the Vertex region for globally hosted models. It is
|
||||
deliberately ignored by the API-key backend, which has no region setting.
|
||||
"""
|
||||
from google import genai
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")
|
||||
|
|
@ -51,7 +64,7 @@ def get_genai_client(http_options: Any | None = None) -> Any:
|
|||
if use_vertex or (not api_key and service_account_configured()):
|
||||
kwargs = {
|
||||
"vertexai": True,
|
||||
"location": os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1"),
|
||||
"location": resolve_google_location(location),
|
||||
"http_options": http_options,
|
||||
}
|
||||
project_id = resolve_project_id()
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from tools.base_tool import (
|
|||
)
|
||||
from tools.google_credentials import (
|
||||
get_access_token,
|
||||
resolve_google_location,
|
||||
resolve_project_id,
|
||||
service_account_configured,
|
||||
has_google_credentials,
|
||||
|
|
@ -241,7 +242,7 @@ class GoogleImagen(BaseTool):
|
|||
}
|
||||
|
||||
if bearer_token:
|
||||
location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1")
|
||||
location = resolve_google_location()
|
||||
url = (
|
||||
f"https://{location}-aiplatform.googleapis.com/v1/projects/"
|
||||
f"{project_id}/locations/{location}/publishers/google/models/"
|
||||
|
|
|
|||
|
|
@ -392,6 +392,38 @@ class CorpusBuilder(BaseTool):
|
|||
except Exception as e:
|
||||
cache_snapshot = {"error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Per-candidate tolerance is useful only while at least one item
|
||||
# survives. If every discovered candidate fails, reporting success
|
||||
# persists an empty index and hides a systemic codec/CLIP failure
|
||||
# until retrieval. A no-result or skip-only run remains valid.
|
||||
total_failure = bool(candidates_seen) and not added_ids and not skipped
|
||||
if total_failure:
|
||||
first_errors = "; ".join(
|
||||
item["error"]
|
||||
for item in errors
|
||||
if item.get("phase") == "process"
|
||||
)[:400]
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
f"All {failed} of {candidates_seen} candidates failed to "
|
||||
"process; corpus index is empty. Check the media decoder "
|
||||
"and the CLIP `transformers`/`torch` compatibility. "
|
||||
f"First errors: {first_errors or '(none recorded)'}"
|
||||
),
|
||||
data={
|
||||
"corpus_dir": str(corpus_dir),
|
||||
"queries_run": len(queries),
|
||||
"candidates_seen": candidates_seen,
|
||||
"clips_added": 0,
|
||||
"clips_skipped_existing": skipped,
|
||||
"clips_failed": failed,
|
||||
"total_corpus_size": len(corp),
|
||||
"errors": errors[:25],
|
||||
},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ class HyperFramesCompose(BaseTool):
|
|||
# We cache per-process so the first call pays ~2-5s and subsequent calls
|
||||
# (get_info spam from the registry) are free.
|
||||
_npm_resolve_cache: Optional[dict[str, str]] = None
|
||||
_cli_probe_cache: Optional[dict[str, str]] = None
|
||||
|
||||
@classmethod
|
||||
def _node_major_version(cls) -> Optional[int]:
|
||||
|
|
@ -301,6 +302,45 @@ class HyperFramesCompose(BaseTool):
|
|||
cls._npm_resolve_cache = {"version": version}
|
||||
return cls._npm_resolve_cache
|
||||
|
||||
@classmethod
|
||||
def _probe_cli(cls) -> dict[str, str]:
|
||||
"""Run the published CLI's doctor command once per process.
|
||||
|
||||
Package resolution alone does not prove that the executable can start:
|
||||
an upstream packaging regression can publish successfully while every
|
||||
CLI command crashes during bootstrap. Provider preflight must not call
|
||||
that state available.
|
||||
"""
|
||||
if cls._cli_probe_cache is not None:
|
||||
return cls._cli_probe_cache
|
||||
|
||||
npx = shutil.which("npx")
|
||||
if not npx:
|
||||
cls._cli_probe_cache = {"error": "npx not on PATH"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[npx, "--yes", cls._NPM_PACKAGE, "doctor", "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls._cli_probe_cache = {"error": "doctor timed out after 20s"}
|
||||
return cls._cli_probe_cache
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
cls._cli_probe_cache = {"error": f"doctor failed: {type(exc).__name__}"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
if proc.returncode != 0:
|
||||
output = "\n".join(filter(None, [proc.stderr, proc.stdout])).strip()
|
||||
tail = output.splitlines()[-1][:200] if output else f"exit {proc.returncode}"
|
||||
cls._cli_probe_cache = {"error": f"doctor failed: {tail}"}
|
||||
else:
|
||||
cls._cli_probe_cache = {"status": "ok"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
def _runtime_check(self) -> dict[str, Any]:
|
||||
"""Return availability state for the HyperFrames runtime.
|
||||
|
||||
|
|
@ -336,6 +376,12 @@ class HyperFramesCompose(BaseTool):
|
|||
f"{npm_resolve['error']}"
|
||||
)
|
||||
|
||||
cli_probe: dict[str, str] = {}
|
||||
if not reasons:
|
||||
cli_probe = self._probe_cli()
|
||||
if "error" in cli_probe:
|
||||
reasons.append(f"published CLI is not executable: {cli_probe['error']}")
|
||||
|
||||
return {
|
||||
"runtime_available": not reasons,
|
||||
"node_major": node_major,
|
||||
|
|
@ -344,6 +390,8 @@ class HyperFramesCompose(BaseTool):
|
|||
"npm_package": self._NPM_PACKAGE,
|
||||
"npm_package_version": npm_resolve.get("version"),
|
||||
"npm_resolve_error": npm_resolve.get("error"),
|
||||
"cli_probe_status": cli_probe.get("status"),
|
||||
"cli_probe_error": cli_probe.get("error"),
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -308,13 +308,6 @@ class VeoVideo(BaseTool):
|
|||
client._api_client, "vertexai", False
|
||||
)
|
||||
|
||||
if is_vertex:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="Google Veo video generation via google-genai is only supported using the Gemini Developer API (API key) backend. "
|
||||
"Please configure GEMINI_API_KEY/GOOGLE_API_KEY or use the FAL.ai backend.",
|
||||
)
|
||||
|
||||
prompt = inputs["prompt"]
|
||||
operation = inputs.get("operation", "text_to_video")
|
||||
model_variant = inputs.get("model_variant", "veo3.1")
|
||||
|
|
@ -501,7 +494,19 @@ class VeoVideo(BaseTool):
|
|||
success=False,
|
||||
error="No video asset returned in the response.",
|
||||
)
|
||||
client.files.download(file=video_asset)
|
||||
if not is_vertex:
|
||||
# The Files service is a Gemini Developer API feature. Vertex
|
||||
# returns bytes inline when no GCS output URI is requested.
|
||||
client.files.download(file=video_asset)
|
||||
elif not getattr(video_asset, "video_bytes", None):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
"Vertex AI returned a video without inline bytes "
|
||||
f"(uri={getattr(video_asset, 'uri', None)!r}). Configure "
|
||||
"the request without an output GCS URI so bytes are returned inline."
|
||||
),
|
||||
)
|
||||
|
||||
output_path = Path(inputs.get("output_path", "veo_output.mp4"))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
|
|||
|
|
@ -31,11 +31,14 @@ the agent to re-ask the user rather than substituting a different engine.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
|
|
@ -387,6 +390,49 @@ class VideoCompose(BaseTool):
|
|||
except Exception:
|
||||
return False
|
||||
|
||||
def _mux_external_audio(self, video_path: Path, audio_path: str | Path) -> ToolResult:
|
||||
"""Atomically replace a rendered video's audio with the approved mix."""
|
||||
|
||||
audio = Path(audio_path).resolve()
|
||||
if not audio.is_file():
|
||||
return ToolResult(success=False, error=f"Mixed audio not found: {audio}")
|
||||
|
||||
temp_output = video_path.with_name(
|
||||
f".{video_path.stem}.audio-mux-{time.time_ns()}{video_path.suffix}"
|
||||
)
|
||||
try:
|
||||
self.run_command([
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(video_path),
|
||||
"-i", str(audio),
|
||||
"-map", "0:v:0",
|
||||
"-map", "1:a:0",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "192k",
|
||||
"-af", "apad",
|
||||
"-shortest",
|
||||
"-movflags", "+faststart",
|
||||
str(temp_output),
|
||||
])
|
||||
if not temp_output.is_file():
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Audio mux completed but output file is missing: {temp_output}",
|
||||
)
|
||||
temp_output.replace(video_path)
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, error=f"Could not mux mixed audio: {exc}")
|
||||
finally:
|
||||
if temp_output.exists():
|
||||
temp_output.unlink()
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={"output": str(video_path), "has_mixed_audio": True},
|
||||
artifacts=[str(video_path)],
|
||||
)
|
||||
|
||||
def _compose(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
"""FFmpeg composition: concat video cuts, add audio, burn subtitles.
|
||||
|
||||
|
|
@ -714,6 +760,124 @@ class VideoCompose(BaseTool):
|
|||
)
|
||||
return comp
|
||||
|
||||
@staticmethod
|
||||
def _cuts_to_cinematic_scenes(cuts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Adapt canonical sequential cuts to CinematicRenderer's scene contract."""
|
||||
|
||||
scenes: list[dict[str, Any]] = []
|
||||
timeline_cursor = 0.0
|
||||
hard_transitions = {"cut", "none"}
|
||||
title_types = {"hero_title", "text_card", "title"}
|
||||
|
||||
for index, cut in enumerate(cuts):
|
||||
try:
|
||||
source_in = float(cut.get("in_seconds", 0))
|
||||
source_out = float(cut.get("out_seconds", source_in))
|
||||
speed = max(float(cut.get("speed", 1.0)), 0.1)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
duration = max(0.0, (source_out - source_in) / speed)
|
||||
if duration <= 0:
|
||||
continue
|
||||
|
||||
scene_id = str(cut.get("id") or f"cut-{index + 1}")
|
||||
source = str(cut.get("source") or "")
|
||||
cut_type = str(cut.get("type") or "").lower()
|
||||
common = {
|
||||
"id": scene_id,
|
||||
"startSeconds": timeline_cursor,
|
||||
"durationSeconds": duration,
|
||||
}
|
||||
|
||||
if cut_type in title_types or not source:
|
||||
scene: dict[str, Any] = {
|
||||
**common,
|
||||
"kind": "title",
|
||||
"text": str(
|
||||
cut.get("text")
|
||||
or cut.get("title")
|
||||
or cut.get("reason")
|
||||
or scene_id
|
||||
),
|
||||
}
|
||||
if source:
|
||||
scene["backgroundSrc"] = source
|
||||
scene["backgroundTrimBeforeSeconds"] = source_in
|
||||
scene["backgroundTrimAfterSeconds"] = source_out
|
||||
else:
|
||||
scene = {
|
||||
**common,
|
||||
"kind": "video",
|
||||
"src": source,
|
||||
"trimBeforeSeconds": source_in,
|
||||
"trimAfterSeconds": source_out,
|
||||
"playbackRate": speed,
|
||||
}
|
||||
if str(cut.get("transition_in") or "").lower() in hard_transitions:
|
||||
scene["fadeInFrames"] = 0
|
||||
if str(cut.get("transition_out") or "").lower() in hard_transitions:
|
||||
scene["fadeOutFrames"] = 0
|
||||
|
||||
scenes.append(scene)
|
||||
timeline_cursor += duration
|
||||
|
||||
return scenes
|
||||
|
||||
@staticmethod
|
||||
def _stage_remotion_media(value: Any, public_dir: Path) -> int:
|
||||
"""Copy local media references into a Remotion public dir in-place.
|
||||
|
||||
OffthreadVideo's compositor rejects ``file://`` sources. Rewriting
|
||||
staged files to relative ``staticFile()`` paths works for video and
|
||||
image components on every platform.
|
||||
"""
|
||||
|
||||
staged_by_source: dict[Path, str] = {}
|
||||
media_keys = {"source", "src", "backgroundSrc"}
|
||||
|
||||
def visit(node: Any, parent_key: str | None = None) -> Any:
|
||||
if isinstance(node, dict):
|
||||
for key, child in list(node.items()):
|
||||
node[key] = visit(child, key)
|
||||
return node
|
||||
if isinstance(node, list):
|
||||
for index, child in enumerate(node):
|
||||
node[index] = visit(child, parent_key)
|
||||
return node
|
||||
if not isinstance(node, str) or parent_key not in media_keys:
|
||||
return node
|
||||
if node.startswith(("http://", "https://", "data:")):
|
||||
return node
|
||||
|
||||
if node.lower().startswith("file://"):
|
||||
parsed = urlsplit(node)
|
||||
decoded_path = unquote(parsed.path)
|
||||
if len(parsed.netloc) == 2 and parsed.netloc[1] == ":":
|
||||
raw_path = f"{parsed.netloc}{decoded_path}"
|
||||
elif parsed.netloc and parsed.netloc.lower() != "localhost":
|
||||
raw_path = f"//{parsed.netloc}{decoded_path}"
|
||||
else:
|
||||
raw_path = decoded_path
|
||||
# Standard Windows file URIs use file:///C:/...; pathlib on
|
||||
# Windows needs the drive path without the URI's leading slash.
|
||||
if len(raw_path) >= 3 and raw_path[0] == "/" and raw_path[2] == ":":
|
||||
raw_path = raw_path[1:]
|
||||
else:
|
||||
raw_path = node
|
||||
source = Path(raw_path).resolve()
|
||||
if not source.is_file():
|
||||
return node
|
||||
if source not in staged_by_source:
|
||||
digest = hashlib.sha256(str(source).encode("utf-8")).hexdigest()[:12]
|
||||
name = f"{digest}-{source.name}"
|
||||
public_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, public_dir / name)
|
||||
staged_by_source[source] = name
|
||||
return staged_by_source[source]
|
||||
|
||||
visit(value)
|
||||
return len(staged_by_source)
|
||||
|
||||
def _render_via_atelier(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
|
|
@ -840,6 +1004,11 @@ class VideoCompose(BaseTool):
|
|||
error=f"Atelier render completed but output file missing: {output_path}",
|
||||
)
|
||||
|
||||
if inputs.get("audio_path"):
|
||||
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
|
||||
if not mux_result.success:
|
||||
return mux_result
|
||||
|
||||
# --- Atelier post-render review -------------------------------------
|
||||
# The cut-schema paths run _run_final_review (technical/visual/audio
|
||||
# probes + transcript-vs-script). Atelier MUST do the same so hero
|
||||
|
|
@ -1063,8 +1232,12 @@ class VideoCompose(BaseTool):
|
|||
try:
|
||||
from styles.playbook_loader import load_playbook
|
||||
playbook = load_playbook(playbook_name)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(
|
||||
"Could not load style playbook %r for Remotion theme: %s",
|
||||
playbook_name,
|
||||
exc,
|
||||
)
|
||||
|
||||
if playbook:
|
||||
vl = playbook.get("visual_language", {})
|
||||
|
|
@ -1418,6 +1591,8 @@ class VideoCompose(BaseTool):
|
|||
# would only take effect on a direct _remotion_render() call.
|
||||
if inputs.get("remotion_timeout_ms") is not None:
|
||||
remotion_inputs["remotion_timeout_ms"] = inputs["remotion_timeout_ms"]
|
||||
if inputs.get("public_dir") is not None:
|
||||
remotion_inputs["public_dir"] = inputs["public_dir"]
|
||||
render_result = self._remotion_render(remotion_inputs)
|
||||
|
||||
# Governance: NEVER silently fall back to FFmpeg when Remotion fails.
|
||||
|
|
@ -1437,6 +1612,11 @@ class VideoCompose(BaseTool):
|
|||
f"Per governance: renderer downgrade requires user approval."
|
||||
),
|
||||
)
|
||||
if inputs.get("audio_path"):
|
||||
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
|
||||
if not mux_result.success:
|
||||
return mux_result
|
||||
render_result.data["has_mixed_audio"] = True
|
||||
else:
|
||||
# --- FFmpeg fallback: only when Remotion is unavailable ---
|
||||
options = inputs.get("options", {})
|
||||
|
|
@ -1546,7 +1726,12 @@ class VideoCompose(BaseTool):
|
|||
try:
|
||||
from styles.playbook_loader import load_playbook # type: ignore
|
||||
playbook_data = load_playbook(playbook_name)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(
|
||||
"Could not load style playbook %r for HyperFrames bridge: %s",
|
||||
playbook_name,
|
||||
exc,
|
||||
)
|
||||
playbook_data = None
|
||||
|
||||
hf_inputs: dict[str, Any] = {
|
||||
|
|
@ -1677,8 +1862,6 @@ class VideoCompose(BaseTool):
|
|||
types, and transitions using React-based frame-accurate rendering.
|
||||
Accepts edit_decisions (with resolved file paths) or raw composition_data.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
if not shutil.which("npx"):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
|
|
@ -1700,16 +1883,6 @@ class VideoCompose(BaseTool):
|
|||
# Deep-copy props so we don't mutate the original
|
||||
props = json.loads(json.dumps(composition_data))
|
||||
|
||||
# Convert absolute file paths to file:// URIs for Remotion's
|
||||
# Img and OffthreadVideo components
|
||||
for cut in props.get("cuts", []):
|
||||
source = cut.get("source", "")
|
||||
if source and not source.startswith(("http://", "https://", "file://")):
|
||||
resolved = Path(source).resolve()
|
||||
if resolved.exists():
|
||||
posix = resolved.as_posix()
|
||||
cut["source"] = f"file:///{posix}" if not posix.startswith("/") else f"file://{posix}"
|
||||
|
||||
# Build a custom themeConfig from the playbook's actual colors.
|
||||
# This ensures every video gets a unique visual identity derived
|
||||
# from its production decisions — not picked from a preset menu.
|
||||
|
|
@ -1723,11 +1896,6 @@ class VideoCompose(BaseTool):
|
|||
if theme_config:
|
||||
props["themeConfig"] = theme_config
|
||||
|
||||
# Write props to temp file for Remotion CLI
|
||||
props_path = output_path.parent / ".remotion_props.json"
|
||||
with open(props_path, "w", encoding="utf-8") as f:
|
||||
json.dump(props, f)
|
||||
|
||||
# remotion-composer lives at project root
|
||||
composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer"
|
||||
if not composer_dir.exists():
|
||||
|
|
@ -1741,6 +1909,39 @@ class VideoCompose(BaseTool):
|
|||
renderer_family = (composition_data or {}).get("renderer_family", "explainer-data")
|
||||
composition_id = self._get_composition_id(renderer_family)
|
||||
|
||||
if composition_id == "CinematicRenderer":
|
||||
if not props.get("scenes") and props.get("cuts"):
|
||||
props["scenes"] = self._cuts_to_cinematic_scenes(props["cuts"])
|
||||
props.pop("cuts", None)
|
||||
if not props.get("scenes"):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="CinematicRenderer received cuts but none could be adapted into scenes.",
|
||||
)
|
||||
|
||||
requested_public_dir = inputs.get("public_dir")
|
||||
cleanup_public_dir = False
|
||||
public_dir: Path | None = None
|
||||
if requested_public_dir:
|
||||
public_dir = Path(requested_public_dir).resolve()
|
||||
if not public_dir.is_dir():
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Remotion public_dir does not exist or is not a directory: {public_dir}",
|
||||
)
|
||||
else:
|
||||
public_dir = output_path.parent / f".remotion-public-{output_path.stem}"
|
||||
cleanup_public_dir = True
|
||||
|
||||
staged_count = self._stage_remotion_media(props, public_dir)
|
||||
if not staged_count and cleanup_public_dir:
|
||||
public_dir = None
|
||||
|
||||
# Write the fully adapted/staged props, never the original cut payload.
|
||||
props_path = output_path.parent / ".remotion_props.json"
|
||||
with open(props_path, "w", encoding="utf-8") as f:
|
||||
json.dump(props, f)
|
||||
|
||||
cmd = [
|
||||
"npx", "remotion", "render",
|
||||
str(composer_dir / "src" / "index.tsx"),
|
||||
|
|
@ -1753,6 +1954,8 @@ class VideoCompose(BaseTool):
|
|||
# API Remotion recommends for file paths and is cross-platform safe.
|
||||
f"--props={props_path}",
|
||||
]
|
||||
if public_dir is not None:
|
||||
cmd.append(f"--public-dir={public_dir}")
|
||||
|
||||
# Apply media profile dimensions
|
||||
profile_name = inputs.get("profile")
|
||||
|
|
@ -1770,7 +1973,8 @@ class VideoCompose(BaseTool):
|
|||
# opaque failure. Pass it through and give the subprocess enough headroom
|
||||
# so run_command() does not kill Remotion before its own timeout fires.
|
||||
remotion_timeout_ms = inputs.get("remotion_timeout_ms")
|
||||
subprocess_timeout = 600
|
||||
scene_count = len(props.get("scenes") or props.get("cuts") or [])
|
||||
subprocess_timeout = max(600, scene_count * 15)
|
||||
if remotion_timeout_ms:
|
||||
try:
|
||||
ms = int(remotion_timeout_ms)
|
||||
|
|
@ -1808,6 +2012,8 @@ class VideoCompose(BaseTool):
|
|||
finally:
|
||||
if props_path.exists():
|
||||
props_path.unlink()
|
||||
if cleanup_public_dir and public_dir is not None and public_dir.exists():
|
||||
shutil.rmtree(public_dir, ignore_errors=True)
|
||||
|
||||
if not output_path.exists():
|
||||
return ToolResult(
|
||||
|
|
@ -1821,6 +2027,7 @@ class VideoCompose(BaseTool):
|
|||
"operation": "remotion_render",
|
||||
"output": str(output_path),
|
||||
"profile": profile_name,
|
||||
"staged_media_count": staged_count,
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue