#!/usr/bin/env python3 """ Gemini 3.1 Pro architecture consultation for Starsend platform. Sends the full plan + relevant Recoil source files to Gemini for analysis, critique, and counter-proposal. Supports multi-round conversation. Usage: python3 gemini_consult.py # Round 1: initial analysis python3 gemini_consult.py --round 2 # Round 2: follow-up with Claude's response python3 gemini_consult.py --round 3 # Round 3: convergence """ import argparse import json import os import sys import time from pathlib import Path from recoil.core.paths import PIPELINE_ROOT, RECOIL_ROOT, DEFAULT_PROJECT OUTPUT_DIR = PIPELINE_ROOT / "gemini_consultation" MODEL = "gemini-3.1-pro-preview" def load_file(path: Path, label: str) -> str: """Load a file with a labeled header.""" if not path.exists(): return f"\n{'='*60}\n[FILE: {label}]\n[FILE NOT FOUND: {path}]\n{'='*60}\n" content = path.read_text() return f"\n{'='*60}\n[FILE: {label}]\n[PATH: {path}]\n{'='*60}\n{content}\n" def build_context_bundle() -> str: """Build the full context bundle from Recoil source files.""" project = DEFAULT_PROJECT files = [ (RECOIL_ROOT / "_engine" / "tools" / "test_nbp_direct.py", "test_nbp_direct.py — V4 triptych generation + scene reference chaining (PROVEN CODE)"), (RECOIL_ROOT / "_engine" / "lib" / "config_loader.py", "config_loader.py — Shared config loader (camera, film stock, identity lock)"), (RECOIL_ROOT / project / "visual" / "project_config.json", "project_config.json — Camera body, film stock, HEX palette, budget"), ] bundle = "" for path, label in files: bundle += load_file(path, label) # Storyboard excerpt (first 4 shots) sb_path = RECOIL_ROOT / project / "storyboards" / "storyboard_ep_001.json" if sb_path.exists(): sb = json.loads(sb_path.read_text()) excerpt = { "version": sb["version"], "project": sb["project"], "episode": sb["episode"], "title": sb["title"], "visual_vocabulary": sb.get("visual_vocabulary", {}), "characters": sb.get("characters", {}), "location": sb.get("location", ""), "atmosphere": sb.get("atmosphere", ""), "cinematic": sb.get("cinematic", ""), "lens_package": sb.get("lens_package", {}), "shots": sb["shots"][:5], # First 5 shots "_total_shots": len(sb["shots"]), } bundle += f"\n{'='*60}\n" bundle += "[FILE: storyboard_ep_001.json — EXCERPT (first 5 of " bundle += f"{len(sb['shots'])} shots, showing full shot schema)]\n" bundle += f"{'='*60}\n" bundle += json.dumps(excerpt, indent=2) bundle += "\n" # Breakdown excerpt (characters + first 5 locations) bd_path = RECOIL_ROOT / project / "visual" / "breakdown.json" if bd_path.exists(): bd = json.loads(bd_path.read_text()) # Include full character data for first 2 characters, summary for others all_char_names = list(bd.get("characters", {}).keys()) lead_chars = set(all_char_names[:2]) chars_excerpt = {} for name, data in bd.get("characters", {}).items(): if name in lead_chars: chars_excerpt[name] = data else: chars_excerpt[name] = { "display_name": data.get("display_name", name), "episode_count": data.get("episode_count", 0), "first_appearance": data.get("first_appearance"), "_note": "Full data available but truncated for context" } locs = dict(list(bd.get("locations", {}).items())[:5]) excerpt = { "version": bd["version"], "project": bd["project"], "episodes_processed": bd["episodes_processed"], "characters": chars_excerpt, "locations": locs, "_total_locations": len(bd.get("locations", {})), "habitat_zones": bd.get("habitat_zones", {}), "props": dict(list(bd.get("props", {}).items())[:5]), "_total_props": len(bd.get("props", {})), } bundle += f"\n{'='*60}\n" lead_names = "+".join(sorted(lead_chars)) bundle += f"[FILE: breakdown.json — EXCERPT ({lead_names} full, " bundle += f"others summarized, 5 of {len(bd.get('locations', {}))} locations)]\n" bundle += f"{'='*60}\n" bundle += json.dumps(excerpt, indent=2) bundle += "\n" # Character reference images inventory (first lead character) lead_char = all_char_names[0] if all_char_names else None if lead_char: picks_path = RECOIL_ROOT / project / "visual" / "lora_candidates" / lead_char / "picks" keystones_path = RECOIL_ROOT / project / "visual" / "lora_candidates" / lead_char / "keystones" ref_inventory = [] if picks_path.exists(): for f in sorted(picks_path.iterdir()): if f.suffix.lower() in ('.png', '.jpg', '.jpeg'): ref_inventory.append(f" picks/{f.name} ({f.stat().st_size / 1024:.0f} KB)") if keystones_path.exists(): for f in sorted(keystones_path.iterdir()): if f.suffix.lower() in ('.png', '.jpg', '.jpeg'): ref_inventory.append(f" keystones/{f.name} ({f.stat().st_size / 1024:.0f} KB)") bundle += f"\n{'='*60}\n" bundle += f"[INVENTORY: {lead_char} reference images in lora_candidates/]\n" bundle += f"{'='*60}\n" bundle += "\n".join(ref_inventory) bundle += "\n" return bundle ARCHITECTURE_PLAN = """ # Starsend Project — Frontier Model Visual Production Platform ## Context Through 4 iterations of testing NBP (Gemini 3 Pro Image Preview) as a direct renderer for Leviathan EP001 shots 1-3, we proved that frontier models can replace the Flux 2 + LoRA pipeline — but need a fundamentally different prompt architecture. Frontier models accept reference images inline, use natural language (not LoRA triggers or comma-separated T2I keywords), and don't need word-count budgets or negative prompts. **What worked in testing:** - Triptych generation (single 21:9 strip with 3 panels) forces within-shot consistency - Labeled reference images (text labels between image Parts) help NBP distinguish character vs scene refs - Character identity holds with 3 white-bg reference images - ENV sanitization (regex-strip human-presence language) keeps people out of environment shots - Scene reference chaining (Shot 1 ENV hero → fed as ref to Shots 2-3) helps environment consistency **What's still broken:** - Cross-shot environments look different (scene ref alone isn't enough — need location ref images) - Screen direction inconsistency in triptych panels (Shot 3 mirrored facing) - Missing emotion/strain in action shots (prompt needs action/emotion emphasis) - Location refs directory is EMPTY (0 images for 72 defined locations) - All 7 Jinx wardrobe phases have null reference_images in breakdown.json - EP002-060 storyboards lack spatial data (only EP001 was backfilled) **New insight (2x2 / 3x3 grid method):** Instead of only 1x3 triptychs, use multi-panel grid layouts (2x2, 3x3) to generate multiple consistent frames in a single call. A 3x3 at 1:1 produces 9 consistent panels from ONE API call ($0.134). The best panel gets extracted, upscaled, then used as a hero reference for the production triptych. This creates a two-pass pipeline: **planning grid → hero selection → production frames**. **The ask:** Build a comprehensive, standalone platform that compiles frontier-model-ready prompt+reference packages from existing Recoil data, generates frames via configurable grid layouts, and prepares upload bundles for models without API access. ## Architecture ``` ~/CLAUDE_PROJECTS/starsend/ ├── lib/ │ ├── recoil_bridge.py # Read storyboard, breakdown, config, refs from recoil/ │ ├── frontier_compiler.py # Compile PromptPackage (text + ref manifest + directives) │ ├── reference_manager.py # Select right refs per shot (angle, expression, wardrobe) │ └── model_profiles.py # Per-model capabilities (NBP, Kling, Veo, Seedance) ├── orchestrator/ │ ├── scene_planner.py # Scene-aware generation ordering (ENV first) │ ├── grid_engine.py # Generate grids (1x2, 1x3, 2x2, 3x3), split, extract panels │ └── generation_runner.py # Main loop: plan → compile → generate → split → save ├── tools/ │ ├── generate_location_refs.py # Generate missing location ref images via NBP │ ├── build_upload_bundle.py # Package prompts + refs for manual frontier model upload │ ├── backfill_storyboard.py # Fix spatial data, populate triptych_prompt fields │ └── compare_frames.py # CLI frame comparison utility ├── editors/ │ ├── review.html # Side-by-side frame review UI (with grid layout selector) │ └── review_server.py # Local server (port 8430) ├── config/ │ ├── pipeline_config.json # Pipeline settings (recoil_engine_root path, defaults) │ └── model_profiles.json # NBP, Kling, Veo capability definitions ├── assets/ │ └── grid_templates/ # Grid reference images (1x2, 1x3, 2x2, 3x3 blank grids) └── output/ ├── frames/ep_{NNN}/ # Generated grid images + split panels ├── bundles/ # Upload bundles for manual frontier model use └── assets/loc/ # Generated location reference images ``` **Key principle:** Starsend reads from Recoil by path (`RECOIL_ROOT = ~/CLAUDE_PROJECTS/recoil`), never copies data. It writes results to its own `output/` directory. It can optionally write back to Recoil (location refs, backfill patches) when explicitly asked. ## NBP Capabilities (Confirmed via API Docs) | Parameter | Options | |-----------|---------| | **Aspect ratios** | `1:1, 1:4, 1:8, 2:3, 3:2, 3:4, 4:1, 4:3, 4:5, 5:4, 8:1, 9:16, 16:9, 21:9` | | **Image sizes** | `512px, 1K, 2K, 4K` | | **Reference images** | Up to 14 total (Gemini 3 Pro: 6 objects + 5 characters) | | **Models** | `gemini-3-pro-image-preview` ($0.134), `gemini-3.1-flash-image-preview`, `gemini-2.5-flash-image` ($0.039) | ## Grid Layouts | Layout | Grid | Aspect Ratio | Panels | Best For | |--------|------|-------------|--------|----------| | **Triptych** | 1x3 | 21:9 | 3 | First/hero/last frame sequence (current default) | | **Diptych** | 1x2 | 2:1 | 2 | First/last only (shots with empty last_frame) | | **Quad** | 2x2 | 1:1 | 4 | 4 key moments or 4 angles of same moment | | **Full Grid** | 3x3 | 1:1 | 9 | Scene planning — 9 consistent angles/moments | ### Two-Pass Generation Strategy (3x3 → Triptych) 1. **Pass 1 — Scene Planning Grid** (3x3 at 1:1, $0.134) - Generate 9 panels showing the scene from different angles/moments - Grid prompt: "Generate a 3x3 grid of 9 cinematic stills showing [scene]. Each panel shows a different camera angle. Maintain identical character appearance, costume, and lighting across all panels." - Upload a blank 3x3 grid template image + "Follow the structure of this reference exactly" 2. **Pass 2 — Hero Selection** - Review the 9 panels, pick the best composition - Extract that 1/9th panel (~341x341 from a 1024x1024 grid) 3. **Pass 3 — Upscale** (via SeedVR2 or re-generation) 4. **Pass 4 — Production Triptych** (1x3 at 21:9, $0.134) - Using the upscaled hero as scene/character reference - Generate first/hero/last frames with the hero anchoring consistency ## Central Data Type: PromptPackage ```python @dataclass class PromptPackage: prompt_text: str # Natural language prompt panels: list[PanelDescription] # Panel descriptions (2, 3, 4, or 9) references: list[ReferenceImage] # Ordered ref images with labels grid_layout: GridLayout # DIPTYCH, TRIPTYCH, QUAD, FULL_GRID model: str # Target model ID aspect_ratio: str # Computed from grid_layout image_size: str # "1K", "2K", "4K" directives: list[str] # ["no direct-to-camera", "ENV: no people"] is_env: bool # No characters in this shot provides_scene_ref: bool # First ENV shot in scene needs_scene_ref: bool # Subsequent shots reference scene ENV grid_template_ref: Optional[Path] # Grid template image for structural enforcement def to_genai_parts(self) -> list # → google.genai API Parts def to_bundle_manifest(self) -> dict # → upload bundle files ``` ## Compiler Layers 1. **Reference image selection** — Pick optimal character refs by shot angle + expression 2. **Scene context block** — Location, lighting, camera, film stock, wardrobe as TEXT 3. **Panel descriptions** — Per-panel frame text with temporal markers 4. **Behavioral directives** — Camera direction guard, ENV sanitization, screen direction 5. **Reference labeling** — Text labels between image Parts ## ENV Sanitization (port from test_nbp_direct.py): ```python _HUMAN_PATTERNS = [ r",?\\s*a\\s+figure'?s?\\s+\\w+\\s+visible\\s+[^,\\.]*[,\\.]?", r",?\\s*\\b(?:a|the)\\s+(?:figure|person|silhouette|someone)\\b[^,\\.]*[,\\.]?", r"\\b(?:her|his|their)\\s+(?:arms?|hands?|face|body|boots?|feet|legs?)\\b", r"(?:^|\\.\\s*)\\b(?:She|He)\\s+[^\\.]+\\.", r"\\bthe\\s+figure\\b", ] ``` ## Phases Phase 1: Foundation + Recoil Bridge + Grid Templates Phase 2: Frontier Prompt Compiler Phase 3: Grid Engine + Generation Runner Phase 4: Location Reference Generation Phase 5: Upload Bundle Builder Phase 6: Review UI (port 8430) Phase 7: Storyboard Data Backfill ## Cost Estimates Single-pass: ~$259 for 60 episodes Two-pass: ~$508 for 60 episodes Hybrid (practical): ~$194 for 60 episodes ## Key Questions for Gemini 1. Grid template enforcement — will uploading a blank grid template image + "Follow this structure" actually help NBP maintain panel boundaries? Or is there a better approach? 2. Screen direction consistency — what's the best prompting technique to maintain consistent character facing across triptych panels? 3. 3x3 grid quality — at 1024x1024, each sub-panel is ~341x341. Is that enough visual info for hero selection, or should we use higher resolution? 4. Reference image ordering — does the order of Parts in the API call matter for NBP's attention? 5. Cross-shot environment consistency — beyond scene reference chaining, what other techniques would help lock environments? 6. Emotion/action emphasis — how to get NBP to render strain, effort, and emotion rather than static posed characters? 7. Model selection — when should we use gemini-3-pro-image-preview vs gemini-3.1-flash-image-preview vs gemini-2.5-flash-image? Cost/quality tradeoffs? """ def get_round_1_prompt() -> str: return """You are serving as an architecture consultant for the Starsend project — a frontier model visual production platform being built to generate cinematic frames for a vertical microdrama series called "Leviathan." The builder (Claude, an AI assistant) has designed a comprehensive 7-phase plan. Your job is to: 1. **Analyze the architecture plan** — identify strengths, weaknesses, gaps, and risks 2. **Critique the prompting strategy** — you ARE Gemini. You know how your image generation works internally. Where is this plan likely to succeed? Where will it break? 3. **Answer the key questions** listed at the end of the plan (grid template enforcement, screen direction, etc.) 4. **Propose your own architecture** — if you would design this differently, show your alternative. Be specific with code structure, data types, and API usage patterns. 5. **Share undocumented tips** — any Gemini image generation behaviors, tricks, or gotchas that aren't in the public docs but that you know from your training data Be brutally honest. This is a production system that will generate ~1,800 shots across 60 episodes. Wrong architectural decisions here cost real money and weeks of rework. IMPORTANT CONTEXT: - The model being used for image generation is `gemini-3-pro-image-preview` (what the builder calls "NBP" / "Nanobanana") - The target aspect ratio for final delivery is 9:16 (vertical microdrama) - The existing test code (test_nbp_direct.py) has been proven to work — it generates triptychs with scene reference chaining - The Recoil engine is an existing scripting/production system with rich storyboard data (40+ fields per shot) Below is the full plan, followed by all relevant source files from the existing codebase. --- """ def get_round_2_prompt(claude_response: str) -> str: return f"""This is Round 2 of our architecture consultation. In Round 1, you analyzed the Starsend architecture plan and provided your critique and counter-proposal. Now Claude (the builder) has read your analysis and has the following response: --- {claude_response} --- Please: 1. Address Claude's responses to your concerns 2. Identify any remaining disagreements and explain your reasoning 3. Propose specific compromises or merged approaches where your ideas and Claude's can combine 4. Focus on the TOP 3 highest-impact decisions that will determine success or failure 5. Provide concrete code snippets for any patterns you're recommending Remember: we want to converge on a final architecture, not debate indefinitely. Be constructive and specific. """ def get_round_3_prompt(claude_response: str) -> str: return f"""This is Round 3 — the convergence round. Claude has responded to your Round 2 feedback: --- {claude_response} --- Please: 1. State clearly which architectural decisions are now AGREED 2. For any remaining disagreements, give your FINAL recommendation with a clear rationale 3. Produce a FINAL merged architecture summary — the definitive version incorporating the best ideas from both sides 4. List the TOP 5 implementation risks and your mitigation strategy for each 5. Suggest a specific test protocol for the first milestone (EP001 shots 1-3) This is the last round. Make it count. Be definitive. """ def get_round_4_prompt(claude_response: str) -> str: return f"""This is Round 4 — a pushback round. The showrunner (JT, the filmmaker building this platform) has read your entire consultation and disagrees with your complete dismissal of the grid method. Your prior positions from Rounds 1-3 are loaded below for context. Claude's pushback on behalf of the showrunner follows: --- {claude_response} --- Please respond to each of the 5 numbered questions directly. Be specific about: - Resolution math at 4K output - Whether text-only grid prompting (no template images) works - Where grids fit in the pipeline alongside the native vertical batch approach - The shared-seed consistency advantage of grids vs independent Flash candidates - Grid method for bulk reference generation (locations, expressions) Be honest. If you were wrong to dismiss grids entirely, say so. If your concerns still hold at 4K, explain the specific failure mode. """ def get_round_5_prompt(claude_response: str) -> str: return f"""This is Round 5 — integrating community-proven structured grid prompting techniques into the Starsend pipeline. The showrunner (JT, a 20-year filmmaker) found two prompt engineering approaches from the AI video production community. Claude has analyzed them and asks 6 specific questions. Your prior analysis (Rounds 1-4) is loaded for context. IMPORTANT CONTEXT REMINDER: - We are building Starsend, a frontier model visual production platform - Target: 60 episodes x ~31 shots = ~1,800 shots for Leviathan microdrama series - Each shot in our storyboard JSON has: shot_type, camera_angle, focal_length, aperture, action, emotion, lighting, first_frame, hero_frame, last_frame, characters_in_shot, spatial data - We have 12 curated white-bg Jinx reference images + 1 keystone - The grid method was reinstated in Round 4 for planning/reference generation at 4K - The pipeline is: Grid planning → Hero extraction → Pro final render at native 9:16 Now Claude and the showrunner want to know if STRUCTURED grid positions (each cell assigned a specific shot type from the storyboard) can replace the generic "9 angles" approach. Claude's analysis and questions: --- {claude_response} --- Please: 1. Answer all 6 numbered questions directly 2. For question 6, write the EXACT prompt we should use for a test — including the grid instruction, the scene context from the storyboard, the reference image labeling, and the structured panel assignments. Make it copy-paste ready. 3. Evaluate whether structured grid prompting is worth the added prompt complexity vs generic grids 4. Suggest the optimal grid strategy for our pipeline: when to use structured grids, when to use generic grids, when to skip grids entirely 5. If you have any additional techniques from these community prompts that we should adopt, call them out Be concrete. We want to run a test TODAY on EP001 Shot 2. """ def run_consultation(round_num: int = 1, prev_response_file: str = None): """Run a round of Gemini consultation.""" from google import genai api_key = os.environ.get("GOOGLE_API_KEY") if not api_key: print("ERROR: GOOGLE_API_KEY not set") sys.exit(1) client = genai.Client(api_key=api_key) OUTPUT_DIR.mkdir(parents=True, exist_ok=True) # Build the prompt based on round if round_num == 1: print(f"=== STARSEND ARCHITECTURE CONSULTATION — ROUND {round_num} ===") print(f"Model: {MODEL}") print(f"Sending: Full architecture plan + Recoil source files") print() context_bundle = build_context_bundle() prompt = get_round_1_prompt() + ARCHITECTURE_PLAN + "\n\n---\n\n# SOURCE FILES FROM RECOIL CODEBASE\n\n" + context_bundle elif round_num in (2, 3, 4, 5): if not prev_response_file: # Look for the Claude response file prev_response_file = str(OUTPUT_DIR / f"claude_response_round_{round_num - 1}.md") if not Path(prev_response_file).exists(): print(f"ERROR: Previous response file not found: {prev_response_file}") print(f"Please save Claude's response to: {prev_response_file}") sys.exit(1) claude_response = Path(prev_response_file).read_text() print(f"=== STARSEND ARCHITECTURE CONSULTATION — ROUND {round_num} ===") print(f"Model: {MODEL}") print(f"Claude response loaded from: {prev_response_file}") print() if round_num == 2: # Include original context + round 1 response for continuity gemini_r1 = OUTPUT_DIR / "gemini_response_round_1.md" prior_context = "" if gemini_r1.exists(): prior_context = ( "\n\n--- YOUR PREVIOUS ANALYSIS (Round 1) ---\n\n" + gemini_r1.read_text() ) prompt = get_round_2_prompt(claude_response) + prior_context elif round_num == 3: gemini_r2 = OUTPUT_DIR / "gemini_response_round_2.md" prior_context = "" if gemini_r2.exists(): prior_context = ( "\n\n--- YOUR PREVIOUS ANALYSIS (Round 2) ---\n\n" + gemini_r2.read_text() ) prompt = get_round_3_prompt(claude_response) + prior_context elif round_num == 4: # Load all 3 prior Gemini responses for full context prior_context = "" for r in (1, 2, 3): rfile = OUTPUT_DIR / f"gemini_response_round_{r}.md" if rfile.exists(): prior_context += ( f"\n\n--- YOUR ANALYSIS (Round {r}) ---\n\n" + rfile.read_text() ) prompt = get_round_4_prompt(claude_response) + prior_context elif round_num == 5: # Load all 4 prior Gemini responses for full context prior_context = "" for r in (1, 2, 3, 4): rfile = OUTPUT_DIR / f"gemini_response_round_{r}.md" if rfile.exists(): prior_context += ( f"\n\n--- YOUR ANALYSIS (Round {r}) ---\n\n" + rfile.read_text() ) prompt = get_round_5_prompt(claude_response) + prior_context else: print(f"ERROR: Invalid round number: {round_num}") sys.exit(1) # Count tokens (approximate) char_count = len(prompt) est_tokens = char_count // 4 print(f"Prompt size: {char_count:,} chars (~{est_tokens:,} tokens)") print(f"Sending to {MODEL}...") print() t0 = time.time() try: response = client.models.generate_content( model=MODEL, contents=prompt, config={ "temperature": 0.7, "max_output_tokens": 16384, }, ) except Exception as e: print(f"ERROR: API call failed: {e}") sys.exit(1) elapsed = time.time() - t0 response_text = response.text if hasattr(response, 'text') else str(response) # Save response output_file = OUTPUT_DIR / f"gemini_response_round_{round_num}.md" output_file.write_text(response_text) print(f"Response received in {elapsed:.1f}s") print(f"Response length: {len(response_text):,} chars") print(f"Saved to: {output_file}") print() print("=" * 60) print(response_text) print("=" * 60) if round_num < 3: next_file = OUTPUT_DIR / f"claude_response_round_{round_num}.md" print(f"\n\nNext step: Save Claude's response to {next_file}") print(f"Then run: python3 gemini_consult.py --round {round_num + 1}") def main(): parser = argparse.ArgumentParser(description="Gemini architecture consultation for Starsend") parser.add_argument("--round", type=int, default=1, help="Consultation round (1-3)") parser.add_argument("--response-file", type=str, help="Path to Claude's previous response") args = parser.parse_args() run_consultation(args.round, args.response_file) if __name__ == "__main__": main()