diff --git a/.env.example b/.env.example index 225a2b75..394c6252 100644 --- a/.env.example +++ b/.env.example @@ -44,14 +44,14 @@ NEXTAUTH_SECRET="your-super-secret-key-change-in-production" # LOG_LEVEL="info" # Options: trace, debug, info, warn, error, fatal # Cron Job Secret (for daily credit reset) -CRON_SECRET="IkD/VzyrCRc6c/146TjKhIzOZ9HFq+Meo00y+wQpws8=" +CRON_SECRET="your-secret-key-here" # Media Generation - Wiro.ai (optional) # WIRO_API_KEY=your_wiro_api_key # WIRO_VIDEO_MODELS="google/veo3.1-fast" # Comma-separated list of video models # WIRO_IMAGE_MODELS="google/nano-banana-pro,google/nano-banana" # Comma-separated list of image models -# Media Generation - Fal.ai (optional - currently disabled) +# Media Generation - Fal.ai (optional) # FAL_API_KEY=your_fal_api_key -# FAL_VIDEO_MODELS="" # Comma-separated list of video models -# FAL_IMAGE_MODELS="" # Comma-separated list of image models \ No newline at end of file +# FAL_VIDEO_MODELS="fal-ai/veo3,fal-ai/kling-video/v2/master/text-to-video" # Comma-separated list of video models +# FAL_IMAGE_MODELS="fal-ai/flux-pro/v1.1-ultra,fal-ai/flux/dev" # Comma-separated list of image models \ No newline at end of file diff --git a/src/app/api/media-generate/status/route.ts b/src/app/api/media-generate/status/route.ts new file mode 100644 index 00000000..2702cd8e --- /dev/null +++ b/src/app/api/media-generate/status/route.ts @@ -0,0 +1,53 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { getMediaGeneratorPlugin } from "@/lib/plugins/media-generators"; + +/** + * Polling endpoint for media generation status + * Used by providers that don't support WebSocket (e.g., Fal.ai) + */ +export async function GET(request: NextRequest) { + const session = await auth(); + + if (!session?.user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const searchParams = request.nextUrl.searchParams; + const provider = searchParams.get("provider"); + const socketAccessToken = searchParams.get("token"); + + if (!provider || !socketAccessToken) { + return NextResponse.json( + { error: "Missing provider or token" }, + { status: 400 } + ); + } + + const plugin = getMediaGeneratorPlugin(provider); + + if (!plugin) { + return NextResponse.json( + { error: `Provider "${provider}" not found` }, + { status: 404 } + ); + } + + if (!plugin.checkStatus) { + return NextResponse.json( + { error: "Provider does not support polling" }, + { status: 400 } + ); + } + + try { + const result = await plugin.checkStatus(socketAccessToken); + return NextResponse.json(result); + } catch (error) { + console.error("Status check error:", error); + return NextResponse.json( + { error: error instanceof Error ? error.message : "Status check failed" }, + { status: 500 } + ); + } +} diff --git a/src/components/prompts/media-generator.tsx b/src/components/prompts/media-generator.tsx index 5c333a45..d39f77be 100644 --- a/src/components/prompts/media-generator.tsx +++ b/src/components/prompts/media-generator.tsx @@ -160,64 +160,120 @@ export function MediaGenerator({ const { socketAccessToken, webSocketUrl, provider } = await response.json(); - // Get provider-specific handler - const handler = getProviderWebSocketHandler(provider); + // Create callbacks for completion handling + const handleComplete = (urls: string[]) => { + if (urls.length > 0) { + onMediaGenerated(urls[0]); + toast.success(t("mediaGenerated")); + } + // Reset after a delay + setTimeout(() => { + setStatus("idle"); + setProgress(0); + setStatusKey(null); + setSelectedModel(null); + }, 2000); + }; - // Connect to WebSocket for progress tracking - setStatus("queued"); - setProgress(20); - setStatusKey("connecting"); + // Check if provider uses polling (empty webSocketUrl) or WebSocket + if (!webSocketUrl) { + // Polling mode (for Fal.ai) + setStatus("queued"); + setProgress(20); + setStatusKey("queued"); - const ws = new WebSocket(webSocketUrl); - wsRef.current = ws; + const pollStatus = async () => { + try { + const statusResponse = await fetch( + `/api/media-generate/status?provider=${provider}&token=${encodeURIComponent(socketAccessToken)}` + ); + + if (!statusResponse.ok) { + const data = await statusResponse.json(); + throw new Error(data.error || "Status check failed"); + } - // Create callbacks for the handler - const callbacks: WebSocketCallbacks = { - setProgress, - setStatus, - setStatusMessage: setStatusKey, - setError, - onComplete: (urls: string[]) => { - if (urls.length > 0) { - onMediaGenerated(urls[0]); - toast.success(t("mediaGenerated")); + const statusData = await statusResponse.json(); + + setProgress(statusData.progress); + if (statusData.statusKey) { + setStatusKey(statusData.statusKey); + } + + if (statusData.status === "completed") { + setStatus("completed"); + if (statusData.outputUrls && statusData.outputUrls.length > 0) { + handleComplete(statusData.outputUrls); + } + return; // Stop polling + } + + if (statusData.status === "failed") { + setStatus("error"); + setError("Generation failed"); + return; // Stop polling + } + + // Continue polling + if (statusData.status === "in_queue" || statusData.status === "in_progress") { + setStatus("processing"); + setTimeout(pollStatus, 2000); // Poll every 2 seconds + } + } catch (err) { + setStatus("error"); + setError(err instanceof Error ? err.message : "Polling failed"); } - // Reset after a delay - setTimeout(() => { - setStatus("idle"); - setProgress(0); - setStatusKey(null); - setSelectedModel(null); - }, 2000); - }, - onCleanup: cleanupWebSocket, - }; + }; - ws.onopen = () => { - const initMessage = handler.getInitMessage(socketAccessToken); - if (initMessage) { - ws.send(initMessage); - } - setStatusKey("connected"); - }; + // Start polling + pollStatus(); + } else { + // WebSocket mode (for Wiro.ai and others) + const handler = getProviderWebSocketHandler(provider); - ws.onmessage = (event) => { - handler.handleMessage(event, callbacks); - }; + setStatus("queued"); + setProgress(20); + setStatusKey("connecting"); - ws.onerror = () => { - setStatus("error"); - setError("WebSocket connection error"); - cleanupWebSocket(); - }; + const ws = new WebSocket(webSocketUrl); + wsRef.current = ws; - ws.onclose = () => { - if (status !== "completed" && status !== "error" && status !== "idle") { - // Unexpected close + // Create callbacks for the handler + const callbacks: WebSocketCallbacks = { + setProgress, + setStatus, + setStatusMessage: setStatusKey, + setError, + onComplete: handleComplete, + onCleanup: cleanupWebSocket, + }; + + ws.onopen = () => { + const initMessage = handler.getInitMessage(socketAccessToken); + if (initMessage) { + ws.send(initMessage); + } + setStatusKey("connected"); + }; + + ws.onmessage = (event) => { + handler.handleMessage(event, callbacks); + }; + + ws.onerror = () => { setStatus("error"); - setError("Connection closed unexpectedly"); - } - }; + setError("WebSocket connection error"); + cleanupWebSocket(); + }; + + ws.onclose = () => { + if (status !== "completed" && status !== "error" && status !== "idle") { + // Unexpected close + setStatus("error"); + setError("Connection closed unexpectedly"); + } + }; + } } catch (err) { setStatus("error"); diff --git a/src/lib/plugins/media-generators/fal.ts b/src/lib/plugins/media-generators/fal.ts index d0e92192..52633793 100644 --- a/src/lib/plugins/media-generators/fal.ts +++ b/src/lib/plugins/media-generators/fal.ts @@ -2,13 +2,12 @@ * Fal.ai Media Generator Plugin * * Generates images and videos using Fal.ai API. - * - * NOTE: This plugin is currently DISABLED and serves as a placeholder. + * Uses Fal.ai's queue API for async generation with polling-based status updates. * * Required env vars: * - FAL_API_KEY - * - FAL_VIDEO_MODELS (comma-separated) - * - FAL_IMAGE_MODELS (comma-separated) + * - FAL_VIDEO_MODELS (comma-separated, e.g., "fal-ai/veo3,fal-ai/kling-video/v2/master/image-to-video") + * - FAL_IMAGE_MODELS (comma-separated, e.g., "fal-ai/flux-pro/v1.1-ultra,fal-ai/flux/dev") */ import type { @@ -19,8 +18,11 @@ import type { WebSocketHandler, WebSocketCallbacks, GenerationStatusKey, + PollStatusResult, } from "./types"; +const FAL_QUEUE_BASE = "https://queue.fal.run"; + function parseModels(envVar: string | undefined, type: "image" | "video"): MediaGeneratorModel[] { if (!envVar) return []; return envVar @@ -34,30 +36,151 @@ function parseModels(envVar: string | undefined, type: "image" | "video"): Media })); } -// Map Fal.ai message types to static translation keys (placeholder - uses same keys as Wiro) +// Map Fal.ai status to our status keys const falStatusMap: Record = { - pending: "queued", - in_progress: "generating", - completed: "complete", - failed: "error", + IN_QUEUE: "queued", + IN_PROGRESS: "generating", + COMPLETED: "complete", + FAILED: "error", }; -// Placeholder WebSocket handler for Fal.ai - to be implemented when enabling -const falWebSocketHandler: WebSocketHandler = { - getInitMessage: (_socketAccessToken: string) => { - // Fal.ai may use a different initialization mechanism - return ""; - }, - - handleMessage: (_event: MessageEvent, callbacks: WebSocketCallbacks) => { - // Placeholder - Fal.ai WebSocket handling not implemented yet - callbacks.setError("Fal.ai WebSocket handling is not implemented yet"); - }, -}; - -// Export for potential future use export { falStatusMap }; +// Fal.ai response types +export interface FalQueueResponse { + request_id: string; + response_url: string; + status_url: string; + cancel_url: string; +} + +export interface FalStatusResponse { + status: "IN_QUEUE" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; + queue_position?: number; + response_url?: string; + logs?: Array<{ message: string; timestamp: string }>; +} + +export interface FalImageOutput { + images?: Array<{ url: string; content_type?: string }>; + image?: { url: string }; +} + +export interface FalVideoOutput { + video?: { url: string }; + videos?: Array<{ url: string }>; +} + +/** + * Submit a generation request to Fal.ai queue + */ +async function submitToFalQueue( + modelId: string, + input: Record +): Promise { + const apiKey = process.env.FAL_API_KEY; + if (!apiKey) throw new Error("FAL_API_KEY is not configured"); + + const url = `${FAL_QUEUE_BASE}/${modelId}`; + + const response = await fetch(url, { + method: "POST", + headers: { + "Authorization": `Key ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(input), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Fal.ai API error: ${response.status} - ${errorText}`); + } + + return response.json(); +} + +/** + * Get status of a Fal.ai queue request using the status URL + */ +export async function getFalRequestStatus( + statusUrl: string +): Promise { + const apiKey = process.env.FAL_API_KEY; + if (!apiKey) throw new Error("FAL_API_KEY is not configured"); + + const response = await fetch(statusUrl, { + method: "GET", + headers: { + "Authorization": `Key ${apiKey}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Fal.ai status error: ${response.status} - ${errorText}`); + } + + return response.json(); +} + +/** + * Get result of a completed Fal.ai request using the response URL + */ +export async function getFalRequestResult( + responseUrl: string +): Promise { + const apiKey = process.env.FAL_API_KEY; + if (!apiKey) throw new Error("FAL_API_KEY is not configured"); + + const response = await fetch(responseUrl, { + method: "GET", + headers: { + "Authorization": `Key ${apiKey}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Fal.ai result error: ${response.status} - ${errorText}`); + } + + return response.json(); +} + +/** + * Map aspect ratio to Fal.ai image_size format + */ +function mapAspectRatioToImageSize(aspectRatio?: string): string { + const mapping: Record = { + "1:1": "square", + "16:9": "landscape_16_9", + "9:16": "portrait_16_9", + "4:3": "landscape_4_3", + "3:4": "portrait_4_3", + "3:2": "landscape_4_3", // closest match + "2:3": "portrait_4_3", // closest match + }; + return mapping[aspectRatio || "1:1"] || "square"; +} + +// Fal.ai uses polling, not WebSocket - this handler is for the polling mechanism +const falWebSocketHandler: WebSocketHandler = { + getInitMessage: (socketAccessToken: string) => { + // For Fal.ai, socketAccessToken contains "modelId:requestId" + // This initiates the polling mechanism + return JSON.stringify({ + type: "fal_init", + data: socketAccessToken, + }); + }, + + handleMessage: (_event: MessageEvent, _callbacks: WebSocketCallbacks) => { + // Fal.ai doesn't use WebSocket - polling is handled by the client + // This is a no-op as the actual status checking is done via HTTP polling + }, +}; + export const falGeneratorPlugin: MediaGeneratorPlugin = { id: "fal", name: "Fal.ai", @@ -70,12 +193,10 @@ export const falGeneratorPlugin: MediaGeneratorPlugin = { }, isEnabled: () => { - // Fal.ai is disabled for now - return false even if configured - return false; + return falGeneratorPlugin.isConfigured(); }, getModels: () => { - // Return empty array since disabled if (!falGeneratorPlugin.isEnabled()) { return []; } @@ -84,16 +205,119 @@ export const falGeneratorPlugin: MediaGeneratorPlugin = { return [...imageModels, ...videoModels]; }, - async startGeneration(_request: GenerationRequest): Promise { - throw new Error( - "Fal.ai integration is not yet implemented. Please use Wiro.ai or upload media directly." - ); + async startGeneration(request: GenerationRequest): Promise { + if (!this.isConfigured()) { + throw new Error( + "Fal.ai is not configured. Please set FAL_API_KEY and FAL_VIDEO_MODELS or FAL_IMAGE_MODELS." + ); + } + + const input: Record = { + prompt: request.prompt, + }; + + if (request.type === "video") { + // Video generation parameters + if (request.aspectRatio) { + input.aspect_ratio = request.aspectRatio; + } + if (request.inputImageUrl) { + input.image_url = request.inputImageUrl; + } + } else { + // Image generation parameters + input.image_size = mapAspectRatioToImageSize(request.aspectRatio); + input.num_images = 1; + if (request.inputImageUrl) { + input.image_url = request.inputImageUrl; + } + } + + const queueResponse = await submitToFalQueue(request.model, input); + + // Return status_url and response_url encoded in socketAccessToken for polling + // Format: statusUrl|responseUrl + return { + taskId: queueResponse.request_id, + socketAccessToken: `${queueResponse.status_url}|${queueResponse.response_url}`, + }; }, getWebSocketUrl: () => { - // Placeholder - Fal.ai may use a different WebSocket mechanism + // Fal.ai uses polling, return empty to indicate polling mode return ""; }, webSocketHandler: falWebSocketHandler, + + async checkStatus(socketAccessToken: string): Promise { + // Parse statusUrl|responseUrl from socketAccessToken + const [statusUrl, responseUrl] = socketAccessToken.split("|"); + + if (!statusUrl || !responseUrl) { + throw new Error("Invalid token format"); + } + + const status = await getFalRequestStatus(statusUrl); + + // Map status to our format + const statusKey = falStatusMap[status.status] || "generating"; + + // Calculate progress based on status + let progress = 0; + switch (status.status) { + case "IN_QUEUE": + progress = 25; + break; + case "IN_PROGRESS": + progress = 50; + break; + case "COMPLETED": + progress = 100; + break; + case "FAILED": + progress = 0; + break; + } + + // If completed, fetch the result + let outputUrls: string[] = []; + if (status.status === "COMPLETED") { + const result = await getFalRequestResult(responseUrl); + outputUrls = extractOutputUrls(result); + } + + return { + status: status.status.toLowerCase().replace("_", "_") as PollStatusResult["status"], + statusKey, + progress, + queuePosition: status.queue_position, + outputUrls, + }; + }, }; + +/** + * Extract output URLs from Fal.ai result + */ +function extractOutputUrls(result: FalImageOutput | FalVideoOutput): string[] { + const urls: string[] = []; + + // Image outputs + if ("images" in result && result.images) { + urls.push(...result.images.map((img) => img.url)); + } + if ("image" in result && result.image) { + urls.push(result.image.url); + } + + // Video outputs + if ("videos" in result && result.videos) { + urls.push(...result.videos.map((vid) => vid.url)); + } + if ("video" in result && result.video) { + urls.push(result.video.url); + } + + return urls; +} diff --git a/src/lib/plugins/media-generators/types.ts b/src/lib/plugins/media-generators/types.ts index be59b96d..5f121d2b 100644 --- a/src/lib/plugins/media-generators/types.ts +++ b/src/lib/plugins/media-generators/types.ts @@ -40,6 +40,18 @@ export interface GenerationResult { error?: string; } +/** + * Result from polling-based status check + */ +export interface PollStatusResult { + status: "in_queue" | "in_progress" | "completed" | "failed"; + statusKey: GenerationStatusKey; + progress: number; + queuePosition?: number; + outputUrls: string[]; + error?: string; +} + // WebSocket handler types (client-side) export interface WebSocketCallbacks { setProgress: (value: number | ((prev: number) => number)) => void; @@ -97,11 +109,16 @@ export interface MediaGeneratorPlugin { */ startGeneration: (request: GenerationRequest) => Promise; /** - * Get WebSocket URL for tracking progress + * Get WebSocket URL for tracking progress (empty string = uses polling) */ getWebSocketUrl: () => string; /** * Get client-side WebSocket handler for this provider */ webSocketHandler: WebSocketHandler; + /** + * Check status of a generation task (for polling-based providers) + * Returns null if provider uses WebSocket instead of polling + */ + checkStatus?: (socketAccessToken: string) => Promise; }