burn($disk, $relativePath, $spokenLine); } catch (Throwable $e) { $this->warn($relativePath, $e::class); return false; } } /** * @return list */ public function phrases(string $spokenLine): array { $spokenLine = trim(preg_replace('/\s+/u', ' ', $spokenLine) ?? ''); if ($spokenLine === '') { return []; } $clauses = preg_split('/(?<=[.!?])\s+/u', $spokenLine, flags: PREG_SPLIT_NO_EMPTY) ?: []; $phrases = []; foreach ($clauses as $clause) { $words = preg_split('/\s+/u', trim($clause), flags: PREG_SPLIT_NO_EMPTY) ?: []; foreach (array_chunk($words, self::WORDS_PER_PHRASE) as $chunk) { $phrases[] = implode(' ', $chunk); } } return $phrases; } /** * @param list $speechIntervals * @return list */ public function cues(string $spokenLine, array $speechIntervals): array { $phrases = $this->phrases($spokenLine); $intervals = $this->usableIntervals($speechIntervals); if ($phrases === [] || $intervals === []) { return []; } $weights = array_map( fn (string $phrase): int => max(1, VideoDialogue::wordCount($phrase)), $phrases, ); $totalWeight = array_sum($weights); $totalSpeech = 0.0; foreach ($intervals as [$start, $end]) { $totalSpeech += $end - $start; } $cues = []; $cursor = 0.0; foreach ($phrases as $index => $phrase) { $span = ($weights[$index] / $totalWeight) * $totalSpeech; foreach ($this->place($cursor, $cursor + $span, $intervals) as [$start, $end]) { $cues[] = [ 'text' => $phrase, 'start' => $start, 'end' => $end, ]; } $cursor += $span; } return $cues; } /** * Place each scripted phrase on the words Whisper heard, and leave later ad-lib speech uncaptioned. * * @param list $heard * @return list */ public function cuesFromHeardWords(string $spokenLine, array $heard): array { $phrases = $this->phrases($spokenLine); if ($phrases === [] || $heard === []) { return []; } $scriptWords = []; foreach ($phrases as $phrase) { foreach (preg_split('/\s+/u', $phrase, flags: PREG_SPLIT_NO_EMPTY) ?: [] as $word) { $scriptWords[] = $word; } } $matched = $this->matchHeardWords($scriptWords, $heard); $timedCount = count(array_filter($matched)); if ($timedCount < (int) ceil(count($scriptWords) * 0.5)) { return []; } $cues = []; $offset = 0; foreach ($phrases as $phrase) { $count = VideoDialogue::wordCount($phrase); $slice = array_slice($matched, $offset, $count); $offset += $count; $timed = array_values(array_filter($slice)); if ($timed === []) { continue; } $cues[] = [ 'text' => $phrase, 'start' => $timed[0]['start'], 'end' => $timed[array_key_last($timed)]['end'], ]; } return $cues; } /** * Keep captions inside the time the scripted line should take, so trailing speech does not stretch them. * * @param list $speechIntervals * @return list */ public function limitSpeechToLine(array $speechIntervals, string $spokenLine): array { $words = VideoDialogue::wordCount($spokenLine); if ($words === 0) { return []; } $pace = max(0.5, (float) config('creative.spoken_words_per_second', 2.2)); $remaining = ($words / $pace) + 0.35; $capped = []; foreach ($this->usableIntervals($speechIntervals) as [$start, $end]) { if ($remaining <= 0.05) { break; } $take = min($end - $start, $remaining); $capped[] = [$start, $start + $take]; $remaining -= $take; } return $capped; } /** * @return list */ public function speechIntervals(string $ffmpegLog, float $duration): array { $duration = max(0.0, $duration); $silences = []; $pendingStart = null; foreach (preg_split('/\R/', $ffmpegLog) ?: [] as $line) { if (preg_match('/silence_start:\s*(-?\d+(?:\.\d+)?)/', $line, $matches) === 1) { $pendingStart = (float) $matches[1]; } if (preg_match('/silence_end:\s*(-?\d+(?:\.\d+)?)/', $line, $matches) === 1 && $pendingStart !== null) { $silences[] = [$pendingStart, (float) $matches[1]]; $pendingStart = null; } } if ($pendingStart !== null) { $silences[] = [$pendingStart, $duration]; } $speech = []; $cursor = 0.0; foreach ($silences as [$start, $end]) { $start = min($duration, max(0.0, $start)); $end = min($duration, max($start, $end)); if ($start - $cursor >= 0.12) { $speech[] = [$cursor, $start]; } $cursor = max($cursor, $end); } if ($duration - $cursor >= 0.12) { $speech[] = [$cursor, $duration]; } if ($speech === [] && $duration > 0) { return [[0.0, $duration]]; } return $speech; } /** * @param list $cues */ public function assDocument(array $cues, int $width, int $height): string { $fontSize = max(32, (int) round(min($width, $height) * 0.062)); $outline = max(4, (int) round($fontSize * 0.14)); $marginV = max(16, (int) round($height * 0.07)); $fontName = $this->fontName(); $events = ''; foreach ($cues as $cue) { $events .= sprintf( "Dialogue: 0,%s,%s,Caption,,0,0,0,,%s\n", $this->assTime($cue['start']), $this->assTime($cue['end']), $this->escapeAss($cue['text']), ); } return "[Script Info]\n". "ScriptType: v4.00+\n". "PlayResX: {$width}\n". "PlayResY: {$height}\n". "WrapStyle: 0\n\n". "[V4+ Styles]\n". "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n". "Style: Caption,{$fontName},{$fontSize},&H00FFFFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,{$outline},0,2,40,40,{$marginV},1\n\n". "[Events]\n". "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n". $events; } private function burn(string $disk, string $relativePath, string $spokenLine): bool { if (trim($spokenLine) === '') { return false; } $ffmpeg = $this->ffmpegBinary(); if ($ffmpeg === null) { $this->warn($relativePath, 'ffmpeg is not available'); return false; } if (config("filesystems.disks.{$disk}.driver") !== 'local') { $this->warn($relativePath, 'captions require a local creative disk'); return false; } $storage = Storage::disk($disk); if (! $storage->exists($relativePath)) { $this->warn($relativePath, 'video file is missing'); return false; } $absolute = $storage->path($relativePath); $header = file_get_contents($absolute, false, null, 0, 12); if (! is_string($header) || ! str_contains($header, 'ftyp')) { $this->warn($relativePath, 'file is not an mp4'); return false; } $log = $this->analyze($ffmpeg, $absolute); $dimensions = $this->dimensions($log); if ($dimensions === null) { $this->warn($relativePath, 'could not read the video size'); return false; } [$width, $height, $duration] = $dimensions; $heard = $this->hearWords($ffmpeg, $absolute, $spokenLine); $cues = $this->cuesFromHeardWords($spokenLine, $heard); $timing = 'words'; if ($cues === []) { $cues = $this->cues($spokenLine, $this->limitSpeechToLine($this->speechIntervals($log, $duration), $spokenLine)); $timing = 'speech-window'; } if ($cues === []) { $this->warn($relativePath, 'spoken line produced no captions'); return false; } $work = sys_get_temp_dir().'/captions-'.bin2hex(random_bytes(4)); if (! mkdir($work) && ! is_dir($work)) { $this->warn($relativePath, 'could not create a caption workspace'); return false; } $assPath = $work.'/captions.ass'; $output = $work.'/out.mp4'; try { file_put_contents($assPath, $this->assDocument($cues, $width, $height)); $fontsDir = $this->fontsDirectory(); $encoded = $this->encode($ffmpeg, $absolute, $assPath, $fontsDir, $output, true) || $this->encode($ffmpeg, $absolute, $assPath, $fontsDir, $output, false); if (! $encoded || ! is_file($output) || filesize($output) === 0) { $this->warn($relativePath, 'ffmpeg could not burn captions'); return false; } if (! rename($output, $absolute)) { $this->warn($relativePath, 'could not replace the video with the captioned file'); return false; } Log::channel('ai_video')->info('Spoken captions burned into the video', [ 'path' => $relativePath, 'cues' => count($cues), 'timing' => $timing, ]); return true; } finally { @unlink($assPath); @unlink($output); @rmdir($work); } } private function analyze(string $ffmpeg, string $absolute): string { $result = Process::timeout(90)->run([ $ffmpeg, '-hide_banner', '-i', $absolute, '-af', 'highpass=f=200,lowpass=f=3400,silencedetect=noise=-32dB:d=0.25', '-f', 'null', '-', ]); return $result->errorOutput().$result->output(); } /** * @param list $scriptWords * @param list $heard * @return list */ private function matchHeardWords(array $scriptWords, array $heard): array { $cursor = 0; $heardCount = count($heard); $matched = []; foreach ($scriptWords as $scriptWord) { $needle = $this->normalizeWord($scriptWord); $found = null; if ($needle !== '') { $limit = min($heardCount, $cursor + 4); for ($index = $cursor; $index < $limit; $index++) { if ($this->normalizeWord((string) $heard[$index]['word']) === $needle) { $found = $index; break; } } } if ($found === null) { $matched[] = null; continue; } $matched[] = [ 'start' => (float) $heard[$found]['start'], 'end' => (float) $heard[$found]['end'], ]; $cursor = $found + 1; } return $matched; } /** * @return list */ private function hearWords(string $ffmpeg, string $absolute, string $spokenLine): array { if (app()->runningUnitTests() || blank(config('ai.providers.openai.key'))) { return []; } $wav = sys_get_temp_dir().'/caption-audio-'.bin2hex(random_bytes(4)).'.wav'; try { $extracted = Process::timeout(60)->run([ $ffmpeg, '-y', '-hide_banner', '-loglevel', 'error', '-i', $absolute, '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', $wav, ]); if (! $extracted->successful() || ! is_file($wav)) { return []; } $response = Http::withToken((string) config('ai.providers.openai.key')) ->timeout(60) ->attach('file', (string) file_get_contents($wav), 'speech.wav', ['Content-Type' => 'audio/wav']) ->post(rtrim((string) config('ai.providers.openai.url'), '/').'/audio/transcriptions', [ 'model' => 'whisper-1', 'response_format' => 'verbose_json', 'timestamp_granularities[]' => 'word', 'prompt' => mb_substr($spokenLine, 0, 800), ]); if ($response->failed()) { $this->warn('caption-audio', 'word timing request failed'); return []; } return $this->wordsFromTranscript($response->json() ?? []); } catch (Throwable) { return []; } finally { if (is_file($wav)) { unlink($wav); } } } /** * @param array $payload * @return list */ private function wordsFromTranscript(array $payload): array { $words = []; foreach ($payload['words'] ?? [] as $word) { if (! is_array($word) || ! isset($word['word'], $word['start'], $word['end'])) { continue; } $words[] = [ 'word' => (string) $word['word'], 'start' => (float) $word['start'], 'end' => (float) $word['end'], ]; } if ($words !== []) { return $words; } foreach ($payload['segments'] ?? [] as $segment) { if (! is_array($segment)) { continue; } $parts = preg_split('/\s+/u', trim((string) ($segment['text'] ?? '')), flags: PREG_SPLIT_NO_EMPTY) ?: []; $start = (float) ($segment['start'] ?? 0); $end = (float) ($segment['end'] ?? $start); $count = count($parts); if ($count === 0 || $end <= $start) { continue; } $step = ($end - $start) / $count; foreach ($parts as $index => $part) { $words[] = [ 'word' => $part, 'start' => $start + ($index * $step), 'end' => $start + (($index + 1) * $step), ]; } } return $words; } private function normalizeWord(string $word): string { return strtolower((string) preg_replace('/[^a-z0-9]+/i', '', $word)); } /** * @return array{0: int, 1: int, 2: float}|null */ private function dimensions(string $log): ?array { if (preg_match('/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/', $log, $duration) !== 1) { return null; } if (preg_match('/Video:[^\n]*?(\d{2,5})x(\d{2,5})/', $log, $size) !== 1) { return null; } $seconds = ((int) $duration[1] * 3600) + ((int) $duration[2] * 60) + (float) $duration[3]; if ($seconds <= 0) { return null; } return [(int) $size[1], (int) $size[2], $seconds]; } private function encode(string $ffmpeg, string $input, string $assPath, ?string $fontsDir, string $output, bool $copyAudio): bool { $filter = 'subtitles='.$this->filterPath($assPath); if ($fontsDir !== null) { $filter .= ':fontsdir='.$this->filterPath($fontsDir); } $command = [ $ffmpeg, '-y', '-hide_banner', '-loglevel', 'error', '-i', $input, '-vf', $filter, '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', '-pix_fmt', 'yuv420p', ]; if ($copyAudio) { $command[] = '-c:a'; $command[] = 'copy'; } else { $command[] = '-c:a'; $command[] = 'aac'; } $command[] = '-movflags'; $command[] = '+faststart'; $command[] = $output; return Process::timeout(180)->run($command)->successful(); } private function ffmpegBinary(): ?string { $configured = config('creative.ffmpeg_binary'); if (is_string($configured) && $configured !== '') { return $this->isRunnable($configured) ? $configured : null; } foreach (['ffmpeg', '/home/mehak/bin/ffmpeg', '/usr/bin/ffmpeg', '/usr/local/bin/ffmpeg'] as $candidate) { if ($this->isRunnable($candidate)) { return $candidate; } } return null; } private function isRunnable(string $binary): bool { if (str_contains($binary, '/')) { return is_executable($binary); } $result = Process::run(['bash', '-lc', 'command -v '.escapeshellarg($binary)]); return $result->successful() && trim($result->output()) !== ''; } private function fontsDirectory(): ?string { foreach ([ '/usr/share/fonts/truetype/liberation', '/usr/share/fonts/truetype/dejavu', ] as $directory) { if (is_dir($directory)) { return $directory; } } return null; } private function fontName(): string { $directory = $this->fontsDirectory(); return $directory !== null && str_contains($directory, 'dejavu') ? 'DejaVu Sans' : 'Liberation Sans'; } private function filterPath(string $path): string { return str_replace(['\\', ':', "'"], ['\\\\', '\\:', "\\'"], $path); } private function escapeAss(string $text): string { return str_replace( ['\\', '{', '}'], ['\\\\', '\\{', '\\}'], str_replace("\n", ' ', $text), ); } private function assTime(float $seconds): string { $seconds = max(0, $seconds); $hours = (int) floor($seconds / 3600); $minutes = (int) floor(fmod($seconds, 3600) / 60); $remain = $seconds - ($hours * 3600) - ($minutes * 60); return sprintf('%d:%02d:%05.2f', $hours, $minutes, $remain); } /** * @param list $intervals * @return list */ private function usableIntervals(array $intervals): array { $usable = []; foreach ($intervals as [$start, $end]) { if ($end - $start >= 0.12) { $usable[] = [(float) $start, (float) $end]; } } return $usable; } /** * @param list $intervals * @return list */ private function place(float $fromSpeech, float $toSpeech, array $intervals): array { $segments = []; $consumed = 0.0; foreach ($intervals as [$start, $end]) { $length = $end - $start; $intervalFrom = $consumed; $intervalTo = $consumed + $length; $overlapFrom = max($fromSpeech, $intervalFrom); $overlapTo = min($toSpeech, $intervalTo); if ($overlapTo - $overlapFrom > 0.05) { $segments[] = [ $start + ($overlapFrom - $intervalFrom), $start + ($overlapTo - $intervalFrom), ]; } $consumed += $length; if ($consumed >= $toSpeech) { break; } } return $segments; } private function warn(string $relativePath, string $reason): void { Log::channel('ai_video')->warning('Spoken captions were not added', [ 'path' => $relativePath, 'reason' => mb_substr($reason, 0, 500), ]); } }