From 2a036687b89dcef384064d18bcad02e058c874b6 Mon Sep 17 00:00:00 2001 From: En-gui Date: Thu, 22 Jan 2026 13:20:36 +0900 Subject: [PATCH 1/5] fix merge --- package-lock.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 3e968b1..ccfbace 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14035,7 +14035,6 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" From a4ce80ea78712ccb04f52e2276f272cd2e523a9e Mon Sep 17 00:00:00 2001 From: En-gui Date: Fri, 30 Jan 2026 04:54:37 +0900 Subject: [PATCH 2/5] timeline enhanced --- src/components/video-editor/TimelineRuler.tsx | 43 +-- src/components/video-editor/VideoTimeline.tsx | 311 ++++++++++-------- 2 files changed, 196 insertions(+), 158 deletions(-) diff --git a/src/components/video-editor/TimelineRuler.tsx b/src/components/video-editor/TimelineRuler.tsx index 6041598..de38dfe 100644 --- a/src/components/video-editor/TimelineRuler.tsx +++ b/src/components/video-editor/TimelineRuler.tsx @@ -32,40 +32,25 @@ const MIN_MAJOR_SPACING_PX = 96; const EPSILON = 0.000_01; function formatTickLabel(seconds: number, majorInterval: number) { - if (seconds < EPSILON) { - return "0s"; + if (Math.abs(seconds) < EPSILON) { + return "0:00"; } - if (seconds >= 3600) { - const hours = seconds / 3600; - const decimals = majorInterval < 3600 ? 1 : 0; - return `${hours.toFixed(decimals)}h`; - } - - if (seconds >= 60) { - const minutes = seconds / 60; - const decimals = majorInterval < 60 ? 1 : 0; - return `${minutes.toFixed(decimals)}m`; - } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; - if (seconds >= 1) { - const isNearInteger = Math.abs(seconds - Math.round(seconds)) < 0.005; - const decimals = - isNearInteger && majorInterval >= 1 - ? 0 - : majorInterval < 1 - ? Math.ceil(-Math.log10(majorInterval)) - : Math.min( - 2, - Math.max( - 1, - Math.ceil(-Math.log10(seconds - Math.floor(seconds))), - ), - ); - return `${seconds.toFixed(decimals)}s`; + // Decide on decimal places based on interval + // If interval is less than 1 second, we likely need decimals + if (majorInterval < 1) { + // Show decimals - e.g. 0:00.50 + // Keep seconds part fixed width if possible, or just standard numeric + const s = remainingSeconds.toFixed(2).padStart(5, '0'); + return `${minutes}:${s}`; } - return `${Math.round(seconds * 1000)}ms`; + // Integer seconds - e.g. 0:10, 1:05 + const s = Math.floor(remainingSeconds).toString().padStart(2, '0'); + return `${minutes}:${s}`; } function chooseMajorInterval(pixelsPerSecond: number) { diff --git a/src/components/video-editor/VideoTimeline.tsx b/src/components/video-editor/VideoTimeline.tsx index 64e77d4..2217e24 100644 --- a/src/components/video-editor/VideoTimeline.tsx +++ b/src/components/video-editor/VideoTimeline.tsx @@ -35,7 +35,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ className, ...props }: VideoTimelineProps) { - const { setCurrentTimestamp, addKeyframe, addTrack, updateKeyframe, removeKeyframe, player, setZoom, clearSelection } = useStudio(); + const { setCurrentTimestamp, addKeyframe, addTrack, updateKeyframe, removeKeyframe, player, setZoom, clearSelection, updateProject } = useStudio(); const { t } = useI18n(); const timelineRef = useRef(null); const [validationError, setValidationError] = useState(null); @@ -47,6 +47,14 @@ export const VideoTimeline = React.memo(function VideoTimeline({ const timelineWidth = useMemo(() => pixelsPerSecond * durationSeconds, [pixelsPerSecond, durationSeconds]); const pixelsPerMs = useMemo(() => pixelsPerSecond / 1000, [pixelsPerSecond]); + const [containerWidth, setContainerWidth] = useState(0); + + // Calculate actual display width (max of content and container) + const displayWidth = useMemo(() => Math.max(timelineWidth, containerWidth), [timelineWidth, containerWidth]); + + // Calculate visual duration for Ruler to match the stretched width + const visualDuration = useMemo(() => displayWidth / pixelsPerSecond, [displayWidth, pixelsPerSecond]); + // Sort tracks by type order - memoized const sortedTracks = useMemo(() => { return [...tracks].sort((a, b) => { @@ -65,6 +73,32 @@ export const VideoTimeline = React.memo(function VideoTimeline({ [currentTimestamp, pixelsPerSecond] ); + // Auto-expand timeline duration based on content + useEffect(() => { + if (!project) return; + + const bufferMs = 60000; // 60 seconds buffer to allow "infinite" feel + const minDurationMs = 60000; // 60 seconds minimum base + + // Find the end time of the last keyframe + let maxContentTime = 0; + Object.values(keyframes).flat().forEach(kf => { + const outputEnd = kf.timestamp + kf.duration; + if (outputEnd > maxContentTime) { + maxContentTime = outputEnd; + } + }); + + // Desired duration is content end + buffer + const desiredDuration = Math.max(minDurationMs, maxContentTime + bufferMs); + + // Only update if difference is significant (> 1s) to prevent jitter/loops + if (Math.abs(project.duration - desiredDuration) > 1000) { + // console.log('VideoTimeline: Auto-expanding duration from', project.duration, 'to', desiredDuration); + updateProject(project.id, { duration: desiredDuration }); + } + }, [keyframes, project.duration, project.id, updateProject]); + const handleTimelineClick = useCallback((event: MouseEvent) => { const rect = timelineRef.current?.getBoundingClientRect(); if (!rect) return; @@ -72,16 +106,17 @@ export const VideoTimeline = React.memo(function VideoTimeline({ // Check if click was on empty space (not on a keyframe) const target = event.target as HTMLElement; const isKeyframeClick = target.closest('[aria-checked]'); - + // Clear selection when clicking on empty space if (!isKeyframeClick) { clearSelection(); } - const relativeX = event.clientX - rect.left; + const scrollLeft = timelineRef.current?.scrollLeft || 0; + const relativeX = (event.clientX - rect.left) + scrollLeft; // Convert pixels to seconds const timestamp = relativeX / pixelsPerSecond; - + // Clamp timestamp to valid range const clampedTimestamp = Math.max(0, Math.min(durationSeconds, timestamp)); setCurrentTimestamp(clampedTimestamp); @@ -92,6 +127,20 @@ export const VideoTimeline = React.memo(function VideoTimeline({ } }, [pixelsPerSecond, durationSeconds, setCurrentTimestamp, player, clearSelection]); + // Observer for container width + useEffect(() => { + if (!timelineRef.current) return; + + const observer = new ResizeObserver((entries) => { + for (const entry of entries) { + setContainerWidth(entry.contentRect.width); + } + }); + + observer.observe(timelineRef.current); + return () => observer.disconnect(); + }, []); + const getTrackIdForMediaType = useCallback(async (mediaType: string): Promise => { // Map media type to track type let trackType: 'video' | 'music' | 'voiceover'; @@ -99,8 +148,10 @@ export const VideoTimeline = React.memo(function VideoTimeline({ trackType = 'video'; } else if (mediaType === 'music') { trackType = 'music'; - } else { + } else if (mediaType === 'tts' || mediaType === 'voiceover') { trackType = 'voiceover'; + } else { + trackType = 'voiceover'; // fallback } // Find existing track of this type @@ -128,10 +179,10 @@ export const VideoTimeline = React.memo(function VideoTimeline({ return new Promise((resolve) => { const TIMEOUT_MS = 15000; // Increased timeout for blob URLs let resolved = false; - + // Normalize URL to handle relative paths (especially on Windows) const normalizedUrl = normalizeUrl(url); - + const resolveOnce = (duration: number, source: string) => { if (!resolved) { resolved = true; @@ -139,17 +190,17 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolve(duration); } }; - + // Timeout fallback const timeoutId = setTimeout(() => { console.warn(`Media duration detection timed out for: ${normalizedUrl}`); resolveOnce(5000, 'timeout'); }, TIMEOUT_MS); - + if (type === 'music' || type === 'voiceover') { const audio = new Audio(); audio.preload = 'auto'; // Changed from 'metadata' to 'auto' for better blob URL support - + // Try multiple events for duration detection const handleDurationChange = () => { if (audio.duration && isFinite(audio.duration) && audio.duration > 0) { @@ -158,7 +209,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'durationchange'); } }; - + const handleLoadedMetadata = () => { if (audio.duration && isFinite(audio.duration) && audio.duration > 0) { clearTimeout(timeoutId); @@ -166,7 +217,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'loadedmetadata'); } }; - + const handleCanPlayThrough = () => { if (audio.duration && isFinite(audio.duration) && audio.duration > 0) { clearTimeout(timeoutId); @@ -174,18 +225,18 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'canplaythrough'); } }; - + audio.addEventListener('durationchange', handleDurationChange); audio.addEventListener('loadedmetadata', handleLoadedMetadata); audio.addEventListener('canplaythrough', handleCanPlayThrough); audio.addEventListener('loadeddata', handleCanPlayThrough); - + audio.addEventListener('error', (e) => { clearTimeout(timeoutId); console.error('Audio duration detection error:', e); resolveOnce(5000, 'error'); }); - + audio.src = normalizedUrl; // Force load for some browsers (skip in test environment where load() is not implemented) if (typeof audio.load === 'function') { @@ -198,7 +249,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ } else if (type === 'video') { const video = document.createElement('video'); video.preload = 'auto'; // Changed from 'metadata' to 'auto' - + const handleDurationChange = () => { if (video.duration && isFinite(video.duration) && video.duration > 0) { clearTimeout(timeoutId); @@ -206,7 +257,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'durationchange'); } }; - + const handleLoadedMetadata = () => { if (video.duration && isFinite(video.duration) && video.duration > 0) { clearTimeout(timeoutId); @@ -214,7 +265,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'loadedmetadata'); } }; - + const handleCanPlayThrough = () => { if (video.duration && isFinite(video.duration) && video.duration > 0) { clearTimeout(timeoutId); @@ -222,18 +273,18 @@ export const VideoTimeline = React.memo(function VideoTimeline({ resolveOnce(durationMs, 'canplaythrough'); } }; - + video.addEventListener('durationchange', handleDurationChange); video.addEventListener('loadedmetadata', handleLoadedMetadata); video.addEventListener('canplaythrough', handleCanPlayThrough); video.addEventListener('loadeddata', handleCanPlayThrough); - + video.addEventListener('error', (e) => { clearTimeout(timeoutId); console.error('Video duration detection error:', e); resolveOnce(5000, 'error'); }); - + video.src = normalizedUrl; // Force load for some browsers (skip in test environment where load() is not implemented) if (typeof video.load === 'function') { @@ -254,7 +305,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ event.preventDefault(); setValidationError(null); setNotification(null); - + try { const mediaDataStr = event.dataTransfer.getData('application/json'); if (!mediaDataStr) return; @@ -272,17 +323,20 @@ export const VideoTimeline = React.memo(function VideoTimeline({ const mediaUrl = rawMediaUrl ? normalizeUrl(rawMediaUrl) : null; const mediaId = rawMediaData.id || rawMediaData.jobId || `media-${Date.now()}`; const mediaName = rawMediaData.prompt || rawMediaData.audioName || rawMediaData.name || ''; - - // Map 'audio' type to 'music' - user can drag to voiceover track later + + // Map 'audio' to 'music', and 'tts' to 'voiceover' let normalizedType: 'image' | 'video' | 'music' | 'voiceover' = mediaType; if (mediaType === 'audio') { normalizedType = 'music'; + } else if (mediaType === 'tts') { + normalizedType = 'voiceover'; } - + console.log('Drop data normalized:', { mediaType, normalizedType, mediaUrl, rawMediaUrl, mediaId, mediaName }); - const relativeX = event.clientX - rect.left; - const timestamp = Math.max(0, (relativeX / timelineWidth) * durationSeconds * 1000); + const scrollLeft = timelineRef.current?.scrollLeft || 0; + const relativeX = (event.clientX - rect.left) + scrollLeft; + const timestamp = Math.max(0, (relativeX / displayWidth) * visualDuration * 1000); // Get or create appropriate track const trackId = await getTrackIdForMediaType(normalizedType); @@ -291,7 +345,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ // This ensures the keyframe duration matches the actual media length let duration: number; let originalDuration: number; - + if (mediaUrl && (normalizedType === 'music' || normalizedType === 'voiceover' || normalizedType === 'video')) { // For audio and video, always detect duration from the actual file console.log(`Detecting duration for ${normalizedType} from: ${mediaUrl}`); @@ -316,7 +370,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ data: { type: normalizedType, mediaId: mediaId, - url: mediaUrl, + url: mediaUrl || '', prompt: rawMediaData.prompt, originalDuration, // Store original duration for waveform scaling }, @@ -336,7 +390,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ if (normalizedType === 'video' && mediaUrl) { // Skip audio processing for blob URLs (browser-only, can't be processed server-side) const isBlobUrl = mediaUrl.startsWith('blob:'); - + if (isBlobUrl) { console.log('Blob URL detected, skipping server-side audio processing:', mediaUrl); // For blob URLs, use client-side detection only @@ -373,88 +427,88 @@ export const VideoTimeline = React.memo(function VideoTimeline({ // Fallback to client-side detection hasAudio = await hasAudioTrack(mediaUrl); } - - if (hasAudio) { - console.log('Video has audio, creating muted version...'); - // Create muted version of the video for the video track - const mutedResponse = await fetch('/api/video-tracks/create-muted', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ videoPath: mediaUrl }), - }); - if (mutedResponse.ok) { - const { mutedVideoPath } = await mutedResponse.json(); - console.log('[VideoTimeline] Received mutedVideoPath from API:', mutedVideoPath); - // Normalize URL to handle relative paths (especially on Windows) - finalVideoUrl = normalizeUrl(mutedVideoPath); - console.log('[VideoTimeline] Normalized mutedVideoPath:', finalVideoUrl); - console.log('✓ Using muted video for video track:', finalVideoUrl); - } else { - const errorData = await mutedResponse.json(); - console.error('Failed to create muted video:', errorData); - console.warn('Using original video with audio'); - } + if (hasAudio) { + console.log('Video has audio, creating muted version...'); + // Create muted version of the video for the video track + const mutedResponse = await fetch('/api/video-tracks/create-muted', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ videoPath: mediaUrl }), + }); - console.log('Extracting audio from video...'); + if (mutedResponse.ok) { + const { mutedVideoPath } = await mutedResponse.json(); + console.log('[VideoTimeline] Received mutedVideoPath from API:', mutedVideoPath); + // Normalize URL to handle relative paths (especially on Windows) + finalVideoUrl = normalizeUrl(mutedVideoPath); + console.log('[VideoTimeline] Normalized mutedVideoPath:', finalVideoUrl); + console.log('✓ Using muted video for video track:', finalVideoUrl); + } else { + const errorData = await mutedResponse.json(); + console.error('Failed to create muted video:', errorData); + console.warn('Using original video with audio'); + } + + console.log('Extracting audio from video...'); + + // Extract audio asynchronously + // Note: This requires server-side API call since FFmpeg runs on server + const audioResponse = await fetch('/api/video-tracks/extract-audio', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ videoPath: mediaUrl }), + }); - // Extract audio asynchronously - // Note: This requires server-side API call since FFmpeg runs on server - const audioResponse = await fetch('/api/video-tracks/extract-audio', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ videoPath: mediaUrl }), - }); + if (!audioResponse.ok) { + throw new Error('Failed to extract audio from video'); + } - if (!audioResponse.ok) { - throw new Error('Failed to extract audio from video'); - } - - const { audioPath } = await audioResponse.json(); - console.log('[VideoTimeline] Received audioPath from API:', audioPath); - // Normalize URL to handle relative paths (especially on Windows) - const normalizedAudioPath = normalizeUrl(audioPath); - console.log('[VideoTimeline] Normalized audioPath:', normalizedAudioPath); - console.log('✓ Audio extracted:', normalizedAudioPath); - - // Find available audio track - const audioTrackId = findAvailableAudioTrack( - tracks, - keyframes, - timestamp, - duration - ); - - console.log('Available audio track:', audioTrackId); - - if (audioTrackId) { - // Determine track type for audio keyframe - const audioTrack = tracks.find(t => t.id === audioTrackId); - const audioType = audioTrack?.type === 'voiceover' ? 'voiceover' : 'music'; - - // Add synchronized audio keyframe - await addKeyframe({ - trackId: audioTrackId, + const { audioPath } = await audioResponse.json(); + console.log('[VideoTimeline] Received audioPath from API:', audioPath); + // Normalize URL to handle relative paths (especially on Windows) + const normalizedAudioPath = normalizeUrl(audioPath); + console.log('[VideoTimeline] Normalized audioPath:', normalizedAudioPath); + console.log('✓ Audio extracted:', normalizedAudioPath); + + // Find available audio track + const audioTrackId = findAvailableAudioTrack( + tracks, + keyframes, timestamp, - duration, - data: { - type: audioType, - mediaId: `${mediaId}-audio`, - url: normalizedAudioPath, - prompt: `${mediaName} (audio)`, - originalDuration: duration, - }, - }); - - console.log('✓ Audio keyframe added to track:', audioTrackId); - } else { - console.warn('No available audio track found'); - // Show warning notification - setNotification({ - message: 'No available audio track for extracted audio', - type: 'warning', - }); - } + duration + ); + + console.log('Available audio track:', audioTrackId); + + if (audioTrackId) { + // Determine track type for audio keyframe + const audioTrack = tracks.find(t => t.id === audioTrackId); + const audioType = audioTrack?.type === 'voiceover' ? 'voiceover' : 'music'; + + // Add synchronized audio keyframe + await addKeyframe({ + trackId: audioTrackId, + timestamp, + duration, + data: { + type: audioType, + mediaId: `${mediaId}-audio`, + url: normalizedAudioPath, + prompt: `${mediaName} (audio)`, + originalDuration: duration, + }, + }); + + console.log('✓ Audio keyframe added to track:', audioTrackId); + } else { + console.warn('No available audio track found'); + // Show warning notification + setNotification({ + message: 'No available audio track for extracted audio', + type: 'warning', + }); + } } else { console.log('Video has no audio, skipping extraction'); } @@ -470,7 +524,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ } // Update keyframe data with final video URL (muted if audio was present) - keyframeData.data.url = finalVideoUrl; + keyframeData.data.url = finalVideoUrl || ''; // Add video keyframe with muted video URL if audio was present await addKeyframe(keyframeData); @@ -479,7 +533,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ setValidationError(errorMessage); console.error('Failed to handle drop:', error); } - }, [timelineWidth, durationSeconds, getTrackIdForMediaType, addKeyframe, getMediaDuration, tracks, keyframes]); + }, [displayWidth, visualDuration, getTrackIdForMediaType, addKeyframe, getMediaDuration, tracks, keyframes]); const handleDragOver = useCallback((event: DragEvent) => { event.preventDefault(); @@ -491,7 +545,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ // Find the keyframe in all tracks let sourceKeyframe: VideoKeyFrame | null = null; let sourceTrackId: string | null = null; - + for (const [trackId, trackKeyframes] of Object.entries(keyframes)) { const found = trackKeyframes.find(kf => kf.id === keyframeId); if (found) { @@ -500,12 +554,12 @@ export const VideoTimeline = React.memo(function VideoTimeline({ break; } } - + if (!sourceKeyframe || !sourceTrackId) { console.warn('Source keyframe not found:', keyframeId); return; } - + // If same track, just update timestamp (position change within track) if (sourceTrackId === targetTrackId) { if (Math.round(timestamp) !== sourceKeyframe.timestamp) { @@ -514,17 +568,17 @@ export const VideoTimeline = React.memo(function VideoTimeline({ } return; } - + // Get target track to update media type const targetTrack = tracks.find(t => t.id === targetTrackId); if (!targetTrack) { console.warn('Target track not found:', targetTrackId); return; } - + // Update the keyframe's trackId and media type const newMediaType = targetTrack.type === 'voiceover' ? 'voiceover' : 'music'; - + // Remove from source and add to target await removeKeyframe(keyframeId); await addKeyframe({ @@ -536,7 +590,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ type: newMediaType, }, }); - + console.log(`Moved keyframe ${keyframeId} from ${sourceTrackId} to ${targetTrackId} at ${timestamp}ms`); }, [keyframes, tracks, removeKeyframe, addKeyframe, updateKeyframe]); @@ -559,12 +613,12 @@ export const VideoTimeline = React.memo(function VideoTimeline({ // Or if it's a pinch gesture (ctrlKey is true for trackpad pinch on macOS) if (event.altKey || event.ctrlKey) { event.preventDefault(); - + // Calculate zoom delta // For trackpad pinch, deltaY is typically smaller and smoother // For mouse wheel, deltaY is larger (usually 100 or -100) const delta = -event.deltaY * ZOOM_SENSITIVITY; - + // Apply zoom with exponential scaling for smoother feel const newZoom = Math.max(MIN_ZOOM, Math.min(MAX_ZOOM, zoom * (1 + delta))); setZoom(newZoom); @@ -647,7 +701,7 @@ export const VideoTimeline = React.memo(function VideoTimeline({ > {/* Validation Error Display */} {validationError && ( -
-