Openscreen
diff --git a/src/components/launch/AreaSelector.module.css b/src/components/launch/AreaSelector.module.css
new file mode 100644
index 0000000..50d082a
--- /dev/null
+++ b/src/components/launch/AreaSelector.module.css
@@ -0,0 +1,87 @@
+.container {
+ position: fixed;
+ inset: 0;
+ cursor: crosshair;
+ user-select: none;
+}
+
+.instructions {
+ position: fixed;
+ top: 50%;
+ left: 50%;
+ transform: translate(-50%, -50%);
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ gap: 8px;
+ padding: 20px 32px;
+ background: rgba(0, 0, 0, 0.8);
+ border-radius: 12px;
+ color: #fff;
+ font-size: 14px;
+ pointer-events: none;
+ z-index: 10;
+}
+
+.hint {
+ font-size: 12px;
+ color: rgba(255, 255, 255, 0.5);
+}
+
+.overlay {
+ position: fixed;
+ inset: 0;
+ background: rgba(0, 0, 0, 0.5);
+ pointer-events: none;
+}
+
+.selection {
+ position: fixed;
+ border: 2px solid #34B27B;
+ background: transparent;
+ pointer-events: none;
+ box-shadow: 0 0 0 9999px rgba(0, 0, 0, 0.5);
+}
+
+.dimensions {
+ position: absolute;
+ bottom: -28px;
+ left: 50%;
+ transform: translateX(-50%);
+ padding: 4px 10px;
+ background: rgba(0, 0, 0, 0.8);
+ border-radius: 4px;
+ color: #fff;
+ font-size: 11px;
+ white-space: nowrap;
+}
+
+.handle {
+ position: absolute;
+ width: 10px;
+ height: 10px;
+ background: #34B27B;
+ border: 2px solid #fff;
+ border-radius: 2px;
+}
+
+.topLeft {
+ top: -5px;
+ left: -5px;
+}
+
+.topRight {
+ top: -5px;
+ right: -5px;
+}
+
+.bottomLeft {
+ bottom: -5px;
+ left: -5px;
+}
+
+.bottomRight {
+ bottom: -5px;
+ right: -5px;
+}
+
diff --git a/src/components/launch/AreaSelector.tsx b/src/components/launch/AreaSelector.tsx
new file mode 100644
index 0000000..989288f
--- /dev/null
+++ b/src/components/launch/AreaSelector.tsx
@@ -0,0 +1,109 @@
+import { useState, useRef, useEffect } from "react";
+import styles from "./AreaSelector.module.css";
+
+interface SelectionRect {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+export function AreaSelector() {
+ const [isSelecting, setIsSelecting] = useState(false);
+ const [startPoint, setStartPoint] = useState({ x: 0, y: 0 });
+ const [selection, setSelection] = useState
(null);
+ const containerRef = useRef(null);
+
+ const handleMouseDown = (e: React.MouseEvent) => {
+ setIsSelecting(true);
+ setStartPoint({ x: e.clientX, y: e.clientY });
+ setSelection({ x: e.clientX, y: e.clientY, width: 0, height: 0 });
+ };
+
+ const handleMouseMove = (e: React.MouseEvent) => {
+ if (!isSelecting) return;
+
+ const width = e.clientX - startPoint.x;
+ const height = e.clientY - startPoint.y;
+
+ setSelection({
+ x: width >= 0 ? startPoint.x : e.clientX,
+ y: height >= 0 ? startPoint.y : e.clientY,
+ width: Math.abs(width),
+ height: Math.abs(height),
+ });
+ };
+
+ const handleMouseUp = async () => {
+ setIsSelecting(false);
+ if (selection && selection.width > 50 && selection.height > 50) {
+ await window.electronAPI.selectAreaRegion(selection);
+ }
+ window.close();
+ };
+
+ useEffect(() => {
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (e.key === "Escape") {
+ window.close();
+ }
+ };
+ window.addEventListener("keydown", handleKeyDown);
+ return () => window.removeEventListener("keydown", handleKeyDown);
+ }, []);
+
+ return (
+
+
+ Drag to select the area you want to record
+ Press ESC to cancel
+
+
+ {selection && selection.width > 0 && selection.height > 0 && (
+ <>
+
+
+
+ {Math.round(selection.width)} × {Math.round(selection.height)}
+
+
+
+
+
+
+
+ )}
+
+ {Array.from({ length: bars }).map((_, i) => {
+ const barIndex = i + 1;
+ const threshold = barIndex / bars;
+ const isActive = audioLevel >= threshold;
+ const height = isActive ? Math.max(2, (audioLevel - (barIndex - 1) / bars) * bars * 2) : 2;
+
+ return (
+
+ );
+ })}
+
setIsHovered(true)}
+ onMouseLeave={() => setIsHovered(false)}
+ >
+
+
+
+
-
-
+ const truncateLabel = (label: string, maxLen: number) => {
+ return label.length > maxLen ? label.slice(0, maxLen) + "…" : label;
+ };
+
+ const selectedMicLabel = microphones.find(m => m.deviceId === selectedMicrophone)?.label || "No mic";
+ const selectedCameraLabel = cameras.find(c => c.deviceId === selectedCamera)?.label || "No camera";
+ const sourceTypeButtons: { type: SourceType; icon: typeof MdMonitor; label: string }[] = [
+ { type: "display", icon: MdMonitor, label: "Display" },
+ { type: "window", icon: MdWindow, label: "Window" },
+ ];
+
+ const isSourceSelected = (sourceId: string) => selectedSource?.id === sourceId;
+
+ return (
+
+
-
+
-
+
+
+
+
+
+
+ {showCameraDropdown && (
+
+
+ {cameras.map(cam => (
+
+ ))}
+
+ )}
+
+
+
+
+
+
+
+ {showMicDropdown && (
+
+
+ {microphones.map(mic => (
+
+ ))}
+
+ )}
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
- {/* Separator before hide/close buttons */}
-
-
-
-
+
+
+ {!recording && (
+
+ {currentSources.map(source => (
+
handleSourceSelect(source)}
+ onMouseEnter={() => setHoveredSource(source)}
+ onMouseLeave={() => setHoveredSource(null)}
+ >
+
+
+
+
+ {source.appIcon && sourceType === "window" && (
+
+ )}
+
{source.name}
+
+ {isSourceSelected(source.id) && (
+
+
+
+ )}
+
+ ))}
+
+ )}
);
}
diff --git a/src/components/launch/SourceSelector.module.css b/src/components/launch/SourceSelector.module.css
index e1aabf1..9c670ec 100644
--- a/src/components/launch/SourceSelector.module.css
+++ b/src/components/launch/SourceSelector.module.css
@@ -39,3 +39,62 @@
color: #a1a1aa;
font-size: 0.75rem;
}
+
+.microphoneSection {
+ margin-top: 16px;
+ padding: 12px;
+ background: rgba(38, 38, 48, 0.6);
+ border-radius: 10px;
+ border: 1px solid rgba(60, 60, 80, 0.22);
+}
+
+.microphoneHeader {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ margin-bottom: 8px;
+}
+
+.micIcon {
+ width: 16px;
+ height: 16px;
+ color: #71717a;
+}
+
+.micIconActive {
+ width: 16px;
+ height: 16px;
+ color: #34B27B;
+}
+
+.micLabel {
+ font-size: 0.8rem;
+ color: #d4d4d8;
+ font-weight: 500;
+}
+
+.micSelect {
+ width: 100%;
+ padding: 8px 12px;
+ background: rgba(24, 24, 32, 0.9);
+ border: 1px solid rgba(60, 60, 80, 0.3);
+ border-radius: 8px;
+ color: #f4f4f5;
+ font-size: 0.8rem;
+ cursor: pointer;
+ outline: none;
+ transition: border-color 0.2s;
+}
+
+.micSelect:hover {
+ border-color: rgba(60, 60, 80, 0.5);
+}
+
+.micSelect:focus {
+ border-color: #34B27B;
+}
+
+.micSelect option {
+ background: #18181b;
+ color: #f4f4f5;
+}
diff --git a/src/components/launch/SourceSelector.tsx b/src/components/launch/SourceSelector.tsx
index 61bc2eb..a24ea35 100644
--- a/src/components/launch/SourceSelector.tsx
+++ b/src/components/launch/SourceSelector.tsx
@@ -1,6 +1,6 @@
import { useState, useEffect } from "react";
import { Button } from "../ui/button";
-import { MdCheck } from "react-icons/md";
+import { MdCheck, MdMic, MdMicOff } from "react-icons/md";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs";
import { Card } from "../ui/card";
import styles from "./SourceSelector.module.css";
@@ -13,10 +13,17 @@ interface DesktopSource {
appIcon: string | null;
}
+interface AudioDevice {
+ deviceId: string;
+ label: string;
+}
+
export function SourceSelector() {
const [sources, setSources] = useState
([]);
const [selectedSource, setSelectedSource] = useState(null);
const [loading, setLoading] = useState(true);
+ const [microphones, setMicrophones] = useState([]);
+ const [selectedMicrophone, setSelectedMicrophone] = useState("none");
useEffect(() => {
async function fetchSources() {
@@ -48,12 +55,36 @@ export function SourceSelector() {
fetchSources();
}, []);
+ useEffect(() => {
+ async function fetchMicrophones() {
+ try {
+ await navigator.mediaDevices.getUserMedia({ audio: true });
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ const audioInputs = devices
+ .filter(device => device.kind === 'audioinput')
+ .map(device => ({
+ deviceId: device.deviceId,
+ label: device.label || `Microphone ${device.deviceId.slice(0, 5)}`
+ }));
+ setMicrophones(audioInputs);
+ } catch (error) {
+ console.error('Error loading microphones:', error);
+ }
+ }
+ fetchMicrophones();
+ }, []);
+
const screenSources = sources.filter(s => s.id.startsWith('screen:'));
const windowSources = sources.filter(s => s.id.startsWith('window:'));
const handleSourceSelect = (source: DesktopSource) => setSelectedSource(source);
const handleShare = async () => {
- if (selectedSource) await window.electronAPI.selectSource(selectedSource);
+ if (selectedSource) {
+ await window.electronAPI.selectSource({
+ ...selectedSource,
+ microphoneId: selectedMicrophone === "none" ? null : selectedMicrophone
+ });
+ }
};
if (loading) {
@@ -147,6 +178,29 @@ export function SourceSelector() {
+
+ {selectedMicrophone === "none" ? (
+
+ ) : (
+
+ )}
+ Microphone
+
+
+
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index 357adc9..552c94a 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -875,7 +875,35 @@ const VideoPlayback = forwardRef(({
className="hidden"
preload="metadata"
playsInline
- onLoadedMetadata={handleLoadedMetadata}
+ muted={false}
+ onLoadedMetadata={(e) => {
+ const video = e.currentTarget;
+ video.volume = 1;
+ video.muted = false;
+
+ try {
+ const audioTracks = (video as any).audioTracks;
+ if (audioTracks && audioTracks.length > 0) {
+ console.log(`[VideoPlayback] Video has ${audioTracks.length} audio track(s)`);
+ } else {
+ const stream = (video as any).captureStream?.();
+ if (stream) {
+ const audioTracks = stream.getAudioTracks();
+ if (audioTracks.length > 0) {
+ console.log(`[VideoPlayback] Video stream has ${audioTracks.length} audio track(s)`);
+ } else {
+ console.warn('[VideoPlayback] Video has no audio tracks');
+ }
+ } else {
+ console.warn('[VideoPlayback] Could not check audio tracks');
+ }
+ }
+ } catch (err) {
+ console.warn('[VideoPlayback] Error checking audio tracks:', err);
+ }
+
+ handleLoadedMetadata(e);
+ }}
onDurationChange={e => {
onDurationChange(e.currentTarget.duration);
}}
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index cfb2183..530c8a4 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -48,7 +48,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const stopRecording = useRef(() => {
if (mediaRecorder.current?.state === "recording") {
if (stream.current) {
- stream.current.getTracks().forEach(track => track.stop());
+ stream.current.getTracks().forEach(track => {
+ track.stop();
+ stream.current?.removeTrack(track);
+ });
}
mediaRecorder.current.stop();
setRecording(false);
@@ -73,7 +76,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
mediaRecorder.current.stop();
}
if (stream.current) {
- stream.current.getTracks().forEach(track => track.stop());
+ const tracks = stream.current.getTracks();
+ tracks.forEach(track => {
+ track.stop();
+ stream.current?.removeTrack(track);
+ });
stream.current = null;
}
};
@@ -81,12 +88,35 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const startRecording = async () => {
try {
+ if (stream.current) {
+ console.warn('[useScreenRecorder] Cleaning up existing stream before starting new recording');
+ const tracks = stream.current.getTracks();
+ tracks.forEach(track => {
+ track.stop();
+ stream.current?.removeTrack(track);
+ });
+ stream.current = null;
+ }
+
+ if (mediaRecorder.current) {
+ if (mediaRecorder.current.state !== 'inactive') {
+ mediaRecorder.current.stop();
+ }
+ mediaRecorder.current = null;
+ }
+
const selectedSource = await window.electronAPI.getSelectedSource();
if (!selectedSource) {
alert("Please select a source to record");
return;
}
+ console.log('[useScreenRecorder] Starting recording with source:', {
+ id: selectedSource.id,
+ name: selectedSource.name,
+ microphoneId: selectedSource.microphoneId
+ });
+
const mediaStream = await (navigator.mediaDevices as any).getUserMedia({
audio: false,
video: {
@@ -100,11 +130,224 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
},
},
});
+
+ let audioTracksAdded = 0;
+ const addedAudioTracks: MediaStreamTrack[] = [];
+
+ if (selectedSource.microphoneId) {
+ try {
+ const platform = await window.electronAPI?.getPlatform();
+
+ if (platform === 'darwin') {
+ await new Promise(resolve => setTimeout(resolve, 300));
+ }
+
+ const audioConstraints: MediaTrackConstraints = {
+ deviceId: { exact: selectedSource.microphoneId },
+ };
+
+ if (platform === 'darwin' || platform === 'linux') {
+ try {
+ audioConstraints.echoCancellation = true;
+ audioConstraints.noiseSuppression = true;
+ audioConstraints.autoGainControl = true;
+ } catch (e) {
+ console.warn('[useScreenRecorder] Some audio constraints not supported, continuing without them');
+ }
+ }
+
+ let audioStream: MediaStream | null = null;
+ let retries = 0;
+ const maxRetries = 3;
+
+ while (retries < maxRetries && !audioStream) {
+ try {
+ audioStream = await navigator.mediaDevices.getUserMedia({
+ audio: audioConstraints
+ });
+ break;
+ } catch (constraintError: any) {
+ retries++;
+ if (retries >= maxRetries) {
+ console.warn('[useScreenRecorder] Failed with constraints, trying without advanced constraints:', constraintError);
+ try {
+ audioStream = await navigator.mediaDevices.getUserMedia({
+ audio: { deviceId: { exact: selectedSource.microphoneId } }
+ });
+ break;
+ } catch (fallbackError) {
+ console.error('[useScreenRecorder] Failed to get audio stream after retries:', fallbackError);
+ throw fallbackError;
+ }
+ } else {
+ console.warn(`[useScreenRecorder] Audio access failed, retrying (${retries}/${maxRetries})...`, constraintError);
+ await new Promise(resolve => setTimeout(resolve, 200 * retries));
+ }
+ }
+ }
+
+ if (!audioStream) {
+ throw new Error('Failed to get audio stream after retries');
+ }
+
+ const audioTracks = audioStream.getAudioTracks();
+ if (audioTracks.length > 0) {
+ for (const track of audioTracks) {
+ if (track.readyState === 'live') {
+ if (!mediaStream.getAudioTracks().includes(track)) {
+ mediaStream.addTrack(track);
+ audioTracksAdded++;
+ addedAudioTracks.push(track);
+ }
+
+ track.onended = () => {
+ console.warn('[useScreenRecorder] Audio track ended unexpectedly');
+ };
+
+ track.onmute = () => {
+ console.warn('[useScreenRecorder] Audio track muted');
+ };
+ } else {
+ console.warn('[useScreenRecorder] Audio track not ready, waiting...', track.readyState);
+
+ await new Promise((resolve) => {
+ let timeoutId: NodeJS.Timeout;
+ const checkReady = () => {
+ if (track.readyState === 'live') {
+ if (!mediaStream.getAudioTracks().includes(track)) {
+ mediaStream.addTrack(track);
+ audioTracksAdded++;
+ addedAudioTracks.push(track);
+ }
+
+ track.onended = () => {
+ console.warn('[useScreenRecorder] Audio track ended unexpectedly');
+ };
+
+ track.onmute = () => {
+ console.warn('[useScreenRecorder] Audio track muted');
+ };
+ if (timeoutId) clearTimeout(timeoutId);
+ resolve();
+ } else if (track.readyState === 'ended') {
+ console.error('[useScreenRecorder] Audio track ended before it could be added');
+ if (timeoutId) clearTimeout(timeoutId);
+ resolve();
+ } else {
+ setTimeout(checkReady, 50);
+ }
+ };
+ timeoutId = setTimeout(() => {
+ console.warn('[useScreenRecorder] Timeout waiting for audio track to become ready');
+ resolve();
+ }, 3000);
+ checkReady();
+ });
+ }
+ }
+
+ if (audioTracksAdded > 0) {
+ console.log(`[useScreenRecorder] Added ${audioTracksAdded} audio track(s) to stream`);
+
+ await new Promise(resolve => setTimeout(resolve, 200));
+
+ const verifiedTracks = mediaStream.getAudioTracks().filter((t: MediaStreamTrack) =>
+ t.readyState === 'live' && t.enabled && !t.muted
+ );
+
+ if (verifiedTracks.length !== audioTracksAdded) {
+ console.warn(`[useScreenRecorder] Track count mismatch: added ${audioTracksAdded}, verified ${verifiedTracks.length}`);
+ }
+ } else {
+ console.warn('[useScreenRecorder] No live audio tracks available');
+ }
+ } else {
+ console.warn('[useScreenRecorder] No audio tracks found in audio stream');
+ }
+ } catch (audioError) {
+ console.error('Failed to capture microphone audio:', audioError);
+ }
+ }
+
stream.current = mediaStream;
if (!stream.current) {
throw new Error("Media stream is not available.");
}
- const videoTrack = stream.current.getVideoTracks()[0];
+
+ const audioTracks = stream.current.getAudioTracks();
+ const videoTracks = stream.current.getVideoTracks();
+
+ if (audioTracks.length > 0) {
+ console.log(`[useScreenRecorder] Stream has ${audioTracks.length} audio track(s) before validation`);
+
+ for (const track of audioTracks) {
+ if (!track.enabled) {
+ console.warn('[useScreenRecorder] Audio track is disabled, enabling...');
+ track.enabled = true;
+ }
+
+ if (track.muted) {
+ console.warn('[useScreenRecorder] Audio track is muted - this may affect recording');
+ }
+ }
+
+ const liveAudioTracks = audioTracks.filter(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+
+ if (liveAudioTracks.length === 0) {
+ console.warn('[useScreenRecorder] No live audio tracks - waiting for tracks to become ready...');
+
+ await Promise.race([
+ Promise.all(audioTracks.map(track => {
+ return new Promise((resolve) => {
+ if (track.readyState === 'live' && track.enabled && !track.muted) {
+ resolve();
+ return;
+ }
+
+ const maxWait = 3000;
+ const startTime = Date.now();
+
+ const checkReady = () => {
+ if (track.readyState === 'live' && track.enabled && !track.muted) {
+ resolve();
+ } else if (Date.now() - startTime > maxWait) {
+ console.warn(`[useScreenRecorder] Timeout waiting for track ${track.label} to become ready`);
+ resolve();
+ } else {
+ if (!track.enabled) track.enabled = true;
+ setTimeout(checkReady, 50);
+ }
+ };
+ checkReady();
+ });
+ })),
+ new Promise((resolve) => setTimeout(() => resolve(), 3000))
+ ]);
+ }
+
+ const finalLiveTracks = stream.current.getAudioTracks().filter(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+
+ console.log(`[useScreenRecorder] Final audio track count: ${finalLiveTracks.length} live tracks`);
+ finalLiveTracks.forEach(track => {
+ console.log(`[useScreenRecorder] Final track state: ${track.label}, enabled: ${track.enabled}, readyState: ${track.readyState}, muted: ${track.muted}`);
+ });
+
+ if (finalLiveTracks.length === 0 && audioTracks.length > 0) {
+ console.error('[useScreenRecorder] WARNING: Audio tracks exist but none are live/enabled/unmuted!');
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 150));
+ }
+
+ if (videoTracks.length === 0) {
+ throw new Error("No video tracks available in stream.");
+ }
+
+ const videoTrack = videoTracks[0];
try {
await videoTrack.applyConstraints({
frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE },
@@ -123,23 +366,106 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const videoBitsPerSecond = computeBitrate(width, height);
const mimeType = selectMimeType();
+
+ const finalAudioTracks = stream.current.getAudioTracks();
+ const hasAudio = finalAudioTracks.length > 0 && finalAudioTracks.some(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+
+ if (hasAudio) {
+ const liveTracks = finalAudioTracks.filter(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+ console.log(`[useScreenRecorder] Creating MediaRecorder with ${liveTracks.length} live audio track(s)`);
+ }
+
+ if (hasAudio) {
+ const audioSupported = MediaRecorder.isTypeSupported(mimeType);
+ if (!audioSupported) {
+ console.warn(`[useScreenRecorder] MIME type ${mimeType} may not support audio`);
+ }
+ }
console.log(
`Recording at ${width}x${height} @ ${frameRate ?? TARGET_FRAME_RATE}fps using ${mimeType} / ${Math.round(
videoBitsPerSecond / 1_000_000
- )} Mbps`
+ )} Mbps${hasAudio ? ' (with audio)' : ' (no audio)'}`
);
chunks.current = [];
- const recorder = new MediaRecorder(stream.current, {
+ const recorderOptions: MediaRecorderOptions = {
mimeType,
videoBitsPerSecond,
- });
+ };
+
+ if (hasAudio) {
+ recorderOptions.audioBitsPerSecond = 128_000;
+
+ const audioTracksInfo = finalAudioTracks.map(t => ({
+ label: t.label,
+ enabled: t.enabled,
+ readyState: t.readyState,
+ muted: t.muted
+ }));
+ console.log('[useScreenRecorder] Audio tracks state before recording:', audioTracksInfo);
+ }
+
+ const streamForRecorder = stream.current;
+ const audioTracksBeforeRecorder = streamForRecorder.getAudioTracks();
+ const liveAudioBeforeRecorder = audioTracksBeforeRecorder.filter(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+
+ if (hasAudio && liveAudioBeforeRecorder.length === 0) {
+ console.error('[useScreenRecorder] ERROR: hasAudio is true but no live tracks found!');
+ console.error('[useScreenRecorder] All audio tracks:', audioTracksBeforeRecorder.map(t => ({
+ label: t.label,
+ enabled: t.enabled,
+ readyState: t.readyState,
+ muted: t.muted
+ })));
+ }
+
+ const recorder = new MediaRecorder(streamForRecorder, recorderOptions);
+
+ if (hasAudio) {
+ recorder.addEventListener('start', () => {
+ const tracksAfterStart = streamForRecorder.getAudioTracks();
+ const liveTracksAfterStart = tracksAfterStart.filter(track =>
+ track.readyState === 'live' && track.enabled && !track.muted
+ );
+
+ console.log(`[useScreenRecorder] MediaRecorder started with ${tracksAfterStart.length} total audio track(s), ${liveTracksAfterStart.length} live`);
+ tracksAfterStart.forEach(track => {
+ console.log(`[useScreenRecorder] Track after start: ${track.label}, enabled: ${track.enabled}, readyState: ${track.readyState}, muted: ${track.muted}`);
+ });
+
+ const videoTracks = streamForRecorder.getVideoTracks();
+ console.log(`[useScreenRecorder] MediaRecorder started with ${videoTracks.length} video track(s)`);
+
+ if (liveTracksAfterStart.length === 0 && tracksAfterStart.length > 0) {
+ console.error('[useScreenRecorder] WARNING: Audio tracks exist but none are live after MediaRecorder start!');
+ }
+ });
+
+ recorder.addEventListener('error', (e) => {
+ console.error('[useScreenRecorder] MediaRecorder error:', e);
+ });
+ }
mediaRecorder.current = recorder;
recorder.ondataavailable = e => {
- if (e.data && e.data.size > 0) chunks.current.push(e.data);
+ if (e.data && e.data.size > 0) {
+ chunks.current.push(e.data);
+ if (hasAudio) {
+ console.log(`[useScreenRecorder] Received data chunk: ${e.data.size} bytes, type: ${e.data.type}`);
+ }
+ }
};
recorder.onstop = async () => {
+ const finalAudioTracks = stream.current?.getAudioTracks() || [];
+ console.log(`[useScreenRecorder] Recording stopped. Final audio tracks: ${finalAudioTracks.length}`);
+ console.log(`[useScreenRecorder] Total chunks: ${chunks.current.length}, total size: ${chunks.current.reduce((sum, chunk) => sum + chunk.size, 0)} bytes`);
+
stream.current = null;
if (chunks.current.length === 0) return;
const duration = Date.now() - startTime.current;
@@ -177,7 +503,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
console.error('Failed to start recording:', error);
setRecording(false);
if (stream.current) {
- stream.current.getTracks().forEach(track => track.stop());
+ const tracks = stream.current.getTracks();
+ tracks.forEach(track => {
+ track.stop();
+ stream.current?.removeTrack(track);
+ });
stream.current = null;
}
}
diff --git a/src/lib/exporter/videoDecoder.ts b/src/lib/exporter/videoDecoder.ts
index 34ece77..b30442f 100644
--- a/src/lib/exporter/videoDecoder.ts
+++ b/src/lib/exporter/videoDecoder.ts
@@ -4,27 +4,42 @@ export interface DecodedVideoInfo {
duration: number; // in seconds
frameRate: number;
codec: string;
+ hasAudio: boolean;
}
export class VideoFileDecoder {
private info: DecodedVideoInfo | null = null;
private videoElement: HTMLVideoElement | null = null;
+ private audioStream: MediaStream | null = null;
async loadVideo(videoUrl: string): Promise {
this.videoElement = document.createElement('video');
this.videoElement.src = videoUrl;
this.videoElement.preload = 'metadata';
+ this.videoElement.crossOrigin = 'anonymous';
return new Promise((resolve, reject) => {
this.videoElement!.addEventListener('loadedmetadata', () => {
const video = this.videoElement!;
+ let hasAudio = false;
+ try {
+ const stream = (video as any).captureStream();
+ hasAudio = stream.getAudioTracks().length > 0;
+ if (hasAudio) {
+ this.audioStream = stream;
+ }
+ } catch (e) {
+ console.warn('[VideoFileDecoder] Could not check for audio:', e);
+ }
+
this.info = {
width: video.videoWidth,
height: video.videoHeight,
duration: video.duration,
frameRate: 60,
codec: 'avc1.640033',
+ hasAudio,
};
resolve(this.info);
@@ -36,9 +51,26 @@ export class VideoFileDecoder {
});
}
- /**
- * Get video element for seeking
- */
+ getAudioStream(): MediaStream | null {
+ if (!this.videoElement) {
+ return null;
+ }
+
+ try {
+ const stream = (this.videoElement as any).captureStream();
+ const audioTracks = stream.getAudioTracks();
+ if (audioTracks.length > 0) {
+ const audioStream = new MediaStream();
+ audioTracks.forEach((track: MediaStreamTrack) => audioStream.addTrack(track));
+ return audioStream;
+ }
+ } catch (e) {
+ console.warn('[VideoFileDecoder] Failed to get audio stream:', e);
+ }
+
+ return null;
+ }
+
getVideoElement(): HTMLVideoElement | null {
return this.videoElement;
}
@@ -48,6 +80,11 @@ export class VideoFileDecoder {
}
destroy(): void {
+ if (this.audioStream) {
+ this.audioStream.getTracks().forEach(track => track.stop());
+ this.audioStream = null;
+ }
+
if (this.videoElement) {
this.videoElement.pause();
this.videoElement.src = '';
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index 4d07414..48609a2 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -28,6 +28,10 @@ export class VideoExporter {
private decoder: VideoFileDecoder | null = null;
private renderer: FrameRenderer | null = null;
private encoder: VideoEncoder | null = null;
+ private audioEncoder: AudioEncoder | null = null;
+ private audioContext: AudioContext | null = null;
+ private audioSource: MediaStreamAudioSourceNode | null = null;
+ private audioProcessor: ScriptProcessorNode | null = null;
private muxer: VideoMuxer | null = null;
private cancelled = false;
private encodeQueue = 0;
@@ -38,6 +42,7 @@ export class VideoExporter {
// Track muxing promises for parallel processing
private muxingPromises: Promise[] = [];
private chunkCount = 0;
+ private audioChunkCount = 0;
constructor(config: VideoExporterConfig) {
this.config = config;
@@ -106,8 +111,14 @@ export class VideoExporter {
// Initialize video encoder
await this.initializeEncoder();
+ // Initialize audio encoder if source has audio
+ const hasAudio = videoInfo.hasAudio;
+ if (hasAudio) {
+ await this.initializeAudioEncoder();
+ }
+
// Initialize muxer
- this.muxer = new VideoMuxer(this.config, false);
+ this.muxer = new VideoMuxer(this.config, hasAudio);
await this.muxer.initialize();
// Get the video element for frame extraction
@@ -116,6 +127,13 @@ export class VideoExporter {
throw new Error('Video element not available');
}
+ // Extract audio first if available (separate pass)
+ if (hasAudio && this.audioEncoder && this.audioContext) {
+ console.log('[VideoExporter] Extracting audio...');
+ await this.extractAudio(videoElement, videoInfo.duration);
+ console.log('[VideoExporter] Audio extraction complete');
+ }
+
// Calculate effective duration and frame count (excluding trim regions)
const effectiveDuration = this.getEffectiveDuration(videoInfo.duration);
const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
@@ -218,6 +236,10 @@ export class VideoExporter {
await this.encoder.flush();
}
+ if (this.audioEncoder && this.audioEncoder.state === 'configured') {
+ await this.audioEncoder.flush();
+ }
+
// Wait for all muxing operations to complete
await Promise.all(this.muxingPromises);
@@ -338,7 +360,222 @@ export class VideoExporter {
this.cleanup();
}
+ private async extractAudio(videoElement: HTMLVideoElement, duration: number): Promise {
+ if (!this.audioEncoder || !this.audioContext || !this.audioSource) {
+ return;
+ }
+
+ const trimRegions = this.config.trimRegions || [];
+ if (trimRegions.length === 0) {
+ return new Promise((resolve, reject) => {
+ const effectiveDuration = this.getEffectiveDuration(duration);
+ videoElement.currentTime = 0;
+ videoElement.play().catch(reject);
+
+ const timeout = setTimeout(() => {
+ videoElement.pause();
+ resolve();
+ }, effectiveDuration * 1000);
+
+ videoElement.addEventListener('ended', () => {
+ clearTimeout(timeout);
+ resolve();
+ }, { once: true });
+ });
+ }
+
+ const sortedTrims = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
+ let currentPos = 0;
+ const processSegment = async (segmentStart: number, segmentEnd: number): Promise => {
+ if (segmentStart >= segmentEnd || this.cancelled) return;
+
+ videoElement.currentTime = segmentStart / 1000;
+ await new Promise(resolve => {
+ videoElement.addEventListener('seeked', () => resolve(), { once: true });
+ });
+
+ const duration = (segmentEnd - segmentStart) / 1000;
+ videoElement.play().catch(() => {});
+
+ await new Promise(resolve => {
+ const timeout = setTimeout(() => {
+ videoElement.pause();
+ resolve();
+ }, duration * 1000);
+
+ const checkTime = () => {
+ if (videoElement.currentTime * 1000 >= segmentEnd || this.cancelled) {
+ clearTimeout(timeout);
+ videoElement.pause();
+ resolve();
+ } else {
+ requestAnimationFrame(checkTime);
+ }
+ };
+ checkTime();
+ });
+ };
+
+ for (const trim of sortedTrims) {
+ if (currentPos < trim.startMs) {
+ await processSegment(currentPos, trim.startMs);
+ }
+ currentPos = Math.max(currentPos, trim.endMs);
+ }
+
+ if (currentPos < duration * 1000) {
+ await processSegment(currentPos, duration * 1000);
+ }
+ }
+
+ private async initializeAudioEncoder(): Promise {
+ const audioStream = this.decoder?.getAudioStream();
+ if (!audioStream) {
+ console.warn('[VideoExporter] No audio stream available');
+ return;
+ }
+
+ try {
+ const audioContext = new AudioContext({ sampleRate: 48000 });
+ const source = audioContext.createMediaStreamSource(audioStream);
+
+ this.audioContext = audioContext;
+ this.audioSource = source;
+
+ const audioEncoder = new AudioEncoder({
+ output: (chunk, meta) => {
+ const isFirstChunk = this.audioChunkCount === 0;
+ this.audioChunkCount++;
+
+ const muxingPromise = (async () => {
+ try {
+ if (isFirstChunk && meta?.decoderConfig) {
+ const metadata: EncodedAudioChunkMetadata = {
+ decoderConfig: meta.decoderConfig,
+ };
+ await this.muxer!.addAudioChunk(chunk, metadata);
+ } else {
+ await this.muxer!.addAudioChunk(chunk, meta);
+ }
+ } catch (error) {
+ console.error('[VideoExporter] Audio muxing error:', error);
+ }
+ })();
+
+ this.muxingPromises.push(muxingPromise);
+ },
+ error: (error) => {
+ console.error('[VideoExporter] Audio encoder error:', error);
+ },
+ });
+
+ const audioConfig: AudioEncoderConfig = {
+ codec: 'opus',
+ sampleRate: 48000,
+ numberOfChannels: 2,
+ bitrate: 128_000,
+ };
+
+ const support = await AudioEncoder.isConfigSupported(audioConfig);
+ if (!support.supported) {
+ console.warn('[VideoExporter] Opus audio codec not supported');
+ return;
+ }
+
+ audioEncoder.configure(audioConfig);
+ this.audioEncoder = audioEncoder;
+
+ const bufferSize = 4096;
+ const numberOfChannels = 2;
+ const processor = audioContext.createScriptProcessor(bufferSize, numberOfChannels, numberOfChannels);
+
+ processor.onaudioprocess = (e) => {
+ if (!this.audioEncoder || this.audioEncoder.state !== 'configured' || this.cancelled) {
+ return;
+ }
+
+ const inputData = e.inputBuffer;
+ const length = inputData.length;
+ const sampleRate = inputData.sampleRate;
+ const channels = inputData.numberOfChannels;
+
+ const audioData = new Float32Array(length * channels);
+ for (let channel = 0; channel < channels; channel++) {
+ const channelData = inputData.getChannelData(channel);
+ for (let i = 0; i < length; i++) {
+ audioData[i * channels + channel] = channelData[i];
+ }
+ }
+
+ const audioFrame = new AudioData({
+ format: 'f32-planar',
+ sampleRate,
+ numberOfFrames: length,
+ numberOfChannels: channels,
+ timestamp: Math.round(audioContext.currentTime * 1_000_000),
+ data: audioData,
+ });
+
+ try {
+ this.audioEncoder.encode(audioFrame);
+ } catch (err) {
+ console.error('[VideoExporter] Error encoding audio frame:', err);
+ }
+ audioFrame.close();
+ };
+
+ source.connect(processor);
+ processor.connect(audioContext.destination);
+ this.audioProcessor = processor;
+
+ const videoElement = this.decoder?.getVideoElement();
+ if (videoElement) {
+ videoElement.play().catch(e => {
+ console.warn('[VideoExporter] Failed to play video for audio extraction:', e);
+ });
+ }
+ } catch (error) {
+ console.error('[VideoExporter] Failed to initialize audio encoder:', error);
+ }
+ }
+
private cleanup(): void {
+ if (this.audioProcessor) {
+ try {
+ this.audioProcessor.disconnect();
+ } catch (e) {
+ console.warn('[VideoExporter] Error disconnecting audio processor:', e);
+ }
+ this.audioProcessor = null;
+ }
+
+ if (this.audioSource) {
+ try {
+ this.audioSource.disconnect();
+ } catch (e) {
+ console.warn('[VideoExporter] Error disconnecting audio source:', e);
+ }
+ this.audioSource = null;
+ }
+
+ if (this.audioContext) {
+ this.audioContext.close().catch(e => {
+ console.warn('[VideoExporter] Error closing audio context:', e);
+ });
+ this.audioContext = null;
+ }
+
+ if (this.audioEncoder) {
+ try {
+ if (this.audioEncoder.state === 'configured') {
+ this.audioEncoder.close();
+ }
+ } catch (e) {
+ console.warn('[VideoExporter] Error closing audio encoder:', e);
+ }
+ this.audioEncoder = null;
+ }
+
if (this.encoder) {
try {
if (this.encoder.state === 'configured') {
@@ -372,6 +609,7 @@ export class VideoExporter {
this.encodeQueue = 0;
this.muxingPromises = [];
this.chunkCount = 0;
+ this.audioChunkCount = 0;
this.videoDescription = undefined;
this.videoColorSpace = undefined;
}