Merge pull request #4189 from remotion-dev/test-more-codecs

packages/docs/docs/media-parser/fetch-reader.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-media-parser-fetch-reader.png
 id: fetch-reader
 title: fetchReader
 slug: /media-parser/fetch-reader

packages/docs/docs/media-parser/node-reader.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-media-parser-node-reader.png
 id: node-reader
 title: nodeReader
 slug: /media-parser/node-reader

packages/docs/docs/media-parser/support.mdx

@@ -0,0 +1,77 @@
+---
+image: /generated/articles-docs-media-parser-support.png
+id: support
+title: Runtime support
+slug: /media-parser/support
+crumb: "@remotion/media-parser"
+---
+
+`@remotion/media-parser` works in the Browser, Node.js and Bun.  
+
+However, all of these require some quite modern versions (considered modern as of August 2024).
+
+## For parsing
+
+For parsing a video, the following minimum versions are required:
+
+- Node.js: 20.0.0
+- Bun 1.0.0
+- Chrome 111
+- Edge 111
+- Safari 16.4
+- Firefox 128
+
+## For using WebCodecs
+
+WebCodecs support is not tied to `@remotion/media-parser` itself, but if you use it to extract samples, you need to use the following minimum versions:
+
+- Chrome 94
+- Edge 94
+- Safari 16.4 - No support for `AudioDecoder`
+- Firefox - No support, but in development
+
+To check for `AudioDecoder` support, use
+
+```ts
+const audioDecoderSupported = typeof AudioDecoder !== 'undefined';
+```
+
+To check if a video track can be decoded with WebCodecs, use
+
+```ts twoslash title="Checking if a video track can be decoded"
+import {OnVideoTrack} from '@remotion/media-parser';
+
+const onVideoTrack: OnVideoTrack = async (track) => {
+  const {supported} = await VideoDecoder.isConfigSupported(track);
+
+  if (!supported) {
+    // Don't receive unsupported video samples
+    return null;
+  }
+
+  return (videoSample) => {
+    console.log('New video sample !', videoSample);
+    // Accept video sample
+  }
+}
+```
+
+
+```ts twoslash title="Checking if a audio track can be decoded"
+// @noErrors
+import {OnAudioTrack} from '@remotion/media-parser';
+
+const onAudioTrack: OnAudioTrack = async (track) => {
+  const {supported} = await AudioDecoder.isConfigSupported(track);
+
+  if (!supported) {
+    // Don't receive unsupported audio samples
+    return null;
+  }
+
+  return (audioSample) => {
+    console.log('New audio sample!', audioSample);
+    // Accept audio sample
+  }
+}
+```

packages/docs/docs/media-parser/web-file-reader.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-media-parser-web-file-reader.png
 id: web-file-reader
 title: webFileReader
 slug: /media-parser/web-file-reader

packages/docs/docs/media-parser/webcodecs.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-media-parser-webcodecs.png
 id: webcodecs
 title: WebCodecs
 slug: /media-parser/webcodecs
@@ -21,21 +22,20 @@ From MP4 videos:
 - [x] h.264
 - [x] h.265
 - [x] AV1
-- [ ] AAC
-- [ ] MP3
-- [ ] PCM
+- [x] AAC
+- [x] MP3
 
-From WebM videos:
+From Matroska videos:
 
-- [ ] h.264
-- [ ] h.265
+- [x] h.264
+- [x] h.265
 - [x] vp8
 - [x] vp9
 - [x] AV1
-- [ ] AAC
-- [ ] MP3
-- [ ] Opus
-- [ ] PCM
+- [x] AAC
+- [x] MP3
+- [x] Opus
+- [x] PCM (No browser supports it yet)
 - [x] Vorbis
 
 ## API

packages/docs/sidebars.js

@@ -564,6 +564,7 @@ module.exports = {
 				'media-parser/fetch-reader',
 				'media-parser/web-file-reader',
 				'media-parser/webcodecs',
+				'media-parser/support',
 			],
 		},
 		{

packages/docs/src/data/articles.ts

@@ -3338,13 +3338,48 @@ export const articles = [
 		compId: 'articles-docs-media-parser-parse-media',
 		crumb: '@remotion/media-parser',
 	},
+	{
+		id: 'node-reader',
+		title: 'nodeReader',
+		relativePath: 'docs/media-parser/node-reader.mdx',
+		compId: 'articles-docs-media-parser-node-reader',
+		crumb: '@remotion/media-parser',
+	},
+	{
+		id: 'web-file-reader',
+		title: 'webFileReader',
+		relativePath: 'docs/media-parser/web-file-reader.mdx',
+		compId: 'articles-docs-media-parser-web-file-reader',
+		crumb: '@remotion/media-parser',
+	},
 	{
 		id: 'media-parser/index',
 		title: '@remotion/media-parser',
 		relativePath: 'docs/media-parser/index.mdx',
 		compId: 'articles-docs-media-parser-index',
 		crumb: null,
 	},
+	{
+		id: 'support',
+		title: 'Runtime support',
+		relativePath: 'docs/media-parser/support.mdx',
+		compId: 'articles-docs-media-parser-support',
+		crumb: '@remotion/media-parser',
+	},
+	{
+		id: 'webcodecs',
+		title: 'WebCodecs',
+		relativePath: 'docs/media-parser/webcodecs.mdx',
+		compId: 'articles-docs-media-parser-webcodecs',
+		crumb: '@remotion/media-parser',
+	},
+	{
+		id: 'fetch-reader',
+		title: 'fetchReader',
+		relativePath: 'docs/media-parser/fetch-reader.mdx',
+		compId: 'articles-docs-media-parser-fetch-reader',
+		crumb: '@remotion/media-parser',
+	},
 	{
 		id: 'measure-spring',
 		title: 'measureSpring()',

packages/docusaurus-plugin/package.json

@@ -19,7 +19,8 @@
 		"fenceparser": "^2.2.0",
 		"shiki": "0.10.1",
 		"@typescript/twoslash": "3.2.1",
-		"@docusaurus/types": "3.4.0"
+		"@docusaurus/types": "3.4.0",
+		"@types/dom-webcodecs": "0.1.11"
 	},
 	"private": true,
 	"repository": {

packages/example/package.json

@@ -92,7 +92,8 @@
 	},
 	"devDependencies": {
 		"prettier-plugin-organize-imports": "3.2.4",
-		"sharp": "0.32.6"
+		"sharp": "0.32.6",
+		"@types/dom-webcodecs": "0.1.11"
 	},
 	"publishConfig": {
 		"access": "public"

packages/example/src/Encoder/SrcEncoder.tsx

@@ -0,0 +1,135 @@
+import {OnAudioTrack, OnVideoTrack, parseMedia} from '@remotion/media-parser';
+import React, {useCallback, useRef} from 'react';
+import {flushSync} from 'react-dom';
+
+export const SrcEncoder: React.FC<{
+	src: string;
+}> = ({src}) => {
+	const [samples, setSamples] = React.useState<number>(0);
+	const [videoFrames, setVideoFrames] = React.useState<number>(0);
+	const [audioFrames, setAudioFrames] = React.useState<number>(0);
+
+	const ref = useRef<HTMLCanvasElement>(null);
+
+	const onVideoTrack: OnVideoTrack = useCallback(async (track) => {
+		const decoder = await VideoDecoder.isConfigSupported(track);
+
+		if (!decoder.supported) {
+			return null;
+		}
+
+		const videoDecoder = new VideoDecoder({
+			async output(inputFrame) {
+				const image = await createImageBitmap(inputFrame, {
+					resizeHeight: 100,
+					resizeWidth: 100,
+				});
+
+				ref.current?.getContext('2d')?.drawImage(image, 0, 0);
+				flushSync(() => {
+					setVideoFrames((prev) => prev + 1);
+				});
+				inputFrame.close();
+			},
+			error(error) {
+				console.error(error);
+			},
+		});
+		videoDecoder.configure(track);
+
+		return async (chunk) => {
+			flushSync(() => {
+				setSamples((s) => s + 1);
+			});
+			if (videoDecoder.decodeQueueSize > 10) {
+				let resolve = () => {};
+
+				const cb = () => {
+					resolve();
+				};
+
+				await new Promise<void>((r) => {
+					resolve = r;
+					videoDecoder.addEventListener('dequeue', cb);
+				});
+				videoDecoder.removeEventListener('dequeue', cb);
+			}
+			videoDecoder.decode(new EncodedVideoChunk(chunk));
+		};
+	}, []);
+
+	const onAudioTrack: OnAudioTrack = useCallback(async (track) => {
+		console.log(track);
+
+		const {supported} = await AudioDecoder.isConfigSupported(track);
+
+		if (!supported) {
+			return null;
+		}
+
+		if (typeof AudioDecoder === 'undefined') {
+			return null;
+		}
+
+		const audioDecoder = new AudioDecoder({
+			output(inputFrame) {
+				flushSync(() => {
+					setAudioFrames((prev) => prev + 1);
+				});
+				inputFrame.close();
+			},
+			error(error) {
+				console.error(error);
+			},
+		});
+
+		audioDecoder.configure(track);
+
+		return async (audioSample) => {
+			flushSync(() => {
+				setSamples((s) => s + 1);
+			});
+			if (audioDecoder.decodeQueueSize > 10) {
+				console.log('audio decoder queue size', audioDecoder.decodeQueueSize);
+				let resolve = () => {};
+
+				const cb = () => {
+					resolve();
+				};
+
+				await new Promise<void>((r) => {
+					resolve = r;
+					// @ts-expect-error exists
+					audioDecoder.addEventListener('dequeue', cb);
+				});
+				// @ts-expect-error exists
+				audioDecoder.removeEventListener('dequeue', cb);
+			}
+			audioDecoder.decode(new EncodedAudioChunk(audioSample));
+		};
+	}, []);
+
+	const onClick = useCallback(() => {
+		console.time('done');
+		parseMedia({
+			src,
+			onVideoTrack,
+			onAudioTrack,
+		}).then(() => {
+			console.timeEnd('done');
+		});
+	}, [onAudioTrack, onVideoTrack, src]);
+
+	return (
+		<div>
+			{src}{' '}
+			<button type="button" onClick={onClick}>
+				Decode
+			</button>
+			{samples > 0 && <div>{samples} samples</div>}
+			{videoFrames > 0 && <div>{videoFrames} video frames</div>}
+			{audioFrames > 0 && <div>{audioFrames} audio frames</div>}
+			<canvas ref={ref} width={100} height={50} />
+		</div>
+	);
+};

packages/example/src/EncoderDemo.tsx

@@ -1,17 +1,42 @@
 import {parseVideo, reencodeVideo} from '@remotion/browser-renderer';
 import React from 'react';
+import {staticFile} from 'remotion';
+import {SrcEncoder} from './Encoder/SrcEncoder';
 
 export const EncoderDemo: React.FC = () => {
 	return (
 		<>
-			<input
-				type="file"
-				onChange={(e) => parseVideo(e.target.files?.[0] as File)}
-			/>
-			<input
-				type="file"
-				onChange={(e) => reencodeVideo(e.target.files?.[0] as File)}
-			/>
+			<div
+				style={{
+					overflowY: 'scroll',
+				}}
+			>
+				<div>
+					<input
+						type="file"
+						onChange={(e) => parseVideo(e.target.files?.[0] as File)}
+					/>
+					<input
+						type="file"
+						onChange={(e) => reencodeVideo(e.target.files?.[0] as File)}
+					/>
+				</div>
+				<SrcEncoder src={staticFile('av1.webm')} />
+				<SrcEncoder src={staticFile('av1.mp4')} />
+				<SrcEncoder src={staticFile('mp4-mp3.mp4')} />
+				<SrcEncoder src={staticFile('bigbuckbunny.mp4')} />
+				<SrcEncoder src={staticFile('corrupted.mp4')} />
+				<SrcEncoder src={staticFile('matroska-pcm16.mkv')} />
+				<SrcEncoder src={staticFile('matroska-h265-aac.mkv')} />
+				<SrcEncoder src={staticFile('matroska-mp3.mkv')} />
+				<SrcEncoder src={staticFile('vid1.mp4')} />
+				<SrcEncoder src={staticFile('whip.mp3')} />
+				<SrcEncoder src={staticFile('iphone-hevc.mov')} />
+				<SrcEncoder src={staticFile('sample.aac')} />
+				<SrcEncoder src={staticFile('stretched-vp8.webm')} />
+				<SrcEncoder src={staticFile('opus.webm')} />
+				<SrcEncoder src="https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4" />
+			</div>
 		</>
 	);
 };