diff --git a/.gitignore b/.gitignore index 8358956..1107883 100644 --- a/.gitignore +++ b/.gitignore @@ -195,4 +195,6 @@ Cargo.lock !.yarn/versions *.node -dev/ \ No newline at end of file +dev/ +./src/dasp/* +./src/decoder/* \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index c0b69b9..13a619c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,6 @@ version = "0.0.0" crate-type = ["cdylib"] [dependencies] -# dasp = { version = "0.11.0" } napi = { git = "https://github.com/napi-rs/napi-rs", default-features = false, features = [ "napi4", ] } diff --git a/README.md b/README.md index 6ea2f4e..6534a3d 100644 --- a/README.md +++ b/README.md @@ -92,46 +92,55 @@ You can use `OpusEncoder` to encode pcm data to opus and decode opus data to pcm #### Opus Benchmarks -Mediaplex includes benchmarks for the Opus encoder/decoder. Here are the results of the benchmarks on a Windows 11 machine with an i7-8700 3.2GHz processor: +Mediaplex includes benchmarks for the opus encoder/decoder. Here are the results of the benchmarks on a Windows 11 machine with an i7-8700 3.2GHz processor: ```js $ yarn benchmark Running "OpusEncoder Benchmark" suite... -Progress: 100% mediaplex: - 3 502 ops/s, ±0.84% | 16.04% slower + 3 575 ops/s, ±0.75% | 14.82% slower @discordjs/opus: - 3 185 ops/s, ±0.17% | 23.64% slower + 3 169 ops/s, ±0.43% | 24.49% slower + + @evan/opus: + 3 310 ops/s, ±0.18% | 21.13% slower + + @evan/opus (wasm): + 2 259 ops/s, ±0.17% | 46.18% slower opusscript: - 4 171 ops/s, ±0.34% | fastest + 4 197 ops/s, ±0.52% | fastest opusscript (no wasm): - 261 ops/s, ±0.85% | slowest, 93.74% slower + 266 ops/s, ±0.55% | slowest, 93.66% slower -Finished 4 cases! +Finished 6 cases! Fastest: opusscript Slowest: opusscript (no wasm) - Running "OpusDecoder Benchmark" suite... -Progress: 100% mediaplex: - 9 838 ops/s, ±0.38% | 16.96% slower + 9 951 ops/s, ±0.42% | 16.12% slower @discordjs/opus: - 11 848 ops/s, ±0.40% | fastest + 11 864 ops/s, ±0.49% | fastest + + @evan/opus: + 11 470 ops/s, ±0.39% | 3.32% slower + + @evan/opus (wasm): + 7 436 ops/s, ±0.35% | 37.32% slower opusscript: - 6 100 ops/s, ±0.23% | 48.51% slower + 6 101 ops/s, ±0.31% | 48.58% slower opusscript (no wasm): - 2 589 ops/s, ±0.20% | slowest, 78.15% slower + 2 261 ops/s, ±0.24% | slowest, 80.94% slower -Finished 4 cases! +Finished 6 cases! Fastest: @discordjs/opus Slowest: opusscript (no wasm) ``` diff --git a/benchmark/common.mjs b/benchmark/common.mjs index bca82c0..5ae7b5c 100644 --- a/benchmark/common.mjs +++ b/benchmark/common.mjs @@ -2,6 +2,8 @@ import { Buffer } from 'node:buffer'; import djs from '@discordjs/opus'; import opusscript from 'opusscript'; import mediaplex from '../index.js'; +import * as evanOpus from '@evan/opus'; +import * as evanOpusWasm from '@evan/opus/wasm/index.mjs'; export const generatePCMSample = (sampleSize) => { const buffer = Buffer.alloc(sampleSize); @@ -57,4 +59,22 @@ export const createDjsEncoder = (config) => new djs.OpusEncoder(config.SAMPLE_RA export const createOpusScriptWasmEncoder = (config) => new opusscript(config.SAMPLE_RATE, config.CHANNELS, opusscript.Application.AUDIO); export const createOpusScriptAsmEncoder = (config) => new opusscript(config.SAMPLE_RATE, config.CHANNELS, opusscript.Application.AUDIO, { wasm: false +}); +export const createEvanOpusEncoder = (config) => new evanOpus.Encoder({ + application: 'voip', + channels: config.CHANNELS, + sample_rate: config.SAMPLE_RATE +}); +export const createEvanOpusDecoder = (config) => new evanOpus.Decoder({ + channels: config.CHANNELS, + sample_rate: config.SAMPLE_RATE +}); +export const createEvanOpusEncoderWasm = (config) => new evanOpusWasm.Encoder({ + application: 'voip', + channels: config.CHANNELS, + sample_rate: config.SAMPLE_RATE +}); +export const createEvanOpusDecoderWasm = (config) => new evanOpusWasm.Decoder({ + channels: config.CHANNELS, + sample_rate: config.SAMPLE_RATE }); \ No newline at end of file diff --git a/benchmark/decoder.mjs b/benchmark/decoder.mjs index 6738f2d..1bc4e01 100644 --- a/benchmark/decoder.mjs +++ b/benchmark/decoder.mjs @@ -1,5 +1,5 @@ import b from 'benny'; -import { createDjsEncoder, createMediaplexEncoder, createOpusScriptAsmEncoder, createOpusScriptWasmEncoder, generateOpusSample } from './common.mjs'; +import { createDjsEncoder, createMediaplexEncoder, createOpusScriptAsmEncoder, createOpusScriptWasmEncoder, generateOpusSample, createEvanOpusDecoder, createEvanOpusDecoderWasm } from './common.mjs'; const config = { FRAME_SIZE: 960, @@ -11,6 +11,8 @@ const mediaplexEncoder = createMediaplexEncoder(config); const nativeEncoder = createDjsEncoder(config); const wasmEncoder = createOpusScriptWasmEncoder(config); const asmEncoder = createOpusScriptAsmEncoder(config); +const evanOpus = createEvanOpusDecoder(config); +const evanOpusWasm = createEvanOpusDecoderWasm(config); const SAMPLE = generateOpusSample(); @@ -22,6 +24,12 @@ b.suite( b.add('@discordjs/opus', () => { nativeEncoder.decode(SAMPLE); }), + b.add('@evan/opus', () => { + evanOpus.decode(SAMPLE); + }), + b.add('@evan/opus (wasm)', () => { + evanOpusWasm.decode(SAMPLE); + }), b.add('opusscript', () => { wasmEncoder.decode(SAMPLE); }), diff --git a/benchmark/encoder.mjs b/benchmark/encoder.mjs index f458e63..7f8f2f5 100644 --- a/benchmark/encoder.mjs +++ b/benchmark/encoder.mjs @@ -1,5 +1,5 @@ import b from 'benny'; -import { createDjsEncoder, createMediaplexEncoder, createOpusScriptAsmEncoder, createOpusScriptWasmEncoder, generatePCMSample } from './common.mjs'; +import { createDjsEncoder, createMediaplexEncoder, createOpusScriptAsmEncoder, createOpusScriptWasmEncoder, generatePCMSample, createEvanOpusEncoder, createEvanOpusEncoderWasm } from './common.mjs'; const config = { FRAME_SIZE: 960, @@ -11,6 +11,8 @@ const mediaplexEncoder = createMediaplexEncoder(config); const nativeEncoder = createDjsEncoder(config); const wasmEncoder = createOpusScriptWasmEncoder(config); const asmEncoder = createOpusScriptAsmEncoder(config); +const evanOpus = createEvanOpusEncoder(config); +const evanOpusWasm = createEvanOpusEncoderWasm(config); const SAMPLE = generatePCMSample(config.FRAME_SIZE * config.CHANNELS * 6); @@ -22,6 +24,12 @@ b.suite( b.add('@discordjs/opus', () => { nativeEncoder.encode(SAMPLE); }), + b.add('@evan/opus', () => { + evanOpus.encode(SAMPLE); + }), + b.add('@evan/opus (wasm)', () => { + evanOpusWasm.encode(SAMPLE); + }), b.add('opusscript', () => { wasmEncoder.encode(SAMPLE, config.FRAME_SIZE); }), diff --git a/package.json b/package.json index 045942f..da5d6e4 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "license": "MIT", "devDependencies": { "@discordjs/opus": "^0.9.0", + "@evan/opus": "^1.0.2", "@napi-rs/cli": "^2.16.1", "@types/node": "^20.4.8", "ava": "^5.1.1", diff --git a/src/dasp/equalizer/mod.rs b/src/dasp/equalizer/mod.rs new file mode 100644 index 0000000..af97ce9 --- /dev/null +++ b/src/dasp/equalizer/mod.rs @@ -0,0 +1,23 @@ +use super::*; + +#[napi(object)] +pub struct EqualizerCoefficient { + pub beta: i32, + pub alpha: i32, + pub gamma: i32, +} + +#[napi] +pub struct Equalizer { + coefficients: Vec, + format: PCMFormat, + channels: AudioChannel, +} + +#[napi] +impl Equalizer { + #[napi(constructor)] + pub fn new() -> Self { + + } +} \ No newline at end of file diff --git a/src/dasp/mod.rs b/src/dasp/mod.rs new file mode 100644 index 0000000..81962b3 --- /dev/null +++ b/src/dasp/mod.rs @@ -0,0 +1,18 @@ +use napi::bindgen_prelude::*; + +#[napi] +pub enum PCMFormat { + S16LE, + S16BE, + S32LE, + S32BE, +} + +#[napi] +pub enum AudioChannel { + Mono, + Stereo, +} + +mod equalizer; +mod volume; diff --git a/src/dasp/volume/mod.rs b/src/dasp/volume/mod.rs new file mode 100644 index 0000000..34d2501 --- /dev/null +++ b/src/dasp/volume/mod.rs @@ -0,0 +1,165 @@ +use super::*; + +#[napi] +pub struct VolumeTransformer { + volume: i32, + format: PCMFormat, + channels: AudioChannel, +} + +// TODO: investigate SIMD optimizations + +#[napi] +impl VolumeTransformer { + #[napi(constructor)] + pub fn new(volume: i32, format: PCMFormat, channels: AudioChannel) -> Self { + Self { + volume, + format, + channels, + } + } + + #[napi(setter)] + pub fn set_volume(&mut self, volume: i32) { + if volume < 0 { + self.volume = 0; + } else { + self.volume = volume; + } + } + + #[napi(getter)] + pub fn get_volume(&self) -> i32 { + self.volume + } + + #[napi(setter)] + pub fn set_format(&mut self, format: PCMFormat) { + self.format = format; + } + + #[napi(getter)] + pub fn get_format(&self) -> PCMFormat { + self.format + } + + #[napi(setter)] + pub fn set_channels(&mut self, channels: AudioChannel) { + self.channels = channels; + } + + #[napi(getter)] + pub fn get_channels(&self) -> AudioChannel { + self.channels + } + + #[napi] + pub fn process(&self, input: Buffer) -> Buffer { + // avoid computation if volume is 1, aka default volume + if self.volume == 1 { + return input; + } + + match self.channels { + AudioChannel::Mono => self.process_mono_inner(input), + AudioChannel::Stereo => self.process_stereo_inner(input), + } + } + + fn process_stereo_inner(&self, input: Buffer) -> Buffer { + match self.format { + PCMFormat::S16LE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(4) { + let mut sample = i16::from_le_bytes([input[i], input[i + 1]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_le_bytes()); + let mut sample = i16::from_le_bytes([input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_le_bytes()); + } + output.into() + } + PCMFormat::S16BE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(4) { + let mut sample = i16::from_be_bytes([input[i], input[i + 1]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_be_bytes()); + let mut sample = i16::from_be_bytes([input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_be_bytes()); + } + output.into() + } + PCMFormat::S32LE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(8) { + let mut sample = i32::from_le_bytes([input[i], input[i + 1], input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_le_bytes()); + let mut sample = + i32::from_le_bytes([input[i + 4], input[i + 5], input[i + 6], input[i + 7]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_le_bytes()); + } + output.into() + } + PCMFormat::S32BE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(8) { + let mut sample = i32::from_be_bytes([input[i], input[i + 1], input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_be_bytes()); + let mut sample = + i32::from_be_bytes([input[i + 4], input[i + 5], input[i + 6], input[i + 7]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_be_bytes()); + } + output.into() + } + } + } + + fn process_mono_inner(&self, input: Buffer) -> Buffer { + match self.format { + PCMFormat::S16LE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(2) { + let mut sample = i16::from_le_bytes([input[i], input[i + 1]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_le_bytes()); + } + output.into() + } + PCMFormat::S16BE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(2) { + let mut sample = i16::from_be_bytes([input[i], input[i + 1]]); + sample = (sample as i32 * self.volume) as i16; + output.extend_from_slice(&sample.to_be_bytes()); + } + output.into() + } + PCMFormat::S32LE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(4) { + let mut sample = i32::from_le_bytes([input[i], input[i + 1], input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_le_bytes()); + } + output.into() + } + PCMFormat::S32BE => { + let mut output = Vec::with_capacity(input.len()); + for i in (0..input.len()).step_by(4) { + let mut sample = i32::from_be_bytes([input[i], input[i + 1], input[i + 2], input[i + 3]]); + sample = (sample as i32 * self.volume) as i32; + output.extend_from_slice(&sample.to_be_bytes()); + } + output.into() + } + } + } +} diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs new file mode 100644 index 0000000..81d0486 --- /dev/null +++ b/src/decoder/mod.rs @@ -0,0 +1,62 @@ +use napi::bindgen_prelude::*; +use napi::{Error, Result}; +use symphonia::core::codecs::{Decoder as SymphoniaDecoder, DecoderOptions, CODEC_TYPE_NULL}; +use symphonia::core::formats::{Packet, Track}; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::probe::Hint; + +struct Decoder { + decoder: Box, + track: &Track, +} + +impl Decoder { + fn new(src_hint: Option) -> Result { + let cursor = std::io::Cursor::new(Vec::new()); + + let mss = MediaSourceStream::new(Box::new(cursor), Default::default()); + + let mut hint = Hint::new(); + if let Some(h) = src_hint { + hint.with_extension(h.as_str()); + } + + let meta_opts: symphonia::core::meta::MetadataOptions = Default::default(); + let fmt_opts: symphonia::core::formats::FormatOptions = Default::default(); + + let mut probed = symphonia::default::get_probe() + .format(&hint, mss, &fmt_opts, &meta_opts) + .map_err(|e| { + Error::new( + Status::GenericFailure, + format!("failed to probe media: {}", e), + ) + })?; + + let mut format = probed.format; + + let track = format + .tracks() + .iter() + .find(|t| t.codec_params.codec != CODEC_TYPE_NULL) + .ok_or(Error::new(Status::GenericFailure, "no audio tracks found"))?; + + let dec_opts: DecoderOptions = Default::default(); + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &dec_opts) + .map_err(|e| Error::new(Status::GenericFailure, "unsupported codec"))?; + + Ok(Self { decoder, track }) + } + + fn decode(&mut self, data: &[u8]) -> Result<()> { + self + .decoder + .decode(Packet::new_from_slice(self.track.id, ts, dur, buf)); + Ok(()) + } +} + +fn decode(data: &[u8]) -> Result<()> { + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index c253390..98258fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,5 +4,7 @@ #[macro_use] extern crate napi_derive; +// mod dasp; +// mod decoder; mod opus; mod probe; diff --git a/yarn.lock b/yarn.lock index 8944b91..57eb8fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -78,6 +78,13 @@ __metadata: languageName: node linkType: hard +"@evan/opus@npm:^1.0.2": + version: 1.0.2 + resolution: "@evan/opus@npm:1.0.2" + checksum: 5473155e79eb99a6bf5d2b35cbad5b2ff687e9aa5b1e2a7c15bd701eb3eb3986b6452079650823e8211727d8f033a6be86484e10c87f7eef5ebbbbfb7e10c928 + languageName: node + linkType: hard + "@isaacs/cliui@npm:^8.0.2": version: 8.0.2 resolution: "@isaacs/cliui@npm:8.0.2" @@ -1506,6 +1513,7 @@ __metadata: resolution: "mediaplex@workspace:." dependencies: "@discordjs/opus": ^0.9.0 + "@evan/opus": ^1.0.2 "@napi-rs/cli": ^2.16.1 "@types/node": ^20.4.8 ava: ^5.1.1