From 2620ab13884b7140942ec3e98879c647483d6bcc Mon Sep 17 00:00:00 2001 From: asubb Date: Thu, 19 May 2022 20:30:57 -0400 Subject: [PATCH 1/2] [FFMPEG] prototype --- ffmpeg/README.md | 4 + ffmpeg/build.gradle.kts | 4 + .../main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt | 103 ++++++++++++++++++ settings.gradle.kts | 4 +- 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 ffmpeg/README.md create mode 100644 ffmpeg/build.gradle.kts create mode 100644 ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt diff --git a/ffmpeg/README.md b/ffmpeg/README.md new file mode 100644 index 00000000..499aa6c2 --- /dev/null +++ b/ffmpeg/README.md @@ -0,0 +1,4 @@ +FFMPEG plugin +======== + +The projects injects [ffmpeg](http://ffmpeg.org/) capabilities into the project \ No newline at end of file diff --git a/ffmpeg/build.gradle.kts b/ffmpeg/build.gradle.kts new file mode 100644 index 00000000..3860d00c --- /dev/null +++ b/ffmpeg/build.gradle.kts @@ -0,0 +1,4 @@ +dependencies { + implementation(project(":lib")) + api("org.bytedeco:ffmpeg-platform:5.0-1.5.7") +} \ No newline at end of file diff --git a/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt b/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt new file mode 100644 index 00000000..9e2bca81 --- /dev/null +++ b/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt @@ -0,0 +1,103 @@ +package io.wavebeans.ffmpeg + +import io.wavebeans.lib.BitDepth +import io.wavebeans.lib.io.WavHeader +import org.bytedeco.ffmpeg.avcodec.AVPacket +import org.bytedeco.ffmpeg.avformat.AVFormatContext +import org.bytedeco.ffmpeg.global.avcodec.av_packet_unref +import org.bytedeco.ffmpeg.global.avcodec.avcodec_alloc_context3 +import org.bytedeco.ffmpeg.global.avcodec.avcodec_find_decoder +import org.bytedeco.ffmpeg.global.avcodec.avcodec_open2 +import org.bytedeco.ffmpeg.global.avcodec.avcodec_parameters_to_context +import org.bytedeco.ffmpeg.global.avcodec.avcodec_receive_frame +import org.bytedeco.ffmpeg.global.avcodec.avcodec_send_packet +import org.bytedeco.ffmpeg.global.avformat.av_dump_format +import org.bytedeco.ffmpeg.global.avformat.av_read_frame +import org.bytedeco.ffmpeg.global.avformat.avformat_close_input +import org.bytedeco.ffmpeg.global.avformat.avformat_find_stream_info +import org.bytedeco.ffmpeg.global.avformat.avformat_open_input +import org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_AUDIO +import org.bytedeco.ffmpeg.global.avutil.av_frame_alloc +import org.bytedeco.ffmpeg.global.avutil.av_get_bytes_per_sample +import org.bytedeco.ffmpeg.global.avutil.av_strerror +import org.bytedeco.javacpp.PointerPointer +import java.io.File + +class Ffmpeg { +} + +fun main() { + val fmtCtx = AVFormatContext(null) + val pkt = AVPacket() + val file = "${System.getProperty("user.home")}/tmp/guitar.wav" + avformat_open_input(fmtCtx, file, null, null).throwIfError() + + avformat_find_stream_info(fmtCtx, null as PointerPointer<*>?).throwIfError() + + av_dump_format(fmtCtx, 0, file, 0) + + var streamIdx: Int = -1 + for (it in 0 until fmtCtx.nb_streams()) { + val codecType = fmtCtx.streams(it).codecpar().codec_type() + if (codecType == AVMEDIA_TYPE_AUDIO) { + streamIdx = it + break + } + } + require(streamIdx >= 0) { "audio stream is not found" } + println("Picked stream #$streamIdx") + fmtCtx.streams(streamIdx).codecpar().use { + println("Sample rate: ${it.sample_rate()}") + println("Bit depth: ${it.bits_per_coded_sample()}") + } + + val codecCtx = avcodec_alloc_context3(null) + avcodec_parameters_to_context(codecCtx, fmtCtx.streams(streamIdx).codecpar()) + val codec = avcodec_find_decoder(codecCtx.codec_id()) + requireNotNull(codec) { "Unsupported codec $codecCtx" } + avcodec_open2(codecCtx, codec, null as PointerPointer<*>?).throwIfError() + println("Context=$codecCtx codec=$codec") + + av_frame_alloc().use { frame -> + val wav = WavHeader(BitDepth.BIT_32, 44100.0f, 1, 0x1FFFFFFF); + val wavFile = File(File(file).parentFile, "output.wav") + wavFile.createNewFile() + wavFile.outputStream().use { fos -> + fos.write(wav.header()) + while (av_read_frame(fmtCtx, pkt) >= 0) { + if (pkt.stream_index() == streamIdx) { + avcodec_send_packet(codecCtx, pkt).throwIfError() + avcodec_receive_frame(codecCtx, frame).throwIfError() + + val data = frame.data(0) + val dataSize = av_get_bytes_per_sample(codecCtx.sample_fmt()) + println("dataSize=$dataSize") + val buf = ByteArray(dataSize * frame.nb_samples()) + data.get(buf) + fos.write(buf) + + } + av_packet_unref(pkt) + } + + } + + } + + avformat_close_input(fmtCtx) +} + +fun Int.throwIfError() { + if (this < 0) { + val buf = ByteArray(1024) + val r = av_strerror(this, buf, 1024L) + val end = buf.indexOfFirst { it.toInt() == 0 }.takeIf { it >= 0 } + throw Exception( + if (r >= 0) { + buf.decodeToString(0, end ?: buf.size) + } else { + "unknown error $this" + } + ) + } +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 08edcad6..b2a2f1d5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,4 +12,6 @@ include(":tests") project(":metrics-core").projectDir = file("metrics/core") project(":metrics-prometheus").projectDir = file("metrics/prometheus") project(":filesystems-core").projectDir = file("filesystems/core") -project(":filesystems-dropbox").projectDir = file("filesystems/dropbox") \ No newline at end of file +project(":filesystems-dropbox").projectDir = file("filesystems/dropbox") + +include(":ffmpeg") \ No newline at end of file From 2d2930e19d793d9158eab9172a31a1761809317a Mon Sep 17 00:00:00 2001 From: asubb Date: Fri, 20 May 2022 11:14:53 -0400 Subject: [PATCH 2/2] [FFMPEG] prototype. Reading file through custom IO --- .../main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt | 109 +++++++++++++++--- 1 file changed, 96 insertions(+), 13 deletions(-) diff --git a/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt b/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt index 9e2bca81..2713ea2b 100644 --- a/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt +++ b/ffmpeg/src/main/kotlin/io/wavebeans/ffmpeg/Ffmpeg.kt @@ -3,7 +3,9 @@ package io.wavebeans.ffmpeg import io.wavebeans.lib.BitDepth import io.wavebeans.lib.io.WavHeader import org.bytedeco.ffmpeg.avcodec.AVPacket -import org.bytedeco.ffmpeg.avformat.AVFormatContext +import org.bytedeco.ffmpeg.avformat.AVProbeData +import org.bytedeco.ffmpeg.avformat.Read_packet_Pointer_BytePointer_int +import org.bytedeco.ffmpeg.avformat.Seek_Pointer_long_int import org.bytedeco.ffmpeg.global.avcodec.av_packet_unref import org.bytedeco.ffmpeg.global.avcodec.avcodec_alloc_context3 import org.bytedeco.ffmpeg.global.avcodec.avcodec_find_decoder @@ -11,31 +13,113 @@ import org.bytedeco.ffmpeg.global.avcodec.avcodec_open2 import org.bytedeco.ffmpeg.global.avcodec.avcodec_parameters_to_context import org.bytedeco.ffmpeg.global.avcodec.avcodec_receive_frame import org.bytedeco.ffmpeg.global.avcodec.avcodec_send_packet -import org.bytedeco.ffmpeg.global.avformat.av_dump_format +import org.bytedeco.ffmpeg.global.avformat.AVSEEK_SIZE +import org.bytedeco.ffmpeg.global.avformat.av_probe_input_format import org.bytedeco.ffmpeg.global.avformat.av_read_frame +import org.bytedeco.ffmpeg.global.avformat.avformat_alloc_context import org.bytedeco.ffmpeg.global.avformat.avformat_close_input import org.bytedeco.ffmpeg.global.avformat.avformat_find_stream_info import org.bytedeco.ffmpeg.global.avformat.avformat_open_input +import org.bytedeco.ffmpeg.global.avformat.avio_alloc_context +import org.bytedeco.ffmpeg.global.avutil.AVERROR_EOF +import org.bytedeco.ffmpeg.global.avutil.AVERROR_UNKNOWN import org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_AUDIO import org.bytedeco.ffmpeg.global.avutil.av_frame_alloc import org.bytedeco.ffmpeg.global.avutil.av_get_bytes_per_sample +import org.bytedeco.ffmpeg.global.avutil.av_malloc import org.bytedeco.ffmpeg.global.avutil.av_strerror +import org.bytedeco.javacpp.BytePointer +import org.bytedeco.javacpp.Pointer import org.bytedeco.javacpp.PointerPointer import java.io.File +import java.nio.ByteBuffer class Ffmpeg { } + +const val SEEK_SET = 0 +const val SEEK_CUR = 1 +const val SEEK_END = 2 fun main() { - val fmtCtx = AVFormatContext(null) - val pkt = AVPacket() val file = "${System.getProperty("user.home")}/tmp/guitar.wav" - avformat_open_input(fmtCtx, file, null, null).throwIfError() - avformat_find_stream_info(fmtCtx, null as PointerPointer<*>?).throwIfError() + val buffer = ByteBuffer.wrap(File(file).readBytes()) + + println("buffer=$buffer") + val inputStream = BytePointer(buffer) + val bufferSize = 512 + val ioContext = avio_alloc_context( + BytePointer(av_malloc(bufferSize.toLong())), + bufferSize, + 0, + inputStream, + object : Read_packet_Pointer_BytePointer_int() { + override fun call(opaque: Pointer, buf: BytePointer, buf_size: Int): Int { + try { +// println("read(opaque=$opaque, buf=${buf}, buf_size=$buf_size) buffer=$buffer") + val length = minOf(buffer.remaining(), buf_size/*, buf.capacity().toInt()*/) + if (length <= 0) { + return AVERROR_EOF + } + repeat(length) { + val b = buffer.get() + buf.put(it.toLong(), b) + } +// println("Read $length bytes") + return length + } catch (e: Exception) { + e.printStackTrace() + return AVERROR_UNKNOWN + } + } + }, + null, + object : Seek_Pointer_long_int() { + override fun call(opaque: Pointer, offset: Long, whence: Int): Long { +// println("seek(opaque=$opaque, offset=$offset, whence=$whence") + try { + if (whence == AVSEEK_SIZE) + return buffer.capacity().toLong() + when (whence) { + SEEK_SET -> buffer.position(offset.toInt()) + SEEK_CUR -> buffer.position(buffer.position() + offset.toInt()) + SEEK_END -> buffer.position(buffer.limit()) + else -> throw IllegalArgumentException("Whence $whence is not recognized") + } +// println("Set to ${buffer.position()} position:$buffer") + return buffer.position().toLong() + } catch (e: Exception) { + e.printStackTrace() + return -1 + } + } + } + ) - av_dump_format(fmtCtx, 0, file, 0) + val fmtCtx = avformat_alloc_context() + fmtCtx.pb(ioContext) +// val fmtCtx = AVFormatContext(null) +// avformat_open_input(fmtCtx, file, null, null).throwIfError() +// av_dump_format(fmtCtx, 0, file, 0) + val probeData = AVProbeData() + probeData.buf(BytePointer(buffer)) + probeData.buf_size(1234) + probeData.filename(BytePointer("")) + val format = av_probe_input_format(probeData, 1) + println(">>>> format=${format.long_name().stringBytes.decodeToString()}") + + fmtCtx.iformat(format) + fmtCtx.flags(128) // AVFMT_FLAG_CUSTOM_IO + + buffer.rewind() + avformat_open_input(fmtCtx, "", format, null).throwIfError() + + println("streams ${fmtCtx.streams()}") + println("nb_stream=${fmtCtx.nb_streams()}") + + avformat_find_stream_info(fmtCtx, null as PointerPointer<*>?).throwIfError() var streamIdx: Int = -1 for (it in 0 until fmtCtx.nb_streams()) { val codecType = fmtCtx.streams(it).codecpar().codec_type() @@ -62,16 +146,16 @@ fun main() { val wav = WavHeader(BitDepth.BIT_32, 44100.0f, 1, 0x1FFFFFFF); val wavFile = File(File(file).parentFile, "output.wav") wavFile.createNewFile() - wavFile.outputStream().use { fos -> + wavFile.outputStream().buffered().use { fos -> fos.write(wav.header()) + val pkt = AVPacket() + val dataSize = av_get_bytes_per_sample(codecCtx.sample_fmt()) while (av_read_frame(fmtCtx, pkt) >= 0) { if (pkt.stream_index() == streamIdx) { avcodec_send_packet(codecCtx, pkt).throwIfError() avcodec_receive_frame(codecCtx, frame).throwIfError() val data = frame.data(0) - val dataSize = av_get_bytes_per_sample(codecCtx.sample_fmt()) - println("dataSize=$dataSize") val buf = ByteArray(dataSize * frame.nb_samples()) data.get(buf) fos.write(buf) @@ -79,9 +163,7 @@ fun main() { } av_packet_unref(pkt) } - } - } avformat_close_input(fmtCtx) @@ -89,10 +171,11 @@ fun main() { fun Int.throwIfError() { if (this < 0) { + println("ERROR") val buf = ByteArray(1024) val r = av_strerror(this, buf, 1024L) val end = buf.indexOfFirst { it.toInt() == 0 }.takeIf { it >= 0 } - throw Exception( + throw IllegalStateException( if (r >= 0) { buf.decodeToString(0, end ?: buf.size) } else {