From ec4b5e6ef6f58cdf4096bdc5a171c888421e6d5b Mon Sep 17 00:00:00 2001 From: zoff99 Date: Wed, 1 Nov 2023 11:03:14 +0100 Subject: [PATCH] more more stuff into AVStatus --- src/main/kotlin/Main.kt | 29 +-- .../com/zoffcc/applications/trifa/AVState.kt | 221 +++++++++++++++++- .../zoffcc/applications/trifa/MainActivity.kt | 48 ++-- .../com/zoffcc/applications/trifa2/ChatApp.kt | 193 +-------------- 4 files changed, 245 insertions(+), 246 deletions(-) diff --git a/src/main/kotlin/Main.kt b/src/main/kotlin/Main.kt index f3466733..d3a6002d 100644 --- a/src/main/kotlin/Main.kt +++ b/src/main/kotlin/Main.kt @@ -142,6 +142,7 @@ val SPACE_AFTER_LAST_MESSAGE = 2.dp val SPACE_BEFORE_FIRST_MESSAGE = 10.dp val CAPTURE_VIDEO_WIDTH = 640 // 1280 val CAPTURE_VIDEO_HEIGHT = 480 // 720 +val CAPTURE_VIDEO_FPS = 20 val VIDEO_IN_BOX_WIDTH_SMALL = 80.dp val VIDEO_IN_BOX_HEIGHT_SMALL = 80.dp val VIDEO_IN_BOX_WIDTH_FRACTION_SMALL = 0.5f @@ -354,12 +355,7 @@ fun App() maxLines = 1) Box { IconButton(onClick = { - if (!avstatestore.state.ffmpeg_init_done) - { - val res = AVActivity.ffmpegav_init() - println("ffmpeg init: $res") - avstatestore.state.ffmpeg_init_done = true - } + avstatestore.state.ffmpeg_init_do() val audio_in_devices_get = AVActivity.ffmpegav_get_audio_in_devices() println("ffmpeg audio in devices: " + audio_in_devices_get.size) audio_in_devices.clear() @@ -400,12 +396,7 @@ fun App() IconButton(onClick = { if ((avstatestore.state.audio_in_device_get() != null) && (avstatestore.state.audio_in_device_get() != "")) { - if (!avstatestore.state.ffmpeg_init_done) - { - val res = AVActivity.ffmpegav_init() - println("ffmpeg init: $res") - avstatestore.state.ffmpeg_init_done = true - } + avstatestore.state.ffmpeg_init_do() var audio_in_sources_get: Array = emptyArray() val tmp = AVActivity.ffmpegav_get_in_sources(avstatestore.state.audio_in_device_get(), 0) if (tmp == null) @@ -460,12 +451,7 @@ fun App() maxLines = 1) Box { IconButton(onClick = { - if (!avstatestore.state.ffmpeg_init_done) - { - val res = AVActivity.ffmpegav_init() - println("ffmpeg init: $res") - avstatestore.state.ffmpeg_init_done = true - } + avstatestore.state.ffmpeg_init_do() val video_in_devices_get = AVActivity.ffmpegav_get_video_in_devices() println("ffmpeg video in devices: " + video_in_devices_get.size) video_in_devices.clear() @@ -506,12 +492,7 @@ fun App() IconButton(onClick = { if ((avstatestore.state.video_in_device_get() != null) && (avstatestore.state.video_in_device_get() != "")) { - if (!avstatestore.state.ffmpeg_init_done) - { - val res = AVActivity.ffmpegav_init() - println("ffmpeg init: $res") - avstatestore.state.ffmpeg_init_done = true - } + avstatestore.state.ffmpeg_init_do() var video_in_sources_get: Array = emptyArray() if (avstatestore.state.video_in_device_get() == "video4linux2,v4l2") { diff --git a/src/main/kotlin/com/zoffcc/applications/trifa/AVState.kt b/src/main/kotlin/com/zoffcc/applications/trifa/AVState.kt index 81a65119..a961005a 100644 --- a/src/main/kotlin/com/zoffcc/applications/trifa/AVState.kt +++ b/src/main/kotlin/com/zoffcc/applications/trifa/AVState.kt @@ -1,9 +1,15 @@ package com.zoffcc.applications.trifa +import CAPTURE_VIDEO_FPS +import CAPTURE_VIDEO_HEIGHT +import CAPTURE_VIDEO_WIDTH +import com.zoffcc.applications.ffmpegav.AVActivity +import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_init import global_prefs import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow +import java.nio.ByteBuffer data class AVState(val a: Int) { @@ -26,19 +32,66 @@ data class AVState(val a: Int) CALLAUDIO_CALLING, CALLAUDIO_ENDING } - var ffmpeg_init_done: Boolean = false + private var ffmpeg_init_done: Boolean = false private var audio_in_device = "" private var audio_in_source = "" private var video_in_device = "" private var video_in_source = "" - var calling_state = CALL_STATUS.CALL_NONE - var calling_vstate = CALLVIDEO.CALLVIDEO_NONE - var calling_astate = CALLAUDIO.CALLAUDIO_NONE - var call_with_friend_pubkey: String? = null + private var calling_state = CALL_STATUS.CALL_NONE + private var call_with_friend_pubkey: String? = null init { load_device_information() + ffmpeg_init_done_set(false) + } + + fun ffmpeg_devices_stop() + { + AVActivity.ffmpegav_stop_audio_in_capture() + AVActivity.ffmpegav_close_audio_in_device() + AVActivity.ffmpegav_stop_video_in_capture() + AVActivity.ffmpegav_close_video_in_device() + } + + fun ffmpeg_init_do() + { + if (!ffmpeg_init_done_get()) + { + val res = ffmpegav_init() + println("==================ffmpeg init: $res") + ffmpeg_init_done_set(true) + } + } + + fun call_with_friend_pubkey_get(): String? + { + return call_with_friend_pubkey + } + + fun call_with_friend_pubkey_set(value: String?) + { + call_with_friend_pubkey = value + } + + fun calling_state_get(): CALL_STATUS + { + return calling_state + } + + fun calling_state_set(value: CALL_STATUS) + { + calling_state = value + } + + fun ffmpeg_init_done_get(): Boolean + { + return ffmpeg_init_done + } + + fun ffmpeg_init_done_set(value: Boolean) + { + ffmpeg_init_done = value } fun audio_in_device_get(): String @@ -187,6 +240,164 @@ data class AVState(val a: Int) { } } + + fun start_outgoing_video(friendpubkey: String) + { + if (friendpubkey == null) + { + println("start_outgoing_video: friend pubkey is null!! ERROR !!"); + return + } + println("___________start_outgoing_video_____________") + ffmpeg_init_do() + val video_in_device = video_in_device_get() + val video_in_source = video_in_source_get() + var video_buffer_2: ByteBuffer? = null + + if ((video_in_device != null) && (video_in_device != "")) + { + if ((video_in_source != null) && (video_in_source != "")) + { + println("ffmpeg video in device: " + video_in_device + " " + video_in_source) + val res_vd = AVActivity.ffmpegav_open_video_in_device(video_in_device, video_in_source, + CAPTURE_VIDEO_WIDTH, CAPTURE_VIDEO_HEIGHT, CAPTURE_VIDEO_FPS) + println("ffmpeg open video capture device: $res_vd") + } + } + val frame_width_px1 = CAPTURE_VIDEO_WIDTH + val frame_height_px1 = CAPTURE_VIDEO_HEIGHT + val buffer_size_in_bytes1 = (frame_width_px1 * frame_height_px1 * 3) / 2 + val video_buffer_1 = ByteBuffer.allocateDirect(buffer_size_in_bytes1) + AVActivity.ffmpegav_set_JNI_video_buffer(video_buffer_1, frame_width_px1, frame_height_px1) + val frame_width_px2 = CAPTURE_VIDEO_WIDTH + val frame_height_px2 = CAPTURE_VIDEO_HEIGHT + val y_size = frame_width_px2 * frame_height_px2 + val u_size = (frame_width_px2 * frame_height_px2 / 4) + val v_size = (frame_width_px2 * frame_height_px2 / 4) + val video_buffer_2_y = ByteBuffer.allocateDirect(y_size) + val video_buffer_2_u = ByteBuffer.allocateDirect(u_size) + val video_buffer_2_v = ByteBuffer.allocateDirect(v_size) + AVActivity.ffmpegav_set_JNI_video_buffer2(video_buffer_2_y, video_buffer_2_u, video_buffer_2_v, frame_width_px2, frame_height_px2) + + + val audio_in_device = audio_in_device_get() + val audio_in_source = audio_in_source_get() + var audio_buffer_2: ByteBuffer? = null + + if ((audio_in_device != null) && (audio_in_device != "")) + { + if ((audio_in_source != null) && (audio_in_source != "")) + { + println("ffmpeg audio in device: " + audio_in_device + " " + audio_in_source) + val res_ad = AVActivity.ffmpegav_open_audio_in_device(audio_in_device, audio_in_source) + println("ffmpeg open audio capture device: $res_ad") + } + } + + val buffer_size_in_bytes2 = 50000 // TODO: don't hardcode this + val audio_buffer_1 = ByteBuffer.allocateDirect(buffer_size_in_bytes2) + AVActivity.ffmpegav_set_JNI_audio_buffer2(audio_buffer_1) + + + AVActivity.ffmpegav_set_audio_capture_callback(object : AVActivity.audio_capture_callback + { + override fun onSuccess(read_bytes: Long, out_samples: Int, out_channels: Int, out_sample_rate: Int, pts: Long) + { // Log.i(TAG, "ffmpeg open audio capture onSuccess: $read_bytes $out_samples $out_channels $out_sample_rate $pts") + if ((audio_buffer_2 == null)) + { + audio_buffer_2 = ByteBuffer.allocateDirect(buffer_size_in_bytes2) + MainActivity.set_JNI_audio_buffer(audio_buffer_2) + }/* DEBUG ONLY ---------------------------- + try + { + audio_buffer_1!!.rewind() + val audio_buffer_2_ = ffmpegav_ByteBufferCompat(audio_buffer_1) + Log.i(TAG, "audiobytes1:" + AVActivity.bytesToHex(audio_buffer_2_.array(), 0, 100)) + } catch (e: Exception) + { + e.printStackTrace() + } + DEBUG ONLY ---------------------------- */ + + audio_buffer_2!!.rewind() + audio_buffer_1!!.rewind() + audio_buffer_2!!.put(audio_buffer_1) // can we cache that? what if a friend gets deleted while in a call? and the friend number changes? + val friendnum = MainActivity.tox_friend_by_public_key(friendpubkey) + val tox_audio_res = MainActivity.toxav_audio_send_frame(friend_number = friendnum, sample_count = out_samples.toLong(), channels = out_channels, sampling_rate = out_sample_rate.toLong()) // Log.i(TAG, "tox_audio_res=" + tox_audio_res) + val sample_count_: Int = out_samples + val t_audio_bar_set: Thread = object : Thread() + { + override fun run() + { + var global_audio_in_vu: Float = MainActivity.AUDIO_VU_MIN_VALUE + if (sample_count_ > 0) + { + audio_buffer_1.rewind() + val data_compat = ByteBufferCompat(audio_buffer_1) + val vu_value: Float = AudioBar.audio_vu(data_compat.array(), sample_count_) + global_audio_in_vu = if (vu_value > MainActivity.AUDIO_VU_MIN_VALUE) + { + vu_value + } else + { + 0f + } + } + AudioBar.set_cur_value(global_audio_in_vu.toInt(), AudioBar.audio_in_bar) + } + } + t_audio_bar_set.start()/* DEBUG ONLY ---------------------------- + try + { + audio_buffer_2!!.rewind() + val audio_buffer_2_ = ffmpegav_ByteBufferCompat(audio_buffer_2) + // Log.i(TAG, "audiobytes2:" + AVActivity.bytesToHex(audio_buffer_2_.array(), 0, 100)) + } catch (e: Exception) + { + e.printStackTrace() + } + DEBUG ONLY ---------------------------- */ + } + + override fun onError() + { + } + }) + + AVActivity.ffmpegav_set_video_capture_callback(object : AVActivity.video_capture_callback + { + override fun onSuccess(width: Long, height: Long, pts: Long) + { // Log.i(TAG, "ffmpeg open video capture onSuccess: $width $height $pts") + val frame_width_px: Int = width.toInt() + val frame_height_px: Int = height.toInt() + val buffer_size_in_bytes3 = (frame_width_px * frame_height_px * 1.5f).toInt() + if ((video_buffer_2 == null) || (frame_width_px != frame_width_px2) || (frame_height_px != frame_height_px2)) + { + video_buffer_2 = ByteBuffer.allocateDirect(buffer_size_in_bytes3) + MainActivity.set_JNI_video_buffer2(video_buffer_2, frame_width_px, frame_height_px) + VideoOutFrame.setup_video_out_resolution(frame_width_px, frame_height_px, buffer_size_in_bytes3) + } + video_buffer_2!!.rewind() + video_buffer_2_y.rewind() + video_buffer_2_u.rewind() + video_buffer_2_v.rewind() + video_buffer_2!!.put(video_buffer_2_y) + video_buffer_2!!.put(video_buffer_2_u) + video_buffer_2!!.put(video_buffer_2_v) // can we cache that? what if a friend gets deleted while in a call? and the friend number changes? + val friendnum = MainActivity.tox_friend_by_public_key(friendpubkey) + MainActivity.toxav_video_send_frame_age(friendnum = friendnum, frame_width_px = width.toInt(), frame_height_px = height.toInt(), age_ms = 0) + + video_buffer_2!!.rewind() + VideoOutFrame.new_video_out_frame(video_buffer_2, frame_width_px, frame_height_px) + } + + override fun onError() + { + } + }) + AVActivity.ffmpegav_start_video_in_capture() + AVActivity.ffmpegav_start_audio_in_capture() + } } interface AVStateStore diff --git a/src/main/kotlin/com/zoffcc/applications/trifa/MainActivity.kt b/src/main/kotlin/com/zoffcc/applications/trifa/MainActivity.kt index 546f613e..d4f0de97 100644 --- a/src/main/kotlin/com/zoffcc/applications/trifa/MainActivity.kt +++ b/src/main/kotlin/com/zoffcc/applications/trifa/MainActivity.kt @@ -69,7 +69,6 @@ import org.briarproject.briar.desktop.contact.ContactItem import org.briarproject.briar.desktop.contact.GroupItem import org.briarproject.briar.desktop.contact.GroupPeerItem import set_tox_online_state -import start_outgoing_video import timestampMs import toxdatastore import java.io.File @@ -885,13 +884,13 @@ class MainActivity @JvmStatic fun android_toxav_callback_call_cb_method(friend_number: Long, audio_enabled: Int, video_enabled: Int) { - if (avstatestore.state.calling_state != AVState.CALL_STATUS.CALL_NONE) + if (avstatestore.state.calling_state_get() != AVState.CALL_STATUS.CALL_NONE) { // we are already in some other call state, maybe with another friend return } - if (avstatestore.state.call_with_friend_pubkey != null) + if (avstatestore.state.call_with_friend_pubkey_get() != null) { // we have some call with a friend already return @@ -900,23 +899,23 @@ class MainActivity if (call_answer == 1) { set_av_call_status(1) - avstatestore.state.call_with_friend_pubkey = tox_friend_get_public_key(friend_number) - avstatestore.state.calling_state = AVState.CALL_STATUS.CALL_CALLING - start_outgoing_video(avstatestore.state.call_with_friend_pubkey!!) + avstatestore.state.call_with_friend_pubkey_set(tox_friend_get_public_key(friend_number)) + avstatestore.state.calling_state_set(AVState.CALL_STATUS.CALL_CALLING) + avstatestore.state.start_outgoing_video(avstatestore.state.call_with_friend_pubkey_get()!!) } } @JvmStatic fun android_toxav_callback_video_receive_frame_cb_method(friend_number: Long, frame_width_px: Long, frame_height_px: Long, ystride: Long, ustride: Long, vstride: Long) { - if ((avstatestore.state.call_with_friend_pubkey == null) - || (avstatestore.state.call_with_friend_pubkey != tox_friend_get_public_key(friend_number))) + if ((avstatestore.state.call_with_friend_pubkey_get() == null) + || (avstatestore.state.call_with_friend_pubkey_get() != tox_friend_get_public_key(friend_number))) { // it's not the currently selected friend, so do not play the video frame return } - if (avstatestore.state.calling_state != AVState.CALL_STATUS.CALL_CALLING) + if (avstatestore.state.calling_state_get() != AVState.CALL_STATUS.CALL_CALLING) { // we are not in a call, ignore incoming video frames return; @@ -953,8 +952,8 @@ class MainActivity GlobalScope.launch { try { - if ((avstatestore.state.call_with_friend_pubkey == null) || - (avstatestore.state.call_with_friend_pubkey != tox_friend_get_public_key(friend_number))) + if ((avstatestore.state.call_with_friend_pubkey_get() == null) || + (avstatestore.state.call_with_friend_pubkey_get() != tox_friend_get_public_key(friend_number))) { // not the friend we are in a call with. so ignore callback return@launch @@ -969,27 +968,18 @@ class MainActivity } else if (a_TOXAV_FRIEND_CALL_STATE and ToxVars.TOXAV_FRIEND_CALL_STATE.TOXAV_FRIEND_CALL_STATE_FINISHED.value > 0) { Log.i(TAG, "toxav_call_state:from=$friend_number call ending(1)") - AVActivity.ffmpegav_stop_audio_in_capture() - AVActivity.ffmpegav_close_audio_in_device() - AVActivity.ffmpegav_stop_video_in_capture() - AVActivity.ffmpegav_close_video_in_device() + avstatestore.state.ffmpeg_devices_stop() on_call_ended_actions() - } else if (avstatestore.state.calling_state != AVState.CALL_STATUS.CALL_NONE && + } else if (avstatestore.state.calling_state_get() != AVState.CALL_STATUS.CALL_NONE && a_TOXAV_FRIEND_CALL_STATE == ToxVars.TOXAV_FRIEND_CALL_STATE.TOXAV_FRIEND_CALL_STATE_NONE.value) { Log.i(TAG, "toxav_call_state:from=$friend_number call ending(2)") - AVActivity.ffmpegav_stop_audio_in_capture() - AVActivity.ffmpegav_close_audio_in_device() - AVActivity.ffmpegav_stop_video_in_capture() - AVActivity.ffmpegav_close_video_in_device() + avstatestore.state.ffmpeg_devices_stop() on_call_ended_actions() } else if (a_TOXAV_FRIEND_CALL_STATE and ToxVars.TOXAV_FRIEND_CALL_STATE.TOXAV_FRIEND_CALL_STATE_ERROR.value > 0) { Log.i(TAG, "toxav_call_state:from=$friend_number call ERROR(3)") - AVActivity.ffmpegav_stop_audio_in_capture() - AVActivity.ffmpegav_close_audio_in_device() - AVActivity.ffmpegav_stop_video_in_capture() - AVActivity.ffmpegav_close_video_in_device() + avstatestore.state.ffmpeg_devices_stop() on_call_ended_actions() } } catch (_: Exception) @@ -1001,8 +991,8 @@ class MainActivity fun on_call_ended_actions() { - avstatestore.state.calling_state = AVState.CALL_STATUS.CALL_NONE - avstatestore.state.call_with_friend_pubkey = null + avstatestore.state.calling_state_set(AVState.CALL_STATUS.CALL_NONE) + avstatestore.state.call_with_friend_pubkey_set(null) set_av_call_status(0) Thread.sleep(100) VideoOutFrame.clear_video_out_frame() @@ -1019,14 +1009,14 @@ class MainActivity @JvmStatic fun android_toxav_callback_audio_receive_frame_cb_method(friend_number: Long, sample_count: Long, channels: Int, sampling_rate: Long) { - if ((avstatestore.state.call_with_friend_pubkey == null) - || (avstatestore.state.call_with_friend_pubkey != tox_friend_get_public_key(friend_number))) + if ((avstatestore.state.call_with_friend_pubkey_get() == null) + || (avstatestore.state.call_with_friend_pubkey_get() != tox_friend_get_public_key(friend_number))) { // it's not the currently selected friend, so do not play the audio frame return } - if (avstatestore.state.calling_state != AVState.CALL_STATUS.CALL_CALLING) + if (avstatestore.state.calling_state_get() != AVState.CALL_STATUS.CALL_CALLING) { // we are not in a call, ignore incoming audio frames return diff --git a/src/main/kotlin/com/zoffcc/applications/trifa2/ChatApp.kt b/src/main/kotlin/com/zoffcc/applications/trifa2/ChatApp.kt index b71a4419..39371389 100644 --- a/src/main/kotlin/com/zoffcc/applications/trifa2/ChatApp.kt +++ b/src/main/kotlin/com/zoffcc/applications/trifa2/ChatApp.kt @@ -32,13 +32,8 @@ import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import com.zoffcc.applications.ffmpegav.AVActivity -import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_close_audio_in_device -import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_close_video_in_device -import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_init import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_set_audio_capture_callback import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_set_video_capture_callback -import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_stop_audio_in_capture -import com.zoffcc.applications.ffmpegav.AVActivity.ffmpegav_stop_video_in_capture import com.zoffcc.applications.trifa.AVState import com.zoffcc.applications.trifa.AudioBar import com.zoffcc.applications.trifa.AudioBar.audio_in_bar @@ -110,12 +105,9 @@ fun ChatAppWithScaffold(focusRequester: FocusRequester, displayTextField: Boolea val friendnum = tox_friend_by_public_key(friendpubkey) set_av_call_status(1) - if (avstatestore.state.calling_state == AVState.CALL_STATUS.CALL_CALLING) + if (avstatestore.state.calling_state_get() == AVState.CALL_STATUS.CALL_CALLING) { - ffmpegav_stop_audio_in_capture() - ffmpegav_close_audio_in_device() - ffmpegav_stop_video_in_capture() - ffmpegav_close_video_in_device() + avstatestore.state.ffmpeg_devices_stop() toxav_call_control(friendnum, ToxVars.TOXAV_CALL_CONTROL.TOXAV_CALL_CONTROL_CANCEL.value) on_call_ended_actions() println("toxav: ret 001") @@ -126,8 +118,8 @@ fun ChatAppWithScaffold(focusRequester: FocusRequester, displayTextField: Boolea println("toxav call_res: $call_res") if (call_res == 1) { - avstatestore.state.calling_state = AVState.CALL_STATUS.CALL_CALLING - avstatestore.state.call_with_friend_pubkey = friendpubkey + avstatestore.state.calling_state_set(AVState.CALL_STATUS.CALL_CALLING) + avstatestore.state.call_with_friend_pubkey_set(friendpubkey) println("toxav: set 003") } else @@ -137,7 +129,7 @@ fun ChatAppWithScaffold(focusRequester: FocusRequester, displayTextField: Boolea return@IconButton } - start_outgoing_video(friendpubkey!!) + avstatestore.state.start_outgoing_video(friendpubkey!!) }) { Icon(Icons.Filled.Videocam, null) } @@ -149,181 +141,6 @@ fun ChatAppWithScaffold(focusRequester: FocusRequester, displayTextField: Boolea } } -fun start_outgoing_video(friendpubkey: String) -{ - if (friendpubkey == null) - { - println("start_outgoing_video: friend pubkey is null!! ERROR !!"); - return - } - println("___________start_outgoing_video_____________") - if (!avstatestore.state.ffmpeg_init_done) - { - val res = ffmpegav_init() - println("ffmpeg init: $res") - avstatestore.state.ffmpeg_init_done = true - } - val video_in_device = avstatestore.state.video_in_device_get() - val video_in_source = avstatestore.state.video_in_source_get() - var video_buffer_2: ByteBuffer? = null - - if ((video_in_device != null) && (video_in_device != "")) - { - if ((video_in_source != null) && (video_in_source != "")) - { - println("ffmpeg video in device: " + video_in_device + " " + video_in_source) - val res_vd = AVActivity.ffmpegav_open_video_in_device(video_in_device, video_in_source, CAPTURE_VIDEO_WIDTH, CAPTURE_VIDEO_HEIGHT, 20) - println("ffmpeg open video capture device: $res_vd") - } - } - val frame_width_px1 = CAPTURE_VIDEO_WIDTH - val frame_height_px1 = CAPTURE_VIDEO_HEIGHT - val buffer_size_in_bytes1 = (frame_width_px1 * frame_height_px1 * 3) / 2 - val video_buffer_1 = ByteBuffer.allocateDirect(buffer_size_in_bytes1) - AVActivity.ffmpegav_set_JNI_video_buffer(video_buffer_1, frame_width_px1, frame_height_px1) - val frame_width_px2 = CAPTURE_VIDEO_WIDTH - val frame_height_px2 = CAPTURE_VIDEO_HEIGHT - val y_size = frame_width_px2 * frame_height_px2 - val u_size = (frame_width_px2 * frame_height_px2 / 4) - val v_size = (frame_width_px2 * frame_height_px2 / 4) - val video_buffer_2_y = ByteBuffer.allocateDirect(y_size) - val video_buffer_2_u = ByteBuffer.allocateDirect(u_size) - val video_buffer_2_v = ByteBuffer.allocateDirect(v_size) - AVActivity.ffmpegav_set_JNI_video_buffer2(video_buffer_2_y, video_buffer_2_u, video_buffer_2_v, frame_width_px2, frame_height_px2) - - - val audio_in_device = avstatestore.state.audio_in_device_get() - val audio_in_source = avstatestore.state.audio_in_source_get() - var audio_buffer_2: ByteBuffer? = null - - if ((audio_in_device != null) && (audio_in_device != "")) - { - if ((audio_in_source != null) && (audio_in_source != "")) - { - println("ffmpeg audio in device: " + audio_in_device + " " + audio_in_source) - val res_ad = AVActivity.ffmpegav_open_audio_in_device(audio_in_device, audio_in_source) - println("ffmpeg open audio capture device: $res_ad") - } - } - - val buffer_size_in_bytes2 = 50000 // TODO: don't hardcode this - val audio_buffer_1 = ByteBuffer.allocateDirect(buffer_size_in_bytes2) - AVActivity.ffmpegav_set_JNI_audio_buffer2(audio_buffer_1) - - - ffmpegav_set_audio_capture_callback(object : AVActivity.audio_capture_callback - { - override fun onSuccess(read_bytes: Long, out_samples: Int, out_channels: Int, out_sample_rate: Int, pts: Long) - { - // Log.i(TAG, "ffmpeg open audio capture onSuccess: $read_bytes $out_samples $out_channels $out_sample_rate $pts") - if ((audio_buffer_2 == null)) - { - audio_buffer_2 = ByteBuffer.allocateDirect(buffer_size_in_bytes2) - set_JNI_audio_buffer(audio_buffer_2) - } - - /* DEBUG ONLY ---------------------------- - try - { - audio_buffer_1!!.rewind() - val audio_buffer_2_ = ffmpegav_ByteBufferCompat(audio_buffer_1) - Log.i(TAG, "audiobytes1:" + AVActivity.bytesToHex(audio_buffer_2_.array(), 0, 100)) - } catch (e: Exception) - { - e.printStackTrace() - } - DEBUG ONLY ---------------------------- */ - - audio_buffer_2!!.rewind() - audio_buffer_1!!.rewind() - audio_buffer_2!!.put(audio_buffer_1) - // can we cache that? what if a friend gets deleted while in a call? and the friend number changes? - val friendnum = tox_friend_by_public_key(friendpubkey) - val tox_audio_res = toxav_audio_send_frame( - friend_number = friendnum, - sample_count = out_samples.toLong(), - channels = out_channels, - sampling_rate = out_sample_rate.toLong()) - // Log.i(TAG, "tox_audio_res=" + tox_audio_res) - val sample_count_: Int = out_samples - val t_audio_bar_set: Thread = object : Thread() - { - override fun run() - { - var global_audio_in_vu: Float = AUDIO_VU_MIN_VALUE - if (sample_count_ > 0) - { - audio_buffer_1.rewind() - val data_compat = ByteBufferCompat(audio_buffer_1) - val vu_value: Float = AudioBar.audio_vu(data_compat.array(), sample_count_) - global_audio_in_vu = if (vu_value > AUDIO_VU_MIN_VALUE) - { - vu_value - } else - { - 0f - } - } - AudioBar.set_cur_value(global_audio_in_vu.toInt(), audio_in_bar) - } - } - t_audio_bar_set.start() - - /* DEBUG ONLY ---------------------------- - try - { - audio_buffer_2!!.rewind() - val audio_buffer_2_ = ffmpegav_ByteBufferCompat(audio_buffer_2) - // Log.i(TAG, "audiobytes2:" + AVActivity.bytesToHex(audio_buffer_2_.array(), 0, 100)) - } catch (e: Exception) - { - e.printStackTrace() - } - DEBUG ONLY ---------------------------- */ - } - - override fun onError() - { - } - }) - - ffmpegav_set_video_capture_callback(object : AVActivity.video_capture_callback - { - override fun onSuccess(width: Long, height: Long, pts: Long) - { - // Log.i(TAG, "ffmpeg open video capture onSuccess: $width $height $pts") - val frame_width_px: Int = width.toInt() - val frame_height_px: Int = height.toInt() - val buffer_size_in_bytes3 = (frame_width_px * frame_height_px * 1.5f).toInt() - if ((video_buffer_2 == null) || (frame_width_px != frame_width_px2) || (frame_height_px != frame_height_px2)) - { - video_buffer_2 = ByteBuffer.allocateDirect(buffer_size_in_bytes3) - set_JNI_video_buffer2(video_buffer_2, frame_width_px, frame_height_px) - VideoOutFrame.setup_video_out_resolution(frame_width_px, frame_height_px, buffer_size_in_bytes3) - } - video_buffer_2!!.rewind() - video_buffer_2_y.rewind() - video_buffer_2_u.rewind() - video_buffer_2_v.rewind() - video_buffer_2!!.put(video_buffer_2_y) - video_buffer_2!!.put(video_buffer_2_u) - video_buffer_2!!.put(video_buffer_2_v) - // can we cache that? what if a friend gets deleted while in a call? and the friend number changes? - val friendnum = tox_friend_by_public_key(friendpubkey) - toxav_video_send_frame_age(friendnum = friendnum, frame_width_px = width.toInt(), frame_height_px = height.toInt(), age_ms = 0) - - video_buffer_2!!.rewind() - VideoOutFrame.new_video_out_frame(video_buffer_2, frame_width_px, frame_height_px) - } - - override fun onError() - { - } - }) - AVActivity.ffmpegav_start_video_in_capture() - AVActivity.ffmpegav_start_audio_in_capture() -} - @Composable fun GroupAppWithScaffold(focusRequester: FocusRequester, displayTextField: Boolean = true, groupList: StateGroups, ui_scale: Float) {