From 3a6504bcfc232f7be5f77791585a5bc28bdd27f2 Mon Sep 17 00:00:00 2001 From: Ramil Rakhmatullin Date: Fri, 23 Feb 2024 12:44:33 +0700 Subject: [PATCH 1/5] Added GlRectDrawerWrapper --- .../webrtc/FlutterRTCVideoRenderer.java | 5 +- .../cloudwebrtc/webrtc/GlRectDrawerWrapper.kt | 180 ++++++++++++++++++ 2 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/FlutterRTCVideoRenderer.java b/android/src/main/java/com/cloudwebrtc/webrtc/FlutterRTCVideoRenderer.java index 61500ac844..27d520a281 100755 --- a/android/src/main/java/com/cloudwebrtc/webrtc/FlutterRTCVideoRenderer.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/FlutterRTCVideoRenderer.java @@ -6,6 +6,7 @@ import com.cloudwebrtc.webrtc.utils.AnyThreadSink; import com.cloudwebrtc.webrtc.utils.ConstraintsMap; import com.cloudwebrtc.webrtc.utils.EglUtils; +import com.cloudwebrtc.webrtc.GlRectDrawerWrapper; import java.util.List; @@ -103,7 +104,7 @@ public void onFrameResolutionChanged( public FlutterRTCVideoRenderer(SurfaceTexture texture, TextureRegistry.SurfaceTextureEntry entry) { this.surfaceTextureRenderer = new SurfaceTextureRenderer(""); listenRendererEvents(); - surfaceTextureRenderer.init(EglUtils.getRootEglBaseContext(), rendererEvents); + surfaceTextureRenderer.init(EglUtils.getRootEglBaseContext(), rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawerWrapper()); surfaceTextureRenderer.surfaceCreated(texture); this.texture = texture; @@ -237,7 +238,7 @@ private void tryAddRendererToVideoTrack() throws Exception { surfaceTextureRenderer.release(); listenRendererEvents(); - surfaceTextureRenderer.init(sharedContext, rendererEvents); + surfaceTextureRenderer.init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawerWrapper()); surfaceTextureRenderer.surfaceCreated(texture); videoTrack.addSink(surfaceTextureRenderer); diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt new file mode 100644 index 0000000000..c53528b320 --- /dev/null +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt @@ -0,0 +1,180 @@ +package com.cloudwebrtc.webrtc + +import android.opengl.GLES20 +import org.webrtc.GlRectDrawer +import org.webrtc.GlShader +import org.webrtc.GlUtil +import java.util.HashMap + +class GlRectDrawerWrapper : GlRectDrawer() { + + companion object { + var isNightMode = true + + // 0.1 -> 2.5 // def 1 + var TEXTURE_BRIGHTNESS_PERCENT = 2.5f + + // -0.5 -> 0.5 // def 0 + var TEXTURE_CONTRAST_PERCENT = 0.05f + + private var R = TEXTURE_CONTRAST_PERCENT.toString() + private var G = (TEXTURE_CONTRAST_PERCENT).toString() // +0.035f + private var B = TEXTURE_CONTRAST_PERCENT.toString() + + + private val FULL_RECTANGLE_BUF = GlUtil.createFloatBuffer(floatArrayOf(-1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f)) + private val FULL_RECTANGLE_TEX_BUF = GlUtil.createFloatBuffer(floatArrayOf(0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f)) + } + + + private val shaderOes = "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 interp_tc;\n\n" + + "" + + "uniform samplerExternalOES oes_tex;\n\n" + + "" + + "void main() {\n" + + " gl_FragColor = texture2D(oes_tex, interp_tc);\n" + + "}" + + private val shaderYuv = "precision mediump float;\n" + + "varying vec2 interp_tc;\n\n" + + "uniform sampler2D y_tex;\n" + + "uniform sampler2D u_tex;\n" + + "uniform sampler2D v_tex;\n\n" + + "void main() {\n" + + " float y = texture2D(y_tex, interp_tc).r;\n" + + " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" + + " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" + + " gl_FragColor = vec4(y + 1.403 * v, y - 0.344 * u - 0.714 * v, y + 1.77 * u, 1);\n" + + "}" + + // night mode + + private val shaderOesNightMode = "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 interp_tc;\n\n" + + "" + + "uniform samplerExternalOES oes_tex;\n\n" + + "" + + "void main() {\n" + + " vec4 brightness = vec4($R, $G, $B, 0);\n" + + " gl_FragColor = (texture2D(oes_tex, interp_tc) + brightness) * $TEXTURE_BRIGHTNESS_PERCENT;\n" + + "}" + + private val shaderYuvNightMode = "precision mediump float;\n" + + "varying vec2 interp_tc;\n\n" + + "uniform sampler2D y_tex;\n" + + "uniform sampler2D u_tex;\n" + + "uniform sampler2D v_tex;\n\n" + + "void main() {\n" + + " vec4 brightness = vec4($R, $G, $B, 0);\n" + + " float y = texture2D(y_tex, interp_tc).r;\n" + + " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" + + " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" + + " gl_FragColor = (vec4(y + 1.403 * v, y - 0.344 * u - 0.714 * v, y + 1.77 * u, 1) + brightness) * $TEXTURE_BRIGHTNESS_PERCENT ;\n" + + "}" + + override fun drawOes(oesTextureId: Int, texMatrix: FloatArray?, frameWidth: Int, frameHeight: Int, viewportX: Int, viewportY: Int, viewportWidth: Int, viewportHeight: Int) { + prepareShader(if (isNightMode) shaderOesNightMode else shaderOes, texMatrix!!) + GLES20.glActiveTexture(33984) + GLES20.glBindTexture(36197, oesTextureId) + drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight) + GLES20.glBindTexture(36197, 0) + } + + override fun drawRgb(textureId: Int, texMatrix: FloatArray?, frameWidth: Int, frameHeight: Int, viewportX: Int, viewportY: Int, viewportWidth: Int, viewportHeight: Int) { + super.drawRgb(textureId, texMatrix, frameWidth, frameHeight, viewportX, viewportY, viewportWidth, viewportHeight) + } + + + override fun drawYuv(yuvTextures: IntArray?, texMatrix: FloatArray?, frameWidth: Int, frameHeight: Int, viewportX: Int, viewportY: Int, viewportWidth: Int, viewportHeight: Int) { + prepareShader(if (isNightMode) shaderYuvNightMode else shaderYuv, texMatrix!!) + //{ + var i = 0 + while (i < 3) { + GLES20.glActiveTexture('蓀'.toInt() + i) + GLES20.glBindTexture(3553, yuvTextures!![i]) + ++i + } + //} + + drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight) + + //{ + i = 0 + while (i < 3) { + GLES20.glActiveTexture('蓀'.toInt() + i) + GLES20.glBindTexture(3553, 0) + ++i + } + //} + } + + + + + private val shaders: HashMap = HashMap() + + private fun prepareShader(fragmentShader: String, texMatrix: FloatArray) { + val shader: Shader? + if (shaders.containsKey(fragmentShader)) { + shader = shaders[fragmentShader] + } else { + shader = Shader(fragmentShader) + shaders[fragmentShader] = shader + shader.glShader.useProgram() + when { + fragmentShader === shaderYuv -> { + GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0) + GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1) + GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2) + } + fragmentShader === shaderYuvNightMode -> { + GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0) + GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1) + GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2) + } + fragmentShader === shaderOes -> { + GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0) + + } + fragmentShader === shaderOesNightMode -> { + GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0) + + } + else -> { + throw IllegalStateException("Unknown fragment shader: $fragmentShader") + } + } + GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values.") + shader.run { + glShader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF) + glShader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF) + } + } + shader!!.glShader.useProgram() + GLES20.glUniformMatrix4fv(shader.texMatrixLocation, 1, false, texMatrix, 0) + } + + private fun drawRectangle(x: Int, y: Int, width: Int, height: Int) { + GLES20.glViewport(x, y, width, height) + GLES20.glDrawArrays(5, 0, 4) + } + + class Shader(fragmentShader: String?) { + val glShader: GlShader = GlShader( + "varying vec2 interp_tc;\n" + + "attribute vec4 in_pos;\n" + + "attribute vec4 in_tc;\n\nuniform mat4 texMatrix;\n\n" + + "" + + "void main() {\n" + + " gl_Position = in_pos;\n" + + " interp_tc = (texMatrix * in_tc).xy;\n" + + "}\n", fragmentShader) + val texMatrixLocation: Int + init { + texMatrixLocation = glShader.getUniformLocation("texMatrix") + } + } +} \ No newline at end of file From 15864ed66e8dc47073439cc8bd4012fa1cc403b2 Mon Sep 17 00:00:00 2001 From: Ramil Rakhmatullin Date: Fri, 23 Feb 2024 14:23:39 +0700 Subject: [PATCH 2/5] Added switchNightMode --- .../main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt | 1 + .../java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java | 5 +++++ lib/src/helper.dart | 3 +++ 3 files changed, 9 insertions(+) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt index c53528b320..4c5dd5f91e 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt @@ -9,6 +9,7 @@ import java.util.HashMap class GlRectDrawerWrapper : GlRectDrawer() { companion object { + @JvmField var isNightMode = true // 0.1 -> 2.5 // def 1 diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java index 164a0483d4..9a2feeb6ba 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java @@ -35,6 +35,7 @@ import com.cloudwebrtc.webrtc.utils.PermissionUtils; import com.cloudwebrtc.webrtc.utils.Utils; import com.cloudwebrtc.webrtc.videoRecorder.VideoRecorderFactory; +import com.cloudwebrtc.webrtc.GlRectDrawerWrapper; import com.twilio.audioswitch.AudioDevice; @@ -741,6 +742,10 @@ public void onMethodCall(MethodCall call, @NonNull Result notSafeResult) { } break; } + case "switchNightMode": { + GlRectDrawerWrapper.isNightMode = !GlRectDrawerWrapper.isNightMode; + result.success(null); + } case "captureFrame": { String path = call.argument("path"); String videoTrackId = call.argument("trackId"); diff --git a/lib/src/helper.dart b/lib/src/helper.dart index 0ee12c3014..95b7792915 100644 --- a/lib/src/helper.dart +++ b/lib/src/helper.dart @@ -157,4 +157,7 @@ class Helper { AppleNativeAudioManagement.setAppleAudioConfiguration( AppleNativeAudioManagement.getAppleAudioConfigurationForMode(mode, preferSpeakerOutput: preferSpeakerOutput)); + + static Future switchNightMode() => + WebRTC.invokeMethod('switchNightMode'); } From 366a119aad5de5916567924269aad680adac0359 Mon Sep 17 00:00:00 2001 From: Ramil Rakhmatullin Date: Fri, 23 Feb 2024 14:39:04 +0700 Subject: [PATCH 3/5] Added exception for switchNightMode if the platform is not ANDROID --- lib/src/helper.dart | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/src/helper.dart b/lib/src/helper.dart index 95b7792915..81fa20234c 100644 --- a/lib/src/helper.dart +++ b/lib/src/helper.dart @@ -158,6 +158,11 @@ class Helper { AppleNativeAudioManagement.getAppleAudioConfigurationForMode(mode, preferSpeakerOutput: preferSpeakerOutput)); - static Future switchNightMode() => - WebRTC.invokeMethod('switchNightMode'); + static Future switchNightMode() { + if (WebRTC.platformIsAndroid) { + return WebRTC.invokeMethod('switchNightMode'); + } else { + throw UnimplementedError(); + } + } } From 6e1fd50fd0ba1d65a61487ff4b522d52c4c6f585 Mon Sep 17 00:00:00 2001 From: Ramil Rakhmatullin Date: Fri, 23 Feb 2024 14:46:37 +0700 Subject: [PATCH 4/5] Made isNightMode false as default --- .../src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt index 4c5dd5f91e..e759ed1c5c 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt +++ b/android/src/main/java/com/cloudwebrtc/webrtc/GlRectDrawerWrapper.kt @@ -10,7 +10,7 @@ class GlRectDrawerWrapper : GlRectDrawer() { companion object { @JvmField - var isNightMode = true + var isNightMode = false // 0.1 -> 2.5 // def 1 var TEXTURE_BRIGHTNESS_PERCENT = 2.5f From c5677873e9bfed352ed88d83198066e119dcb0d9 Mon Sep 17 00:00:00 2001 From: Ramil Rakhmatullin Date: Fri, 23 Feb 2024 16:27:51 +0700 Subject: [PATCH 5/5] Added isNightModeEnabled getter --- .../cloudwebrtc/webrtc/MethodCallHandlerImpl.java | 5 ++++- lib/src/helper.dart | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java index 9a2feeb6ba..e048d686ec 100644 --- a/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java +++ b/android/src/main/java/com/cloudwebrtc/webrtc/MethodCallHandlerImpl.java @@ -744,7 +744,10 @@ public void onMethodCall(MethodCall call, @NonNull Result notSafeResult) { } case "switchNightMode": { GlRectDrawerWrapper.isNightMode = !GlRectDrawerWrapper.isNightMode; - result.success(null); + result.success(GlRectDrawerWrapper.isNightMode); + } + case "isNightModeEnabled": { + result.success(GlRectDrawerWrapper.isNightMode); } case "captureFrame": { String path = call.argument("path"); diff --git a/lib/src/helper.dart b/lib/src/helper.dart index 81fa20234c..9aa8665f13 100644 --- a/lib/src/helper.dart +++ b/lib/src/helper.dart @@ -158,9 +158,19 @@ class Helper { AppleNativeAudioManagement.getAppleAudioConfigurationForMode(mode, preferSpeakerOutput: preferSpeakerOutput)); - static Future switchNightMode() { + static Future switchNightMode() async { if (WebRTC.platformIsAndroid) { - return WebRTC.invokeMethod('switchNightMode'); + final isEnabled = await WebRTC.invokeMethod('switchNightMode'); + return isEnabled; + } else { + throw UnimplementedError(); + } + } + + static Future get isNightModeEnabled async { + if (WebRTC.platformIsAndroid) { + final isEnabled = await WebRTC.invokeMethod('isNightModeEnabled'); + return isEnabled; } else { throw UnimplementedError(); }