From 234859a1118f37f5a64c1614f789c9f133d2236b Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Tue, 10 Sep 2024 22:28:34 +0100 Subject: [PATCH 1/8] If avcodec_send_packet() failed, still try to flush the decoder Suspecting the packet decoding failures which happen sometimes cause a memory leak. ==6284== 3,139,437 bytes in 3 blocks are possibly lost in loss record 3,795 of 3,797 ==6284== at 0x484E366: posix_memalign (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==6284== by 0x4FBA544: av_malloc (mem.c:105) ==6284== by 0x4FA0559: av_buffer_alloc (buffer.c:82) ==6284== by 0x4FA05D1: av_buffer_allocz (buffer.c:95) ==6284== by 0x4FA0D7D: pool_alloc_buffer (buffer.c:369) ==6284== by 0x4FA0D7D: av_buffer_pool_get (buffer.c:407) ==6284== by 0x4AC122E: video_get_buffer (get_buffer.c:240) ==6284== by 0x4AC122E: avcodec_default_get_buffer2 (get_buffer.c:276) ==6284== by 0x4AB3BCA: ff_get_buffer (decode.c:1673) ==6284== by 0x4B0E243: alloc_picture (h264_slice.c:195) ==6284== by 0x4B0E243: h264_frame_start (h264_slice.c:528) ==6284== by 0x4B11E9E: h264_field_start (h264_slice.c:1601) ==6284== by 0x4B11E9E: ff_h264_queue_decode_slice (h264_slice.c:2132) ==6284== by 0x4B16986: decode_nal_units (h264dec.c:651) ==6284== by 0x4B16986: h264_decode_frame (h264dec.c:1047) ==6284== by 0x4AB1EEB: decode_simple_internal (decode.c:430) ==6284== by 0x4AB1EEB: decode_simple_receive_frame (decode.c:610) ==6284== by 0x4AB1EEB: decode_receive_frame_internal (decode.c:638) ==6284== by 0x4AB23FB: avcodec_send_packet (decode.c:735) ==6284== ==6284== 3,140,064 bytes in 6 blocks are still reachable in loss record 3,796 of 3,797 ==6284== at 0x484E366: posix_memalign (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==6284== by 0x4FBA544: av_malloc (mem.c:105) ==6284== by 0x4FA0559: av_buffer_alloc (buffer.c:82) ==6284== by 0x4FA05D1: av_buffer_allocz (buffer.c:95) ==6284== by 0x4FA0D7D: pool_alloc_buffer (buffer.c:369) ==6284== by 0x4FA0D7D: av_buffer_pool_get (buffer.c:407) ==6284== by 0x4B0E36D: alloc_picture (h264_slice.c:248) ==6284== by 0x4B0E36D: h264_frame_start (h264_slice.c:528) ==6284== by 0x4B11E9E: h264_field_start (h264_slice.c:1601) ==6284== by 0x4B11E9E: ff_h264_queue_decode_slice (h264_slice.c:2132) ==6284== by 0x4B16986: decode_nal_units (h264dec.c:651) ==6284== by 0x4B16986: h264_decode_frame (h264dec.c:1047) ==6284== by 0x4AB1EEB: decode_simple_internal (decode.c:430) ==6284== by 0x4AB1EEB: decode_simple_receive_frame (decode.c:610) ==6284== by 0x4AB1EEB: decode_receive_frame_internal (decode.c:638) ==6284== by 0x4AB23FB: avcodec_send_packet (decode.c:735) ==6284== by 0x134FFB: motion_processor::run() (motion_processor.cpp:139) ==6284== by 0x13577C: bc_mproc_thread(void*) (motion_processor.cpp:853) ==6284== ==6284== 6,278,874 bytes in 6 blocks are still reachable in loss record 3,797 of 3,797 ==6284== at 0x484E366: posix_memalign (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==6284== by 0x4FBA544: av_malloc (mem.c:105) ==6284== by 0x4FA0559: av_buffer_alloc (buffer.c:82) ==6284== by 0x4FA05D1: av_buffer_allocz (buffer.c:95) ==6284== by 0x4FA0D7D: pool_alloc_buffer (buffer.c:369) ==6284== by 0x4FA0D7D: av_buffer_pool_get (buffer.c:407) ==6284== by 0x4AC122E: video_get_buffer (get_buffer.c:240) ==6284== by 0x4AC122E: avcodec_default_get_buffer2 (get_buffer.c:276) ==6284== by 0x4AB3BCA: ff_get_buffer (decode.c:1673) ==6284== by 0x4B0E243: alloc_picture (h264_slice.c:195) ==6284== by 0x4B0E243: h264_frame_start (h264_slice.c:528) ==6284== by 0x4B11E9E: h264_field_start (h264_slice.c:1601) ==6284== by 0x4B11E9E: ff_h264_queue_decode_slice (h264_slice.c:2132) ==6284== by 0x4B16986: decode_nal_units (h264dec.c:651) ==6284== by 0x4B16986: h264_decode_frame (h264dec.c:1047) ==6284== by 0x4AB1EEB: decode_simple_internal (decode.c:430) ==6284== by 0x4AB1EEB: decode_simple_receive_frame (decode.c:610) ==6284== by 0x4AB1EEB: decode_receive_frame_internal (decode.c:638) ==6284== by 0x4AB23FB: avcodec_send_packet (decode.c:735) --- server/motion_processor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/motion_processor.cpp b/server/motion_processor.cpp index 05581678..f76459e3 100644 --- a/server/motion_processor.cpp +++ b/server/motion_processor.cpp @@ -143,7 +143,7 @@ void motion_processor::run() bc_log(Warning, "motion processor: avcodec_send_packet failed: %s", error); } - while (ret >= 0) { + while (true) { ret = avcodec_receive_frame(decode_ctx, frame); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) break; From 089e6de08de8028785bb16812cef16e4e07e5e4d Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 10:28:17 +0100 Subject: [PATCH 2/8] motion_processor: release cv::Mat objects ==6284== 1,036,944 bytes in 2 blocks are possibly lost in loss record 3,790 of 3,797 ==6284== at 0x4846828: malloc (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==6284== by 0x66E9628: cv::fastMalloc(unsigned long) (in /usr/lib/x86_64-linux-gnu/libopencv_core.so.4.6.0) ==6284== by 0x67B851B: ??? (in /usr/lib/x86_64-linux-gnu/libopencv_core.so.4.6.0) ==6284== by 0x67AE636: cv::Mat::create(int, int const*, int) (in /usr/lib/x86_64-linux-gnu/libopencv_core.so.4.6.0) ==6284== by 0x67AE98F: cv::Mat::create(int, int, int) (in /usr/lib/x86_64-linux-gnu/libopencv_core.so.4.6.0) ==6284== by 0x13354C: motion_processor::detect_opencv(AVFrame*) (motion_processor.cpp:577) ==6284== by 0x1355B2: detect (motion_processor.cpp:255) ==6284== by 0x1355B2: motion_processor::run() (motion_processor.cpp:182) ==6284== by 0x13577C: bc_mproc_thread(void*) (motion_processor.cpp:853) ==6284== by 0x736EA93: start_thread (pthread_create.c:447) ==6284== by 0x73FBA33: clone (clone.S:100) --- server/motion_processor.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/motion_processor.cpp b/server/motion_processor.cpp index f76459e3..fa6fb7d3 100644 --- a/server/motion_processor.cpp +++ b/server/motion_processor.cpp @@ -55,6 +55,9 @@ motion_processor::~motion_processor() } #endif + for (int i = 0; i < sizeof(m_refFrames); i++) { + m_refFrames[i].release(); + } delete output_source; } @@ -576,6 +579,7 @@ int motion_processor::detect_opencv(AVFrame *rawFrame) // first things first... get the original frame from the AVFrame and convert it to an OpenCV Grayscale Mat, and the "downscale" size. cv::Mat m = cv::Mat(dst_h, dst_w, CV_8UC1); if (convert_AVFrame_to_grayMat(rawFrame, m) == 0) { + m.release(); return 0; // error converting frame... } @@ -616,6 +620,7 @@ int motion_processor::detect_opencv(AVFrame *rawFrame) if (m_motionDebug) check_for_new_debug_event(ret); + m.release(); return ret; } @@ -627,8 +632,10 @@ int motion_processor::detect_opencv_advanced(AVFrame *rawFrame) // first things first... get the original frame from the AVFrame and convert it to an OpenCV Grayscale Mat, and the "downscale" size. cv::Mat m = cv::Mat(dst_h, dst_w, CV_8UC1); - if (convert_AVFrame_to_grayMat(rawFrame, m) == 0) + if (convert_AVFrame_to_grayMat(rawFrame, m) == 0) { + m.release(); return 0; // error converting frame... + } // do a quick(-ish) blur to reduce noise... cv::GaussianBlur(m, m, cv::Size(21, 21), 0); @@ -662,6 +669,7 @@ int motion_processor::detect_opencv_advanced(AVFrame *rawFrame) } m_motionTriggered = (num_md_frames >= m_minMotionFrames); + m.release(); return m_motionTriggered; } From d5ed51ac85ceb1596dbca68b9a88731111f5dfbd Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 12:00:41 +0100 Subject: [PATCH 3/8] libav: sort features enabled Nothing new added, only sorting into categories and within categories was applied. --- misc/BCMK | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/misc/BCMK b/misc/BCMK index 89066380..6ed36ff6 100644 --- a/misc/BCMK +++ b/misc/BCMK @@ -54,70 +54,70 @@ libav/config.mak: \ --disable-everything \ \ - --enable-indev=v4l2 \ - --enable-vaapi \ --enable-hwaccel=h264_vaapi \ --enable-hwaccel=hevc_vaapi \ + --enable-indev=v4l2 \ + --enable-vaapi \ \ --enable-protocol=file \ - --enable-protocol=pipe \ --enable-protocol=http \ --enable-protocol=https \ + --enable-protocol=pipe \ --enable-protocol=tls \ \ + --enable-muxer=hls \ + --enable-muxer=image2 \ --enable-muxer=matroska \ --enable-muxer=mjpeg \ - --enable-muxer=rtp \ --enable-muxer=mp4 \ - --enable-muxer=image2 \ + --enable-muxer=mpegts \ --enable-muxer=rawvideo \ + --enable-muxer=rtp \ \ - --enable-demuxer=rtsp \ + --enable-demuxer=concat \ --enable-demuxer=matroska \ --enable-demuxer=mjpeg \ --enable-demuxer=rawvideo \ - --enable-demuxer=concat \ + --enable-demuxer=rtsp \ \ --enable-bsf=aac_adtstoasc \ - --enable-bsf=h264_mp4toannexb \ --enable-bsf=extract_extradata \ + --enable-bsf=h264_mp4toannexb \ \ + --enable-decoder=aac \ + --enable-decoder=ac3 \ --enable-decoder=h264 \ --enable-decoder=hevc \ - --enable-decoder=mpeg4 \ --enable-decoder=mjpeg \ - --enable-decoder=rawvideo \ - --enable-decoder=aac \ - --enable-decoder=ac3 \ --enable-decoder=mp2 \ --enable-decoder=mp3 \ + --enable-decoder=mpeg4 \ + --enable-decoder=pcm_alaw \ --enable-decoder=pcm_f32le \ --enable-decoder=pcm_f64le \ - --enable-decoder=pcm_s16le \ - --enable-decoder=pcm_alaw \ --enable-decoder=pcm_mulaw \ + --enable-decoder=pcm_s16le \ + --enable-decoder=rawvideo \ \ --enable-parser=h264 \ --enable-parser=hevc \ - --enable-parser=mpeg4video \ --enable-parser=mjpeg \ + --enable-parser=mpeg4video \ \ - --enable-encoder=mjpeg \ - --enable-encoder=mpeg4 \ - --enable-encoder=rawvideo \ --enable-encoder=aac \ --enable-encoder=h264_vaapi \ - --enable-encoder=mjpeg_vaapi \ --enable-encoder=hevc_vaapi \ + --enable-encoder=mjpeg \ + --enable-encoder=mjpeg_vaapi \ + --enable-encoder=mpeg4 \ + --enable-encoder=rawvideo \ \ - --enable-filter=scale \ - --enable-filter=fps \ - --enable-filter=scale_vaapi \ --enable-filter=aresample \ - --enable-filter=hwupload \ + --enable-filter=fps \ --enable-filter=hwdownload \ - --enable-muxer=hls \ - --enable-muxer=mpegts \ + --enable-filter=hwupload \ + --enable-filter=scale \ + --enable-filter=scale_vaapi \ \ From ab11b7b4f41ee202fae5996811d4c3703aa3f9ab Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 12:04:54 +0100 Subject: [PATCH 4/8] libav: enable features for sending synthetic stream via RTSP --- misc/BCMK | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/BCMK b/misc/BCMK index 6ed36ff6..95530c42 100644 --- a/misc/BCMK +++ b/misc/BCMK @@ -73,6 +73,7 @@ libav/config.mak: --enable-muxer=mpegts \ --enable-muxer=rawvideo \ --enable-muxer=rtp \ + --enable-muxer=rtsp \ \ --enable-demuxer=concat \ --enable-demuxer=matroska \ @@ -112,12 +113,14 @@ libav/config.mak: --enable-encoder=mpeg4 \ --enable-encoder=rawvideo \ \ + --enable-filter=aevalsrc \ --enable-filter=aresample \ --enable-filter=fps \ --enable-filter=hwdownload \ --enable-filter=hwupload \ --enable-filter=scale \ --enable-filter=scale_vaapi \ + --enable-filter=testsrc \ \ From 8a15e717d7fc83585af64160862499072a4f7ef1 Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 21:25:36 +0100 Subject: [PATCH 5/8] Fix memleak: free frame data when snapshot encoding fails This might help against this big leak (representing 64 minutes of operation): ==16436== 684,426,964 bytes in 5,941 blocks are definitely lost in loss record 3,813 of 3,813 ==16436== at 0x484E366: posix_memalign (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==16436== by 0x4FBD544: av_malloc (mem.c:105) ==16436== by 0x4FB85A9: av_image_alloc (imgutils.c:248) ==16436== by 0x1229A6: snapshot_writer::scale_frame(AVFrame*, bool&) (media_writer.cpp:612) ==16436== by 0x122B2F: snapshot_writer::write_frame(AVFrame*) (media_writer.cpp:659) ==16436== by 0x1231FF: push_packet (media_writer.cpp:522) ==16436== by 0x1231FF: push_packet (media_writer.cpp:471) ==16436== by 0x1231FF: snapshot_writer::feed(stream_packet const&) (media_writer.cpp:431) ==16436== by 0x139B13: recorder::run() (recorder.cpp:139) ==16436== by 0x7029BB3: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33) ==16436== by 0x7370A93: start_thread (pthread_create.c:447) ==16436== by 0x73FDA33: clone (clone.S:100) --- server/media_writer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/server/media_writer.cpp b/server/media_writer.cpp index b88d6a6b..fe664e23 100644 --- a/server/media_writer.cpp +++ b/server/media_writer.cpp @@ -663,6 +663,7 @@ int snapshot_writer::write_frame(AVFrame *rawFrame) if (ret < 0) { bc_avlog(ret, "avcodec_send_frame: snapshot encoding failed"); + if (allocated) av_freep(&frame->data[0]); av_frame_free(&frame); return -1; } From 0bed67bc78dd92906584991d58418117567145d5 Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 23:26:14 +0100 Subject: [PATCH 6/8] Fix memleak: free the opts dict after HLS avformat_write_header() ==16436== 1,934,398 bytes in 44,986 blocks are indirectly lost in loss record 3,809 of 3,813 ==16436== at 0x484DB80: realloc (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==16436== by 0x4FBD777: av_strdup (mem.c:275) ==16436== by 0x4FA8DD0: av_dict_set (dict.c:99) ==16436== by 0x1251D4: bc_streaming_hls_packet_write(bc_record*, stream_packet const&) (streaming.cpp:362) ==16436== by 0x12064C: bc_record::run() (bc-thread.cpp:434) ==16436== by 0x12173C: bc_device_thread(void*) (bc-thread.cpp:142) ==16436== by 0x7370A93: start_thread (pthread_create.c:447) ==16436== by 0x73FDA33: clone (clone.S:100) ==16436== ==16436== 3,779,364 (720,000 direct, 3,059,364 indirect) bytes in 45,000 blocks are definitely lost in loss record 3,810 of 3,813 ==16436== at 0x484E366: posix_memalign (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==16436== by 0x4FBD544: av_malloc (mem.c:105) ==16436== by 0x4FBD70C: av_mallocz (mem.c:256) ==16436== by 0x4FA8E49: av_dict_set (dict.c:112) ==16436== by 0x1251D4: bc_streaming_hls_packet_write(bc_record*, stream_packet const&) (streaming.cpp:362) ==16436== by 0x12064C: bc_record::run() (bc-thread.cpp:434) ==16436== by 0x12173C: bc_device_thread(void*) (bc-thread.cpp:142) ==16436== by 0x7370A93: start_thread (pthread_create.c:447) ==16436== by 0x73FDA33: clone (clone.S:100) --- server/streaming.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/streaming.cpp b/server/streaming.cpp index b0d73c31..2bf1a652 100644 --- a/server/streaming.cpp +++ b/server/streaming.cpp @@ -360,11 +360,14 @@ int bc_streaming_hls_packet_write(struct bc_record *bc_rec, const stream_packet if (bc_rec->hls_stream_ctx[ctx_index]->streams[0]->codecpar->codec_id == AV_CODEC_ID_MPEG4) { // Write header for every MP4 fragment to be used as independent segments in HLS playlist av_dict_set(&muxer_opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0); - if (avformat_write_header(bc_rec->hls_stream_ctx[ctx_index], &muxer_opts) < 0) { + int ret; + ret = avformat_write_header(bc_rec->hls_stream_ctx[ctx_index], &muxer_opts); + av_dict_free(&muxer_opts); + if (ret < 0) { bc_rec->log.log(Error, "Failed to write header in fragmented MP4 payload"); return -1; } } return 1; -} \ No newline at end of file +} From 122260163e0b4b6c9c465d0fb947c2d7d442b5eb Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Thu, 12 Sep 2024 00:18:38 +0100 Subject: [PATCH 7/8] Put prepared valgrind command line into systemd service file --- debian/bluecherry.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/bluecherry.service b/debian/bluecherry.service index 7c7049b1..0f8f7599 100644 --- a/debian/bluecherry.service +++ b/debian/bluecherry.service @@ -6,7 +6,7 @@ After=syslog.target network.target mysql.service Type=simple User=bluecherry Group=bluecherry -#ExecStart=/usr/sbin/bc-server -s -u bluecherry -g bluecherry +#ExecStart=/usr/bin/valgrind --tool=memcheck --leak-check=yes --leak-check=full --show-leak-kinds=all --log-file=/tmp/bluecherry-valgrind.log /usr/sbin/bc-server -s ExecStart=/usr/sbin/bc-server -s Restart=on-failure From 25f0d28914f76a2c9bb10a93bae050f406fb39cf Mon Sep 17 00:00:00 2001 From: Andriy Utkin Date: Wed, 18 Sep 2024 23:48:27 +0100 Subject: [PATCH 8/8] Put prepared rr command line into systemd service file --- debian/bluecherry.service | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/bluecherry.service b/debian/bluecherry.service index 0f8f7599..ff71bbc1 100644 --- a/debian/bluecherry.service +++ b/debian/bluecherry.service @@ -7,6 +7,7 @@ Type=simple User=bluecherry Group=bluecherry #ExecStart=/usr/bin/valgrind --tool=memcheck --leak-check=yes --leak-check=full --show-leak-kinds=all --log-file=/tmp/bluecherry-valgrind.log /usr/sbin/bc-server -s +#ExecStart=/usr/bin/rr record -o /tmp/bluecherry.rr /usr/sbin/bc-server -s ExecStart=/usr/sbin/bc-server -s Restart=on-failure