From d2b4106259072a5e509f398837448d9bb4a80803 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 18 Mar 2025 23:09:32 -0400 Subject: [PATCH] start of audit support --- include/video/hls_writer.h | 7 +- include/video/mp4_writer.h | 22 ++- include/video/stream_protocol.h | 3 + include/video/stream_reader.h | 1 + src/video/hls_writer.c | 83 ++++++++--- src/video/mp4_writer.c | 237 ++++++++++++++++++++------------ src/video/stream_protocol.c | 17 +++ src/video/stream_reader.c | 47 +++++-- 8 files changed, 293 insertions(+), 124 deletions(-) diff --git a/include/video/hls_writer.h b/include/video/hls_writer.h index 5033da71..faeeae80 100644 --- a/include/video/hls_writer.h +++ b/include/video/hls_writer.h @@ -28,8 +28,13 @@ typedef struct hls_writer_t { int initialized; time_t last_cleanup_time; + // Stream indexes in output context + int video_stream_idx; + int audio_stream_idx; + // Per-stream DTS tracking - stream_dts_info_t dts_tracker; + stream_dts_info_t video_dts_tracker; + stream_dts_info_t audio_dts_tracker; } hls_writer_t; /** diff --git a/include/video/mp4_writer.h b/include/video/mp4_writer.h index 759148f6..10a58b98 100644 --- a/include/video/mp4_writer.h +++ b/include/video/mp4_writer.h @@ -11,17 +11,29 @@ // Forward declaration of the MP4 writer structure typedef struct mp4_writer mp4_writer_t; +// Forward declaration for the timestamp tracking structure +typedef struct { + int64_t first_dts; + int64_t first_pts; + int64_t last_dts; + AVRational time_base; + int initialized; +} mp4_timestamp_info_t; + // Full definition of the MP4 writer structure struct mp4_writer { char output_path[MAX_PATH_LENGTH]; char stream_name[MAX_STREAM_NAME]; AVFormatContext *output_ctx; + + // Stream indexes in output context int video_stream_idx; - int has_audio; - int64_t first_dts; - int64_t first_pts; - int64_t last_dts; - AVRational time_base; + int audio_stream_idx; + + // Per-stream timestamp tracking + mp4_timestamp_info_t video_ts_info; + mp4_timestamp_info_t audio_ts_info; + int is_initialized; time_t creation_time; }; diff --git a/include/video/stream_protocol.h b/include/video/stream_protocol.h index a42a167f..102cf717 100644 --- a/include/video/stream_protocol.h +++ b/include/video/stream_protocol.h @@ -16,4 +16,7 @@ int open_input_stream(AVFormatContext **input_ctx, const char *url, int protocol // Find video stream index in the input context int find_video_stream_index(AVFormatContext *input_ctx); +// Find audio stream index in the input context +int find_audio_stream_index(AVFormatContext *input_ctx); + #endif // STREAM_PROTOCOL_H diff --git a/include/video/stream_reader.h b/include/video/stream_reader.h index b983cbb5..792381a6 100644 --- a/include/video/stream_reader.h +++ b/include/video/stream_reader.h @@ -17,6 +17,7 @@ typedef struct { pthread_t thread; AVFormatContext *input_ctx; int video_stream_idx; + int audio_stream_idx; // Added audio stream index int dedicated; // Flag to indicate if this is a dedicated stream reader // Callback function for packet processing diff --git a/src/video/hls_writer.c b/src/video/hls_writer.c index 27eca2a3..897213ee 100644 --- a/src/video/hls_writer.c +++ b/src/video/hls_writer.c @@ -325,6 +325,14 @@ int hls_writer_initialize(hls_writer_t *writer, const AVStream *input_stream) { return -1; } + // Determine if this is a video or audio stream + enum AVMediaType codec_type = input_stream->codecpar->codec_type; + + if (codec_type != AVMEDIA_TYPE_VIDEO && codec_type != AVMEDIA_TYPE_AUDIO) { + log_warn("Unsupported stream type for HLS: %d", codec_type); + return -1; + } + // Create output stream AVStream *out_stream = avformat_new_stream(writer->output_ctx, NULL); if (!out_stream) { @@ -343,29 +351,46 @@ int hls_writer_initialize(hls_writer_t *writer, const AVStream *input_stream) { // Set stream time base out_stream->time_base = input_stream->time_base; + + // Store the stream index based on type + if (codec_type == AVMEDIA_TYPE_VIDEO) { + writer->video_stream_idx = out_stream->index; + writer->video_dts_tracker.time_base = input_stream->time_base; + writer->video_dts_tracker.initialized = 0; + log_info("Added video stream to HLS output at index %d", writer->video_stream_idx); + } else if (codec_type == AVMEDIA_TYPE_AUDIO) { + writer->audio_stream_idx = out_stream->index; + writer->audio_dts_tracker.time_base = input_stream->time_base; + writer->audio_dts_tracker.initialized = 0; + log_info("Added audio stream to HLS output at index %d", writer->audio_stream_idx); + } - // SIMPLIFIED APPROACH: Let FFmpeg handle manifest file creation - // Just use basic options for reliability - AVDictionary *options = NULL; + // If this is the first stream, write the header + if (!writer->initialized) { + // SIMPLIFIED APPROACH: Let FFmpeg handle manifest file creation + // Just use basic options for reliability + AVDictionary *options = NULL; - // Write the header - ret = avformat_write_header(writer->output_ctx, &options); - if (ret < 0) { - char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); - log_error("Failed to write HLS header: %s", error_buf); - av_dict_free(&options); - return ret; - } + // Write the header + ret = avformat_write_header(writer->output_ctx, &options); + if (ret < 0) { + char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); + log_error("Failed to write HLS header: %s", error_buf); + av_dict_free(&options); + return ret; + } - av_dict_free(&options); + av_dict_free(&options); - // SIMPLIFIED APPROACH: Let FFmpeg handle manifest file creation completely - // Remove the fallback mechanism as it might interfere with FFmpeg's own manifest creation - log_info("Letting FFmpeg handle manifest file creation for stream %s", writer->stream_name); + // SIMPLIFIED APPROACH: Let FFmpeg handle manifest file creation completely + // Remove the fallback mechanism as it might interfere with FFmpeg's own manifest creation + log_info("Letting FFmpeg handle manifest file creation for stream %s", writer->stream_name); - writer->initialized = 1; - log_info("Initialized HLS writer for stream %s", writer->stream_name); + writer->initialized = 1; + log_info("Initialized HLS writer for stream %s", writer->stream_name); + } + return 0; } @@ -502,15 +527,33 @@ int hls_writer_write_packet(hls_writer_t *writer, const AVPacket *pkt, const AVS return -1; } + // Determine if this is a video or audio packet and use the appropriate DTS tracker + enum AVMediaType codec_type = input_stream->codecpar->codec_type; + stream_dts_info_t *dts_tracker; + + if (codec_type == AVMEDIA_TYPE_VIDEO) { + dts_tracker = &writer->video_dts_tracker; + out_pkt.stream_index = writer->video_stream_idx; + log_debug("Processing video packet for HLS stream %s", writer->stream_name); + } else if (codec_type == AVMEDIA_TYPE_AUDIO) { + dts_tracker = &writer->audio_dts_tracker; + out_pkt.stream_index = writer->audio_stream_idx; + log_debug("Processing audio packet for HLS stream %s", writer->stream_name); + } else { + log_warn("Unsupported packet type for HLS: %d", codec_type); + av_packet_unref(&out_pkt); + return -1; + } + // Initialize DTS tracker for this stream if needed - stream_dts_info_t *dts_tracker = &writer->dts_tracker; if (!dts_tracker->initialized) { dts_tracker->first_dts = pkt->dts != AV_NOPTS_VALUE ? pkt->dts : pkt->pts; dts_tracker->last_dts = dts_tracker->first_dts; dts_tracker->time_base = input_stream->time_base; dts_tracker->initialized = 1; - log_debug("Initialized DTS tracker for HLS stream %s: first_dts=%lld, time_base=%d/%d", + log_debug("Initialized DTS tracker for HLS %s stream %s: first_dts=%lld, time_base=%d/%d", + codec_type == AVMEDIA_TYPE_VIDEO ? "video" : "audio", writer->stream_name, (long long)dts_tracker->first_dts, dts_tracker->time_base.num, diff --git a/src/video/mp4_writer.c b/src/video/mp4_writer.c index 655f69e3..15da0646 100644 --- a/src/video/mp4_writer.c +++ b/src/video/mp4_writer.c @@ -47,8 +47,16 @@ mp4_writer_t *mp4_writer_create(const char *output_path, const char *stream_name // Initialize writer strncpy(writer->output_path, output_path, sizeof(writer->output_path) - 1); strncpy(writer->stream_name, stream_name, sizeof(writer->stream_name) - 1); - writer->first_dts = AV_NOPTS_VALUE; - writer->last_dts = AV_NOPTS_VALUE; + writer->video_stream_idx = -1; + writer->audio_stream_idx = -1; + writer->video_ts_info.first_dts = AV_NOPTS_VALUE; + writer->video_ts_info.first_pts = AV_NOPTS_VALUE; + writer->video_ts_info.last_dts = AV_NOPTS_VALUE; + writer->video_ts_info.initialized = 0; + writer->audio_ts_info.first_dts = AV_NOPTS_VALUE; + writer->audio_ts_info.first_pts = AV_NOPTS_VALUE; + writer->audio_ts_info.last_dts = AV_NOPTS_VALUE; + writer->audio_ts_info.initialized = 0; writer->is_initialized = 0; writer->creation_time = time(NULL); @@ -104,21 +112,32 @@ static int mp4_writer_initialize(mp4_writer_t *writer, const AVPacket *pkt, cons // Log the full output path log_info("Initializing MP4 writer to output file: %s", writer->output_path); - // Create output format context - ret = avformat_alloc_output_context2(&writer->output_ctx, NULL, "mp4", writer->output_path); - if (ret < 0 || !writer->output_ctx) { - char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); - log_error("Failed to create output format context for MP4 writer: %s", error_buf); - return -1; + // Determine if this is a video or audio stream + enum AVMediaType codec_type = input_stream->codecpar->codec_type; + + // Create output format context if it doesn't exist yet + if (!writer->output_ctx) { + ret = avformat_alloc_output_context2(&writer->output_ctx, NULL, "mp4", writer->output_path); + if (ret < 0 || !writer->output_ctx) { + char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); + log_error("Failed to create output format context for MP4 writer: %s", error_buf); + free(dir_path); + return -1; + } + + // Add metadata + av_dict_set(&writer->output_ctx->metadata, "title", writer->stream_name, 0); + av_dict_set(&writer->output_ctx->metadata, "encoder", "LightNVR", 0); } - // Add video stream + // Create output stream AVStream *out_stream = avformat_new_stream(writer->output_ctx, NULL); if (!out_stream) { log_error("Failed to create output stream for MP4 writer"); avformat_free_context(writer->output_ctx); writer->output_ctx = NULL; + free(dir_path); return -1; } @@ -130,67 +149,82 @@ static int mp4_writer_initialize(mp4_writer_t *writer, const AVPacket *pkt, cons log_error("Failed to copy codec parameters for MP4 writer: %s", error_buf); avformat_free_context(writer->output_ctx); writer->output_ctx = NULL; + free(dir_path); return -1; } // Set stream time base out_stream->time_base = input_stream->time_base; - writer->time_base = input_stream->time_base; - - // Store video stream index - writer->video_stream_idx = 0; // We only have one stream - - // Add metadata - av_dict_set(&writer->output_ctx->metadata, "title", writer->stream_name, 0); - av_dict_set(&writer->output_ctx->metadata, "encoder", "LightNVR", 0); + + // Store stream index and initialize timestamp tracking based on type + if (codec_type == AVMEDIA_TYPE_VIDEO) { + writer->video_stream_idx = out_stream->index; + writer->video_ts_info.time_base = input_stream->time_base; + writer->video_ts_info.initialized = 0; + log_info("Added video stream to MP4 output at index %d", writer->video_stream_idx); + } else if (codec_type == AVMEDIA_TYPE_AUDIO) { + writer->audio_stream_idx = out_stream->index; + writer->audio_ts_info.time_base = input_stream->time_base; + writer->audio_ts_info.initialized = 0; + log_info("Added audio stream to MP4 output at index %d", writer->audio_stream_idx); + } else { + log_warn("Unsupported stream type for MP4: %d", codec_type); + free(dir_path); + return -1; + } - // Set options for fast start (moov atom at beginning of file) - AVDictionary *opts = NULL; - av_dict_set(&opts, "movflags", "+faststart", 0); // Ensure moov atom is at start of file + // Write header if this is the first initialization + if (!writer->is_initialized) { + // Set options for fast start (moov atom at beginning of file) + AVDictionary *opts = NULL; + av_dict_set(&opts, "movflags", "+faststart", 0); // Ensure moov atom is at start of file - // Open output file - ret = avio_open(&writer->output_ctx->pb, writer->output_path, AVIO_FLAG_WRITE); - if (ret < 0) { - char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); - log_error("Failed to open output file for MP4 writer: %s (error: %s)", - writer->output_path, error_buf); + // Open output file + ret = avio_open(&writer->output_ctx->pb, writer->output_path, AVIO_FLAG_WRITE); + if (ret < 0) { + char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); + log_error("Failed to open output file for MP4 writer: %s (error: %s)", + writer->output_path, error_buf); + + // Try to diagnose the issue + struct stat st; + if (stat(dir_path, &st) != 0) { + log_error("Directory does not exist: %s", dir_path); + } else if (!S_ISDIR(st.st_mode)) { + log_error("Path exists but is not a directory: %s", dir_path); + } else if (access(dir_path, W_OK) != 0) { + log_error("Directory is not writable: %s", dir_path); + } - // Try to diagnose the issue - struct stat st; - if (stat(dir_path, &st) != 0) { - log_error("Directory does not exist: %s", dir_path); - } else if (!S_ISDIR(st.st_mode)) { - log_error("Path exists but is not a directory: %s", dir_path); - } else if (access(dir_path, W_OK) != 0) { - log_error("Directory is not writable: %s", dir_path); + avformat_free_context(writer->output_ctx); + writer->output_ctx = NULL; + av_dict_free(&opts); + free(dir_path); + return -1; } - avformat_free_context(writer->output_ctx); - writer->output_ctx = NULL; - av_dict_free(&opts); - return -1; - } + // Write file header + ret = avformat_write_header(writer->output_ctx, &opts); + if (ret < 0) { + char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); + log_error("Failed to write header for MP4 writer: %s", error_buf); + avio_closep(&writer->output_ctx->pb); + avformat_free_context(writer->output_ctx); + writer->output_ctx = NULL; + av_dict_free(&opts); + free(dir_path); + return -1; + } - // Write file header - ret = avformat_write_header(writer->output_ctx, &opts); - if (ret < 0) { - char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE); - log_error("Failed to write header for MP4 writer: %s", error_buf); - avio_closep(&writer->output_ctx->pb); - avformat_free_context(writer->output_ctx); - writer->output_ctx = NULL; av_dict_free(&opts); - return -1; + writer->is_initialized = 1; + log_info("Successfully initialized MP4 writer for stream %s at %s", + writer->stream_name, writer->output_path); } - av_dict_free(&opts); - - writer->is_initialized = 1; - log_info("Successfully initialized MP4 writer for stream %s at %s", - writer->stream_name, writer->output_path); - + free(dir_path); return 0; } @@ -241,49 +275,69 @@ int mp4_writer_write_packet(mp4_writer_t *writer, const AVPacket *in_pkt, const return ret; } + // Determine if this is a video or audio packet and use the appropriate timestamp tracker + enum AVMediaType codec_type = input_stream->codecpar->codec_type; + mp4_timestamp_info_t *ts_info; + int stream_idx; + + if (codec_type == AVMEDIA_TYPE_VIDEO) { + ts_info = &writer->video_ts_info; + stream_idx = writer->video_stream_idx; + log_debug("Processing video packet for MP4 stream %s", writer->stream_name); + } else if (codec_type == AVMEDIA_TYPE_AUDIO) { + ts_info = &writer->audio_ts_info; + stream_idx = writer->audio_stream_idx; + log_debug("Processing audio packet for MP4 stream %s", writer->stream_name); + } else { + log_warn("Unsupported packet type for MP4: %d", codec_type); + av_packet_unref(&pkt); + return -1; + } + // Enhanced timestamp handling to fix non-monotonic DTS issues - if (writer->first_dts == AV_NOPTS_VALUE) { - // Wait for a key frame to start if possible - if (!(in_pkt->flags & AV_PKT_FLAG_KEY)) { - // Skip non-key frames at the beginning + if (!ts_info->initialized) { + // For video, wait for a key frame to start if possible + if (codec_type == AVMEDIA_TYPE_VIDEO && !(in_pkt->flags & AV_PKT_FLAG_KEY)) { + // Skip non-key frames at the beginning for video log_debug("Skipping non-key frame at start of MP4 recording"); av_packet_unref(&pkt); return 0; // Return success but don't process this packet } - // First packet (key frame) - use its DTS as reference - writer->first_dts = pkt.dts != AV_NOPTS_VALUE ? pkt.dts : pkt.pts; - writer->first_pts = pkt.pts != AV_NOPTS_VALUE ? pkt.pts : pkt.dts; - - // Initialize last_dts to avoid comparison with AV_NOPTS_VALUE - writer->last_dts = writer->first_dts; + // First packet - use its DTS as reference + ts_info->first_dts = pkt.dts != AV_NOPTS_VALUE ? pkt.dts : pkt.pts; + ts_info->first_pts = pkt.pts != AV_NOPTS_VALUE ? pkt.pts : pkt.dts; + ts_info->last_dts = ts_info->first_dts; + ts_info->initialized = 1; // For the first packet, set timestamps to 0 pkt.dts = 0; - pkt.pts = pkt.pts != AV_NOPTS_VALUE ? pkt.pts - writer->first_pts : 0; + pkt.pts = pkt.pts != AV_NOPTS_VALUE ? pkt.pts - ts_info->first_pts : 0; // Ensure PTS is valid (not less than DTS) if (pkt.pts < pkt.dts) { pkt.pts = pkt.dts; } - log_debug("MP4 writer initialized with first_dts=%lld, first_pts=%lld", - (long long)writer->first_dts, (long long)writer->first_pts); + log_debug("MP4 writer initialized %s timestamps: first_dts=%lld, first_pts=%lld", + codec_type == AVMEDIA_TYPE_VIDEO ? "video" : "audio", + (long long)ts_info->first_dts, (long long)ts_info->first_pts); } else { // Check for timestamp discontinuities (common in RTSP streams) - int64_t expected_dts = writer->last_dts + - av_rescale_q(1, input_stream->time_base, writer->time_base); + int64_t expected_dts = ts_info->last_dts + + av_rescale_q(1, input_stream->time_base, ts_info->time_base); int64_t dts_diff = 0; if (pkt.dts != AV_NOPTS_VALUE) { - dts_diff = pkt.dts - writer->first_dts; + dts_diff = pkt.dts - ts_info->first_dts; // If there's a large backward jump in DTS (stream reset or loop) - if (dts_diff < 0 || pkt.dts < writer->last_dts) { + if (dts_diff < 0 || pkt.dts < ts_info->last_dts) { // This is a discontinuity - log it - log_warn("DTS discontinuity detected: last_dts=%lld, current_dts=%lld, diff=%lld", - (long long)writer->last_dts, (long long)pkt.dts, - (long long)(pkt.dts - writer->last_dts)); + log_warn("DTS discontinuity detected in %s stream: last_dts=%lld, current_dts=%lld, diff=%lld", + codec_type == AVMEDIA_TYPE_VIDEO ? "video" : "audio", + (long long)ts_info->last_dts, (long long)pkt.dts, + (long long)(pkt.dts - ts_info->last_dts)); // Handle the discontinuity by continuing from the last DTS // Add a small increment to ensure monotonic increase @@ -298,8 +352,8 @@ int mp4_writer_write_packet(mp4_writer_t *writer, const AVPacket *in_pkt, const pkt.dts = dts_diff; // Ensure monotonic increase - if (pkt.dts <= writer->last_dts) { - pkt.dts = writer->last_dts + 1; + if (pkt.dts <= ts_info->last_dts) { + pkt.dts = ts_info->last_dts + 1; } } } else { @@ -310,7 +364,7 @@ int mp4_writer_write_packet(mp4_writer_t *writer, const AVPacket *in_pkt, const // Handle PTS (presentation timestamp) if (pkt.pts != AV_NOPTS_VALUE) { // Calculate proper offset from the first PTS - int64_t pts_diff = pkt.pts - writer->first_pts; + int64_t pts_diff = pkt.pts - ts_info->first_pts; // If PTS goes backwards or would be less than DTS, adjust it if (pts_diff < 0 || pkt.pts < pkt.dts) { @@ -331,14 +385,17 @@ int mp4_writer_write_packet(mp4_writer_t *writer, const AVPacket *in_pkt, const } // Update last DTS to maintain monotonic increase - writer->last_dts = pkt.dts; + ts_info->last_dts = pkt.dts; - // Write packet - pkt.stream_index = writer->video_stream_idx; + // Set the correct stream index + pkt.stream_index = stream_idx; // Log key frame information for debugging - if (pkt.flags & AV_PKT_FLAG_KEY) { - log_debug("Writing keyframe to MP4: pts=%lld, dts=%lld", + if (codec_type == AVMEDIA_TYPE_VIDEO && (pkt.flags & AV_PKT_FLAG_KEY)) { + log_debug("Writing video keyframe to MP4: pts=%lld, dts=%lld", + (long long)pkt.pts, (long long)pkt.dts); + } else if (codec_type == AVMEDIA_TYPE_AUDIO) { + log_debug("Writing audio packet to MP4: pts=%lld, dts=%lld", (long long)pkt.pts, (long long)pkt.dts); } @@ -408,9 +465,13 @@ void mp4_writer_close(mp4_writer_t *writer) { // Safely copy other fields creation_time = writer->creation_time; was_initialized = writer->is_initialized; - first_dts = writer->first_dts; - last_dts = writer->last_dts; - time_base = writer->time_base; + + // Use video timestamp info for duration calculation + if (writer->video_ts_info.initialized) { + first_dts = writer->video_ts_info.first_dts; + last_dts = writer->video_ts_info.last_dts; + time_base = writer->video_ts_info.time_base; + } // Extract the output context and immediately set it to NULL to prevent double-free output_ctx = writer->output_ctx; diff --git a/src/video/stream_protocol.c b/src/video/stream_protocol.c index 7cca2c8a..e9796fe5 100644 --- a/src/video/stream_protocol.c +++ b/src/video/stream_protocol.c @@ -216,3 +216,20 @@ int find_video_stream_index(AVFormatContext *input_ctx) { return -1; } + +/** + * Find audio stream index in the input context + */ +int find_audio_stream_index(AVFormatContext *input_ctx) { + if (!input_ctx) { + return -1; + } + + for (unsigned int i = 0; i < input_ctx->nb_streams; i++) { + if (input_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + return i; + } + } + + return -1; +} diff --git a/src/video/stream_reader.c b/src/video/stream_reader.c index 6071223c..0197f91b 100644 --- a/src/video/stream_reader.c +++ b/src/video/stream_reader.c @@ -117,6 +117,14 @@ static void *stream_reader_thread(void *arg) { return NULL; } + // Find audio stream (optional) + ctx->audio_stream_idx = find_audio_stream_index(ctx->input_ctx); + if (ctx->audio_stream_idx == -1) { + log_info("No audio stream found in %s", ctx->config.url); + } else { + log_info("Found audio stream at index %d in %s", ctx->audio_stream_idx, ctx->config.url); + } + // Initialize packet pkt = av_packet_alloc(); if (!pkt) { @@ -407,6 +415,15 @@ static void *stream_reader_thread(void *arg) { continue; // Keep trying } + // Find audio stream again (optional) + ctx->audio_stream_idx = find_audio_stream_index(ctx->input_ctx); + if (ctx->audio_stream_idx == -1) { + log_info("No audio stream found in %s after reconnect", ctx->config.url); + } else { + log_info("Found audio stream at index %d in %s after reconnect", + ctx->audio_stream_idx, ctx->config.url); + } + continue; } else { log_ffmpeg_error(ret, "Error reading frame"); @@ -414,19 +431,24 @@ static void *stream_reader_thread(void *arg) { } } - // Process video packets directly - if (pkt->stream_index == ctx->video_stream_idx) { - // Check if this is a key frame (for logging) - int is_key_frame = (pkt->flags & AV_PKT_FLAG_KEY) ? 1 : 0; + // Process video and audio packets + if (pkt->stream_index == ctx->video_stream_idx || + (ctx->audio_stream_idx != -1 && pkt->stream_index == ctx->audio_stream_idx)) { - if (is_key_frame) { - log_debug("Processing keyframe: pts=%lld, dts=%lld, size=%d", + // Check if this is a key frame (for logging, video only) + if (pkt->stream_index == ctx->video_stream_idx) { + int is_key_frame = (pkt->flags & AV_PKT_FLAG_KEY) ? 1 : 0; + + if (is_key_frame) { + log_debug("Processing video keyframe: pts=%lld, dts=%lld, size=%d", + (long long)pkt->pts, (long long)pkt->dts, pkt->size); + } + } else { + // Log audio packets at debug level + log_debug("Processing audio packet: pts=%lld, dts=%lld, size=%d", (long long)pkt->pts, (long long)pkt->dts, pkt->size); } - // Removed packet throttling mechanism to improve quality - // Always process all frames for better quality - // CRITICAL FIX: Double-check that we're still running and have a valid callback // This prevents use-after-free issues during shutdown if (!ctx->running || !ctx->packet_callback) { @@ -438,9 +460,14 @@ static void *stream_reader_thread(void *arg) { packet_callback_t callback = ctx->packet_callback; void *callback_data = ctx->callback_data; + // Get the appropriate stream based on packet type + const AVStream *stream = (pkt->stream_index == ctx->video_stream_idx) ? + ctx->input_ctx->streams[ctx->video_stream_idx] : + ctx->input_ctx->streams[ctx->audio_stream_idx]; + // Final check before calling the callback if (callback) { - ret = callback(pkt, ctx->input_ctx->streams[ctx->video_stream_idx], callback_data); + ret = callback(pkt, stream, callback_data); if (ret < 0) { log_error("Packet callback failed for stream %s: %d", ctx->config.name, ret); // Continue anyway