From ca6556d3e6d91463b299239ebb0b25b9dcf6c2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20V=C3=B6lcker?= <65532189+bjornvolcker@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:02:07 +0100 Subject: [PATCH] Synchronizes SEIs at beginning of a recording (#274) When validating a stream/recording SEIs are not synchronized with the I-frames. There can for examples be multiple SEIs in the beginning that were generated by frames prior to the frames present in the recording. This commit test validates each SEI for every possible way before either accepting or discarding it. Co-authored-by: bjornvolcker --- lib/src/signed_video_h26x_auth.c | 45 +++++++------ lib/src/signed_video_h26x_nalu_list.c | 27 ++++---- tests/check/check_signed_video_auth.c | 94 ++++++--------------------- 3 files changed, 57 insertions(+), 109 deletions(-) diff --git a/lib/src/signed_video_h26x_auth.c b/lib/src/signed_video_h26x_auth.c index 3d544d7..6d74456 100644 --- a/lib/src/signed_video_h26x_auth.c +++ b/lib/src/signed_video_h26x_auth.c @@ -710,9 +710,9 @@ validate_authenticity(signed_video_t *self) valid = SV_AUTH_RESULT_SIGNATURE_PRESENT; num_expected_nalus = -1; num_received_nalus = -1; - // If validation was tried with the very first SEI in stream it cannot be part at. - // Reset the first validation to be able to validate a segment in the middle of the stream. - self->validation_flags.reset_first_validation = (self->gop_info->num_sent_nalus == 1); + // If no valid NAL Units were found, reset validation to be able to make more attepts to + // synchronize the SEIs. + self->validation_flags.reset_first_validation = !has_valid_nalus; } } if (latest->public_key_has_changed) valid = SV_AUTH_RESULT_NOT_OK; @@ -835,6 +835,7 @@ prepare_for_validation(signed_video_t *self) SV_TRY() h26x_nalu_list_item_t *sei = h26x_nalu_list_get_next_sei_item(nalu_list); if (sei) { + sei->in_validation = true; if (!sei->has_been_decoded) { // Decode the SEI and set signature->hash self->latest_validation->public_key_has_changed = false; @@ -1047,21 +1048,20 @@ maybe_validate_gop(signed_video_t *self, h26x_nalu_t *nalu) svrc_t status = SV_UNKNOWN_FAILURE; SV_TRY() + bool update_validation_status = false; bool public_key_has_changed = false; char sei_validation_status = 'U'; // Keep validating as long as there are pending GOPs. bool stop_validating = false; while (has_pending_gop(self) && !stop_validating) { // Initialize latest validation. - if (!self->validation_flags.has_auth_result) { + if (!self->validation_flags.has_auth_result || validation_flags->is_first_validation) { latest->authenticity = SV_AUTH_RESULT_SIGNATURE_PRESENT; latest->number_of_expected_picture_nalus = 0; latest->number_of_received_picture_nalus = 0; latest->number_of_pending_picture_nalus = -1; latest->public_key_has_changed = public_key_has_changed; - } - if (validation_flags->is_first_validation) { - // Reset in_validation. + // Reset |in_validation|. update_sei_in_validation(self, true, NULL, NULL); } @@ -1081,30 +1081,35 @@ maybe_validate_gop(signed_video_t *self, h26x_nalu_t *nalu) // Fetch the |tmp_validation_status| for later use. update_sei_in_validation(self, false, &sei_validation_status, NULL); } - // Update |validation_status|. - SV_THROW(h26x_nalu_list_update_status(nalu_list, true)); // The flag |is_first_validation| is used to ignore the first validation if we start the // validation in the middle of a stream. Now it is time to reset it. validation_flags->is_first_validation = !validation_flags->signing_present; + if (validation_flags->reset_first_validation) { + validation_flags->is_first_validation = true; + validation_flags->reset_first_validation = false; + } else { + update_validation_status = true; + } self->gop_info->verified_signature_hash = -1; self->validation_flags.has_auth_result = true; public_key_has_changed |= latest->public_key_has_changed; // Pass on public key failure. + } - if (validation_flags->is_first_validation) { - // Reset any set linked hashes if the session is still waiting for a first validation. - reset_linked_hash(self); - } - // All statistics but pending NALUs have already been collected. - latest->number_of_pending_picture_nalus = h26x_nalu_list_num_pending_items(nalu_list); - - DEBUG_LOG("Validated GOP as %s", kAuthResultValidStr[latest->authenticity]); - DEBUG_LOG("Expected number of NALUs = %d", latest->number_of_expected_picture_nalus); - DEBUG_LOG("Received number of NALUs = %d", latest->number_of_received_picture_nalus); - DEBUG_LOG("Number of pending NALUs = %d", latest->number_of_pending_picture_nalus); + SV_THROW(h26x_nalu_list_update_status(nalu_list, update_validation_status)); + if (validation_flags->is_first_validation) { + update_sei_in_validation(self, false, NULL, &sei_validation_status); + // Reset any set linked hashes if the session is still waiting for a first validation. + reset_linked_hash(self); } + // All statistics but pending NALUs have already been collected. + latest->number_of_pending_picture_nalus = h26x_nalu_list_num_pending_items(nalu_list); + DEBUG_LOG("Validated GOP as %s", kAuthResultValidStr[latest->authenticity]); + DEBUG_LOG("Expected number of NALUs = %d", latest->number_of_expected_picture_nalus); + DEBUG_LOG("Received number of NALUs = %d", latest->number_of_received_picture_nalus); + DEBUG_LOG("Number of pending NALUs = %d", latest->number_of_pending_picture_nalus); SV_CATCH() SV_DONE(status) diff --git a/lib/src/signed_video_h26x_nalu_list.c b/lib/src/signed_video_h26x_nalu_list.c index 8116ad5..9ba801d 100644 --- a/lib/src/signed_video_h26x_nalu_list.c +++ b/lib/src/signed_video_h26x_nalu_list.c @@ -415,32 +415,25 @@ h26x_nalu_list_remove_missing_items(h26x_nalu_list_t *list) bool found_first_pending_nalu = false; bool found_decoded_sei = false; + int num_removed_items = 0; h26x_nalu_list_item_t *item = list->first_item; while (item && !(found_first_pending_nalu && found_decoded_sei)) { // Reset the invalid verification failure if we have not past the first pending item. // Remove the missing NALU in the front. - if (item->tmp_validation_status == 'M' && (item == list->first_item)) { + if (item->tmp_validation_status == 'M' && item->in_validation) { const h26x_nalu_list_item_t *item_to_remove = item; item = item->next; h26x_nalu_list_remove_and_free_item(list, item_to_remove); + num_removed_items++; continue; } - if (item->has_been_decoded && item->tmp_validation_status != 'U') { - // Usually, these items were added because we verified hashes with a SEI not associated with - // this recording. This can happen if we export to file or fast forward in a recording. The - // SEI used to generate these missing items is set to 'U'. - item->tmp_validation_status = 'U'; + if (item->has_been_decoded && item->tmp_validation_status != 'U' && item->in_validation) { found_decoded_sei = true; } - // TODO: Resetting the item validation in the current GOP may affect the validation of the next - // GOP. This needs to be fixed. - if (item->tmp_validation_status == 'N') { - // Reset validation status to 'P' for the next validation. - item->tmp_validation_status = 'P'; - } item = item->next; if (item && item->tmp_validation_status == 'P') found_first_pending_nalu = true; } + if (num_removed_items > 0) DEBUG_LOG("Removed %d missing items to list", num_removed_items); } /* Searches for, and returns, the next pending SEI item. */ @@ -482,8 +475,14 @@ h26x_nalu_list_get_stats(const h26x_nalu_list_t *list, continue; } if (item->tmp_validation_status == 'M') local_num_missing_nalus++; - if (item->tmp_validation_status == 'N' || item->tmp_validation_status == 'E') - local_num_invalid_nalus++; + if (item->nalu && item->nalu->is_gop_sei) { + if (item->in_validation && + (item->tmp_validation_status == 'N' || item->tmp_validation_status == 'E')) + local_num_invalid_nalus++; + } else { + if (item->tmp_validation_status == 'N' || item->tmp_validation_status == 'E') + local_num_invalid_nalus++; + } if (item->tmp_validation_status == '.') { // Do not count SEIs, since they are marked valid if the signature could be verified, which // happens for out-of-sync SEIs for example. diff --git a/tests/check/check_signed_video_auth.c b/tests/check/check_signed_video_auth.c index 1f08dcb..9fb4860 100644 --- a/tests/check/check_signed_video_auth.c +++ b/tests/check/check_signed_video_auth.c @@ -880,15 +880,6 @@ START_TEST(all_seis_arrive_late_two_gops_scrapped) // IPPSPIPPSPIPPSSPISPISP // - // IPPS -> (signature) -> PPPU 3 pending - // IPPSPIPPS -> (invalid) -> NNNUNPPP. 3 pending - // IPPSPIPPS -> (invalid) -> NNN.NPPP. 3 pending - // IPPSS -> (invalid) -> NMNN.. 0 pending, 1 missing - // PIS -> (invalid) -> MMMNP. 1 pending, 3 missing - // ISPIS -> (valid) -> ...P. 1 pending - // 11 pending - // IPPSPIPPSPIPPSSPISPISP (future) - // // IPPS -> (signature) -> PPPU 3 pending // IPPSPIPPS -> (signature) -> PPPUPPPPU 7 pending // IPPSPIPPSPIPPS -> (valid) -> .....PPPUPPPP. 7 pending @@ -897,24 +888,11 @@ START_TEST(all_seis_arrive_late_two_gops_scrapped) // ISPIS -> (valid) -> ...P. 1 pending // 22 pending signed_video_accumulated_validation_t final_validation = { - SV_AUTH_RESULT_NOT_OK, false, 22, 19, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; - struct validation_stats expected = {.valid_gops = 1, - .invalid_gops = 4, - .missed_nalus = 4, - .pending_nalus = 11, - .has_signature = 1, + SV_AUTH_RESULT_OK, false, 22, 19, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; + const struct validation_stats expected = {.valid_gops = 4, + .pending_nalus = 22, + .has_signature = 2, .final_validation = &final_validation}; - if (settings[_i].auth_level == SV_AUTHENTICITY_LEVEL_FRAME) { - // IPPS -> (signature) -> PMMPPU 3 pending, 2 missing - // IPPSPIPPS -> (invalid) -> NMMNNUNPPP. 3 pending, 2 missing - // IPPSPIPPS -> (invalid) -> NNN.NPPP. 3 pending - // IPPSS -> (invalid) -> NMNN.. 0 pending, 1 missing - // PIS -> (missing) -> MMM.P. 1 pending, 3 missing - // ISPIS -> (valid) -> ...P. 1 pending - // 11 pending - expected.valid_gops_with_missing_info = 1; - expected.invalid_gops = 3; - } validate_nalu_list(NULL, list, expected, true); test_stream_free(list); @@ -1120,33 +1098,19 @@ START_TEST(remove_two_gop_in_start_of_stream) test_stream_check_types(list, "ISPPSPISPPPPISPPISP"); signed_video_accumulated_validation_t final_validation = { - SV_AUTH_RESULT_NOT_OK, false, 19, 16, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; + SV_AUTH_RESULT_OK, false, 19, 16, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; // ISPPSPISPPPPISPPISP // - // IS PU -> (signature) - // ISPPS NUNN. -> (invalid) - // PIS MMMNP. -> (invalid) - // ISPPPPIS ......P. -> (valid) - // ISPPIS ....P. -> (valid) - // TODO: There is a flaw in this scenario. It would be best to expect no invalid GOPs in the - // stream, as the first validation should reset the validation status to ensure consistency. - struct validation_stats expected = {.valid_gops = 2, - .invalid_gops = 2, - .pending_nalus = 4, - .missed_nalus = 2, - .has_signature = 1, + // IS PU -> (signature) 1 pending + // ISPPS PUPPU -> (signature) 3 pending + // ISPPSPIS .U..U.P. -> (valid) 1 pending + // ISPPPPIS ......P. -> (valid) 1 pending + // ISPPIS ....P. -> (valid) 1 pending + // 7 pending + const struct validation_stats expected = {.valid_gops = 3, + .pending_nalus = 7, + .has_signature = 2, .final_validation = &final_validation}; - // ISPPSPISPPPPISPPISP - // IS PMU -> (signature) - // ISPPS NMUNN. -> (invalid) - // PIS MMM.P. -> (valid with missing info) - // ISPPPPIS ......P. -> (valid) - // ISPPIS ....P. -> (valid) - if (settings[_i].auth_level == SV_AUTHENTICITY_LEVEL_FRAME) { - expected.valid_gops_with_missing_info = 1; - expected.invalid_gops = 1; - expected.final_validation->authenticity = SV_AUTH_RESULT_NOT_OK; - } validate_nalu_list(NULL, list, expected, true); test_stream_free(list); @@ -1540,21 +1504,13 @@ END_TEST START_TEST(file_export_with_two_useless_seis) { test_stream_t *list = generate_delayed_sei_list(settings[_i], true); + // Remove the first three GOPs. // IPPPPIPPPIPPSP IPPSPIPPSSPISPISP test_stream_t *scrapped = test_stream_pop(list, 14); test_stream_free(scrapped); // IPPSPIPPSSPISPISP // - // IPPS PPPU -> (signature) -> 3 pending - // IPPSPIPPS NNNUNPPP. -> (invalid) -> 3 pending - // IPPSS NMNN.. -> (invalid) -> 0 pending - // PIS MMMNP. -> (invalid) -> 1 pending - // ISPIS ...P. -> (valid) -> 1 pending - // 8 pending - // - // IPPSPIPPSSPISPISP (future) - // // IPPS PPPU -> (signature) -> 3 pending // IPPSPIPPS PPPUPPPPU -> (signature) -> 7 pending // IPPSPIPPSS ...U.PPPU. -> (valid) -> 3 pending @@ -1562,23 +1518,11 @@ START_TEST(file_export_with_two_useless_seis) // ISPIS ...P. -> (valid) -> 1 pending // 15 pending signed_video_accumulated_validation_t final_validation = { - SV_AUTH_RESULT_NOT_OK, false, 17, 14, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; - struct validation_stats expected = {.valid_gops = 1, - .invalid_gops = 3, - .missed_nalus = 4, - .pending_nalus = 8, - .has_signature = 1, + SV_AUTH_RESULT_OK, false, 17, 14, 3, SV_PUBKEY_VALIDATION_NOT_FEASIBLE, true, 0, 0}; + const struct validation_stats expected = {.valid_gops = 3, + .pending_nalus = 15, + .has_signature = 2, .final_validation = &final_validation}; - if (settings[_i].auth_level == SV_AUTHENTICITY_LEVEL_FRAME) { - // IPPS PMPPU -> (signature) -> 3 pending, 1 missing - // IPPSPIPPS NMNNUNPPP. -> (invalid) -> 3 pending - // IPPSS NMNN.. -> (invalid) -> 0 pending, 1 missing - // PIS MMM.P. -> (missing) -> 1 pending, 3 missing - // ISPIS ...P. -> (valid) -> 1 pending - // 8 pending - expected.valid_gops_with_missing_info = 1; - expected.invalid_gops = 2; - } validate_nalu_list(NULL, list, expected, true);