Skip to content

Commit

Permalink
Merge pull request #1348 from CastagnaIT/fix_mem_allocation_nexus
Browse files Browse the repository at this point in the history 
[backport] Removed extra memory reserved / fix memory heap corruption
  • Loading branch information
@glennguy
glennguy authored Aug 13, 2023
2 parents f52cfa1 + 234b079 commit bae2955
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 27 deletions.
7 changes: 2 additions & 5 deletions src/samplereader/FragmentedSampleReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,7 @@ AP4_Result CFragmentedSampleReader::ReadSample()

if (m_decrypter)
{
// Make sure that the decrypter is NOT allocating memory!
// If decrypter and addon are compiled with different DEBUG / RELEASE
// options freeing HEAP memory will fail.
m_sampleData.Reserve(m_encrypted.GetDataSize() + 4096);
m_sampleData.Reserve(m_encrypted.GetDataSize());
if (AP4_FAILED(result =
m_decrypter->DecryptSampleData(m_poolId, m_encrypted, m_sampleData, NULL)))
{
Expand All @@ -174,7 +171,7 @@ AP4_Result CFragmentedSampleReader::ReadSample()
}
else if (useDecryptingDecoder)
{
m_sampleData.Reserve(m_encrypted.GetDataSize() + 1024);
m_sampleData.Reserve(m_encrypted.GetDataSize());
m_singleSampleDecryptor->DecryptSampleData(m_poolId, m_encrypted, m_sampleData, nullptr, 0,
nullptr, nullptr);
}
Expand Down
17 changes: 6 additions & 11 deletions wvdecrypter/wvdecrypter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1162,11 +1162,8 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,

if (fragInfo.nal_length_size_ && (!iv || bytes_of_cleartext_data[0] > 0))
{
//Note that we assume that there is enough data in data_out to hold everything without reallocating.

//check NAL / subsample
const AP4_Byte *packet_in(data_in.GetData()), *packet_in_e(data_in.GetData() + data_in.GetDataSize());
AP4_Byte *packet_out(data_out.UseData() + data_out.GetDataSize());
AP4_UI16 *clrb_out(iv ? reinterpret_cast<AP4_UI16*>(data_out.UseData() + sizeof(subsample_count)):nullptr);
unsigned int nalunitcount(0), nalunitsum(0), configSize(0);

Expand All @@ -1178,19 +1175,18 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,
//look if we have to inject sps / pps
if (fragInfo.annexb_sps_pps_.GetDataSize() && (*packet_in & 0x1F) != 9 /*AVC_NAL_AUD*/)
{
memcpy(packet_out, fragInfo.annexb_sps_pps_.GetData(), fragInfo.annexb_sps_pps_.GetDataSize());
packet_out += fragInfo.annexb_sps_pps_.GetDataSize();
data_out.AppendData(fragInfo.annexb_sps_pps_.GetData(),
fragInfo.annexb_sps_pps_.GetDataSize());
if(clrb_out) *clrb_out += fragInfo.annexb_sps_pps_.GetDataSize();
configSize = fragInfo.annexb_sps_pps_.GetDataSize();
fragInfo.annexb_sps_pps_.SetDataSize(0);
}

//Anex-B Start pos
packet_out[0] = packet_out[1] = packet_out[2] = 0; packet_out[3] = 1;
packet_out += 4;
memcpy(packet_out, packet_in, nalsize);
// Annex-B Start pos
static AP4_Byte annexbStartCode[4] = {0x00, 0x00, 0x00, 0x01};
data_out.AppendData(annexbStartCode, 4);
data_out.AppendData(packet_in, nalsize);
packet_in += nalsize;
packet_out += nalsize;
if (clrb_out) *clrb_out += (4 - fragInfo.nal_length_size_);
++nalunitcount;

Expand Down Expand Up @@ -1230,7 +1226,6 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,
static_cast<unsigned int>(packet_in_e - packet_in), subsample_count);
return AP4_ERROR_NOT_SUPPORTED;
}
data_out.SetDataSize(data_out.GetDataSize() + data_in.GetDataSize() + configSize + (4 - fragInfo.nal_length_size_) * nalunitcount);
}
else
data_out.AppendData(data_in.GetData(), data_in.GetDataSize());
Expand Down
17 changes: 6 additions & 11 deletions wvdecrypter/wvdecrypter_android.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1121,11 +1121,8 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,

if (fragInfo.nal_length_size_ && (!iv || bytes_of_cleartext_data[0] > 0))
{
//Note that we assume that there is enough data in data_out to hold everything without reallocating.

//check NAL / subsample
const AP4_Byte *packet_in(data_in.GetData()), *packet_in_e(data_in.GetData() + data_in.GetDataSize());
AP4_Byte *packet_out(data_out.UseData() + data_out.GetDataSize());
AP4_UI16 *clrb_out(iv ? reinterpret_cast<AP4_UI16*>(data_out.UseData() + sizeof(subsample_count)) : nullptr);
unsigned int nalunitcount(0), nalunitsum(0), configSize(0);

Expand All @@ -1137,19 +1134,18 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,
//look if we have to inject sps / pps
if (fragInfo.annexb_sps_pps_.GetDataSize() && (*packet_in & 0x1F) != 9 /*AVC_NAL_AUD*/)
{
memcpy(packet_out, fragInfo.annexb_sps_pps_.GetData(), fragInfo.annexb_sps_pps_.GetDataSize());
packet_out += fragInfo.annexb_sps_pps_.GetDataSize();
data_out.AppendData(fragInfo.annexb_sps_pps_.GetData(),
fragInfo.annexb_sps_pps_.GetDataSize());
if (clrb_out) *clrb_out += fragInfo.annexb_sps_pps_.GetDataSize();
configSize = fragInfo.annexb_sps_pps_.GetDataSize();
fragInfo.annexb_sps_pps_.SetDataSize(0);
}

//Anex-B Start pos
packet_out[0] = packet_out[1] = packet_out[2] = 0; packet_out[3] = 1;
packet_out += 4;
memcpy(packet_out, packet_in, nalsize);
// Annex-B Start pos
static AP4_Byte annexbStartCode[4] = {0x00, 0x00, 0x00, 0x01};
data_out.AppendData(annexbStartCode, 4);
data_out.AppendData(packet_in, nalsize);
packet_in += nalsize;
packet_out += nalsize;
if (clrb_out) *clrb_out += (4 - fragInfo.nal_length_size_);
++nalunitcount;

Expand Down Expand Up @@ -1188,7 +1184,6 @@ AP4_Result WV_CencSingleSampleDecrypter::DecryptSampleData(AP4_UI32 pool_id,
fragInfo.nal_length_size_, (int)(packet_in_e - packet_in), subsample_count);
return AP4_ERROR_NOT_SUPPORTED;
}
data_out.SetDataSize(data_out.GetDataSize() + data_in.GetDataSize() + configSize + (4 - fragInfo.nal_length_size_) * nalunitcount);
}
else
{
Expand Down

0 comments on commit bae2955

Please sign in to comment.