diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c index 45805eec4c..c9566db693 100644 --- a/modules/codec/omxil/omxil.c +++ b/modules/codec/omxil/omxil.c @@ -1040,11 +1040,8 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode ) h264_isavcC(p_dec->fmt_in->p_extra, p_dec->fmt_in->i_extra) ) { size_t i_filled_len = 0; - uint8_t *p_buf = h264_avcC_to_AnnexB_NAL( - p_dec->fmt_in->p_extra, p_dec->fmt_in->i_extra, - &i_filled_len, NULL ); - - if( p_buf == NULL ) + if( !h264_avcC_to_AnnexB_NAL( p_dec->fmt_in->p_extra, p_dec->fmt_in->i_extra, + &p_buf, &i_filled_len, NULL ) { msg_Dbg(p_dec, "h264_avcC_to_AnnexB_NAL() failed"); goto error; diff --git a/modules/packetizer/h264.c b/modules/packetizer/h264.c index 29a6a3a842..5a1a1707d6 100644 --- a/modules/packetizer/h264.c +++ b/modules/packetizer/h264.c @@ -368,21 +368,20 @@ static int Open( vlc_object_t *p_this ) * The fmt_out.p_extra should contain all the SPS and PPS with 4 byte startcodes */ if( h264_isavcC( p_dec->fmt_in->p_extra, p_dec->fmt_in->i_extra ) ) { - free( p_dec->fmt_out.p_extra ); - size_t i_size; - p_dec->fmt_out.p_extra = h264_avcC_to_AnnexB_NAL( p_dec->fmt_in->p_extra, - p_dec->fmt_in->i_extra, - &i_size, - &p_sys->i_avcC_length_size ); - p_dec->fmt_out.i_extra = i_size; - p_sys->b_recovered = !!p_dec->fmt_out.i_extra; - - if(!p_dec->fmt_out.p_extra) + size_t i_newextra_size; uint8_t *p_newextra; + if( !h264_avcC_to_AnnexB_NAL( p_dec->fmt_in->p_extra, + p_dec->fmt_in->i_extra, + &p_newextra, &i_newextra_size, + &p_sys->i_avcC_length_size ) ) { msg_Err( p_dec, "Invalid AVC extradata"); Close( p_this ); return VLC_EGENERIC; } + free( p_dec->fmt_out.p_extra ); + p_dec->fmt_out.p_extra = p_newextra; + p_dec->fmt_out.i_extra = i_newextra_size; + p_sys->b_recovered = !!p_dec->fmt_out.i_extra; } else { diff --git a/modules/packetizer/h264_nal.c b/modules/packetizer/h264_nal.c index f5d1975c31..ca45964665 100644 --- a/modules/packetizer/h264_nal.c +++ b/modules/packetizer/h264_nal.c @@ -107,12 +107,12 @@ bool h264_isavcC( const uint8_t *p_buf, size_t i_buf ) ); } -static size_t get_avcC_to_AnnexB_NAL_size( const uint8_t *p_buf, size_t i_buf ) +static bool get_avcC_to_AnnexB_NAL_size( const uint8_t *p_buf, size_t i_buf, size_t *pi_res ) { size_t i_total = 0; if( i_buf < H264_MIN_AVCC_SIZE ) - return 0; + return false; p_buf += 5; i_buf -= 5; @@ -126,63 +126,70 @@ static size_t get_avcC_to_AnnexB_NAL_size( const uint8_t *p_buf, size_t i_buf ) for ( unsigned int i = 0; i < i_loop_end; i++ ) { if( i_buf < 2 ) - return 0; + return false; uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1]; if(i_nal_size > i_buf - 2) - return 0; + return false; i_total += i_nal_size + 4; p_buf += i_nal_size + 2; i_buf -= i_nal_size + 2; } if( j == 0 && i_buf < 1 ) - return 0; + return false; } - return i_total; + *pi_res = i_total; + return true; } -uint8_t *h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf, - size_t *pi_result, uint8_t *pi_nal_length_size ) +bool h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf, + uint8_t **pp_result, size_t *pi_result, + uint8_t *pi_nal_length_size ) { - *pi_result = get_avcC_to_AnnexB_NAL_size( p_buf, i_buf ); /* Does check min size */ - if( *pi_result == 0 ) - return NULL; + size_t i_result; + if( !get_avcC_to_AnnexB_NAL_size( p_buf, i_buf, &i_result ) ) /* Does check min size */ + return false; /* Read infos in first 6 bytes */ - if ( pi_nal_length_size ) - *pi_nal_length_size = (p_buf[4] & 0x03) + 1; + const uint8_t i_nal_length_size = (p_buf[4] & 0x03) + 1; - uint8_t *p_ret; - uint8_t *p_out_buf = p_ret = malloc( *pi_result ); - if( !p_out_buf ) + uint8_t *p_out_buf = NULL; + if( i_result > 0 ) { - *pi_result = 0; - return NULL; - } - - p_buf += 5; + p_out_buf = malloc( i_result ); + if( !p_out_buf ) + return false; - for ( unsigned int j = 0; j < 2; j++ ) - { - const unsigned int i_loop_end = p_buf[0] & (j == 0 ? 0x1f : 0xff); - p_buf++; + p_buf += 5; - for ( unsigned int i = 0; i < i_loop_end; i++) + for ( unsigned int j = 0; j < 2; j++ ) { - uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1]; - p_buf += 2; + const unsigned int i_loop_end = p_buf[0] & (j == 0 ? 0x1f : 0xff); + p_buf++; + + for ( unsigned int i = 0; i < i_loop_end; i++) + { + uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1]; + p_buf += 2; - memcpy( p_out_buf, annexb_startcode4, 4 ); - p_out_buf += 4; + memcpy( p_out_buf, annexb_startcode4, 4 ); + p_out_buf += 4; - memcpy( p_out_buf, p_buf, i_nal_size ); - p_out_buf += i_nal_size; - p_buf += i_nal_size; + memcpy( p_out_buf, p_buf, i_nal_size ); + p_out_buf += i_nal_size; + p_buf += i_nal_size; + } } } - return p_ret; + *pp_result = p_out_buf; + *pi_result = i_result; + + if ( pi_nal_length_size ) + *pi_nal_length_size = i_nal_length_size; + + return true; } void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len, diff --git a/modules/packetizer/h264_nal.h b/modules/packetizer/h264_nal.h index 86e9a26f20..c242dda15e 100644 --- a/modules/packetizer/h264_nal.h +++ b/modules/packetizer/h264_nal.h @@ -206,8 +206,9 @@ block_t *h264_NAL_to_avcC( uint8_t i_nal_length_size, const size_t *p_sps_ext_size, uint8_t i_sps_ext_count); /* Convert AVCDecoderConfigurationRecord SPS/PPS to Annex B format */ -uint8_t * h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf, - size_t *pi_result, uint8_t *pi_nal_length_size ); +bool h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf, + uint8_t **pp_result, size_t *pi_result, + uint8_t *pi_nal_length_size ); bool h264_get_dpb_values( const h264_sequence_parameter_set_t *, uint8_t *pi_max_num_reorder, uint8_t *pi_max_dec_buffering );