vlc_es: change i_extra into size_t

This MR changes es_format_t::i_extra from a signed int to size_t.

This variable is used to describe the size of the buffer pointed by
p_extra, and is never strictly negative.

Users of this member are also updated in this commit at once. I'm not
sure it's easy to split, except if callers are casted to size_t before
usage in a preparatory change, and then the cast is removed in a cleanup
change.

Some users were already using size_t though, like videotoolbox decoder
or hxxx_helpers.c. The main blocker for splitting is that most clients
not doing that are actually using pointers to i_extra instead of using a
copy, which is much more verbose to fix locally.

Almost No unneeded cast is added to fix signedness mismatch in this
patch. Most notably, only mpeg/ts_psi.c gets a new cast when using
__MIN() with fmt->i_extra, since the dvbpsi_descriptor_t::i_length
parameter is signed.

avformat/demux.c gets a temporary cast which is no-op with a version
libavutil >= 57 but which would trigger the warning before that. It may
be removed when libavutil < 57 is dropped.

include/vlc_es.h

@@ -666,7 +666,7 @@ struct es_format_t
     int      i_level;         /**< codec specific information: indicates maximum restrictions on the stream (resolution, bitrate, codec features ...) */
 
     bool     b_packetized;  /**< whether the data is packetized (ie. not truncated) */
-    int     i_extra;        /**< length in bytes of extra data pointer */
+    size_t   i_extra;       /**< length in bytes of extra data pointer */
     void    *p_extra;       /**< extra data needed by some decoders or muxers */
 
 };

modules/access/live555.cpp

@@ -2483,7 +2483,7 @@ parseOpusConfigStr(char const* configStr, size_t& configSize, audio_format_t& fm
     fmt.i_rate = header.input_sample_rate;
     /* Build extradata with the proper params */
     opus_prepare_header( header.channels, header.input_sample_rate, &header );
-    int i_extra;
+    size_t i_extra;
     uint8_t *p_extra;
     if( opus_write_header( &p_extra, &i_extra, &header, NULL ) )
     {
@@ -2491,7 +2491,7 @@ parseOpusConfigStr(char const* configStr, size_t& configSize, audio_format_t& fm
         i_extra = 0;
     }
     opus_header_clean(&header);
-    configSize = (size_t)i_extra;
+    configSize = i_extra;
     return p_extra;
 }
 

modules/access/rtp/xiph.c

@@ -120,7 +120,7 @@ static ssize_t xiph_header (void **pextra, const uint8_t *buf, size_t len)
         buf + idlen + cmtlen
     };
     void *extra;
-    int  extra_size;
+    size_t extra_size;
     if (xiph_PackHeaders (&extra_size, &extra, sizes, payloads, 3))
         return -1;
     *pextra = extra;

modules/codec/flac.c

@@ -523,7 +523,7 @@ static void ProcessHeader( decoder_t *p_dec )
 
     /* Decode STREAMINFO */
     msg_Dbg( p_dec, "decode STREAMINFO" );
-    int i_extra = p_dec->fmt_in->i_extra;
+    size_t i_extra = p_dec->fmt_in->i_extra;
 
     static const char header[4] = { 'f', 'L', 'a', 'C' };
 

modules/codec/opus.c

@@ -353,7 +353,7 @@ static int ProcessHeaders( decoder_t *p_dec )
     const void *pp_data[XIPH_MAX_HEADER_COUNT];
     unsigned i_count;
 
-    int i_extra = p_dec->fmt_in->i_extra;
+    size_t i_extra = p_dec->fmt_in->i_extra;
     const uint8_t *p_extra = p_dec->fmt_in->p_extra;
     uint8_t *p_alloc = NULL;
 

modules/codec/opus_header.c

@@ -389,7 +389,7 @@ static int opus_header_to_packet(const OpusHeader *h, unsigned char *packet, int
     return p.pos;
 }
 
-int opus_write_header(uint8_t **p_extra, int *i_extra, OpusHeader *header, const char *vendor)
+int opus_write_header(uint8_t **p_extra, size_t *i_extra, OpusHeader *header, const char *vendor)
 {
     unsigned char header_data[100];
     const int packet_size = opus_header_to_packet(header, header_data,

modules/codec/opus_header.h

@@ -50,6 +50,6 @@ void opus_header_init(OpusHeader *);
 void opus_header_clean(OpusHeader *);
 int opus_header_parse(const unsigned char *header, int len, OpusHeader *h);
 void opus_prepare_header(unsigned channels, unsigned rate, OpusHeader *header);
-int opus_write_header(uint8_t **p_extra, int *i_extra, OpusHeader *header, const char *vendor);
+int opus_write_header(uint8_t **p_extra, size_t *i_extra, OpusHeader *header, const char *vendor);
 
 #endif

modules/codec/stl.c

@@ -441,7 +441,7 @@ static int ParseGSI(decoder_t *dec, decoder_sys_t *p_sys)
     }
 
     if (GSI_BLOCK_SIZE != dec->fmt_in->i_extra) {
-        msg_Err(dec, "EBU header is not in expected size (%d)\n", dec->fmt_in->i_extra);
+        msg_Err(dec, "EBU header is not in expected size (%zu)\n", dec->fmt_in->i_extra);
         return VLC_EGENERIC;
     }
 

modules/codec/substx3g.c

@@ -569,11 +569,13 @@ static int OpenDecoder( vlc_object_t *p_this )
  * Encoder entry/exit
  *****************************************************************************/
 #ifdef ENABLE_SOUT
-static void FillExtradataTx3g( void **pp_extra, int *pi_extra )
+static void FillExtradataTx3g(void **pp_extra, size_t *pi_extra)
 {
     size_t i_extra = 32 + 37;
     uint8_t *p_extra = calloc( 1, i_extra );
-    if( p_extra )
+    if (p_extra == NULL)
+        return;
+
     {
         p_extra[4] = 0x01;/* 1  center, horizontal */
         p_extra[5] = 0xFF;/* -1 bottom, vertical */

modules/demux/avformat/demux.c

@@ -850,7 +850,7 @@ static int Demux( demux_t *p_demux )
     if( side_data_size_query > 0 ) {
         // ignore new extradata which is the same as previous version
         size_t side_data_size = (size_t)side_data_size_query;
-        if( side_data_size != (size_t)p_track->es_format.i_extra ||
+        if (side_data_size != p_track->es_format.i_extra ||
             memcmp( side_data, p_track->es_format.p_extra, side_data_size ) != 0 )
         {
             msg_Warn( p_demux, "New extra data found, seek may not work as expected" );

modules/demux/mp4/avci.h

@@ -179,10 +179,12 @@ static uint8_t * AVCi_create_avcC( uint16_t i_res, bool b_interlaced, int *pi_av
 }
 #endif
 
-static uint8_t * AVCi_create_AnnexB( uint16_t i_res, bool b_interlaced, int *pi_avcC )
+static uint8_t * AVCi_create_AnnexB(uint16_t i_res, bool b_interlaced, size_t *pi_avcC)
 {
     const uint8_t *p_pps, *p_sps;
     uint8_t i_sps, i_pps;
+
+    // TODO: early return
     uint8_t *p_data = NULL;
     const uint8_t rgi_startcode[] = {0,0,0,1};
     if( AVCi_lookup( i_res, b_interlaced,

modules/demux/mpeg/ts_psi.c

@@ -1170,7 +1170,7 @@ static void OpusSetup(demux_t *demux, uint8_t *p, size_t len, es_format_t *p_fmt
 
     if (h.channels) {
         uint8_t *p_extra = NULL;
-        int i_extra = 0;
+        size_t i_extra = 0;
         opus_write_header(&p_extra, &i_extra, &h, NULL /* FIXME */);
         if (p_extra) {
             es_format_Change(p_fmt, AUDIO_ES, VLC_CODEC_OPUS);
@@ -1466,7 +1466,7 @@ static void PMTSetupEs0xA0( demux_t *p_demux, ts_es_t *p_es,
         p_fmt->p_extra = malloc( p_fmt->i_extra );
         if( p_fmt->p_extra )
             memcpy( p_fmt->p_extra, &p_dr->p_data[10],
-                    __MIN( p_fmt->i_extra, p_dr->i_length - 10 ) );
+                    __MIN( p_fmt->i_extra, (size_t)(p_dr->i_length - 10) ) );
         else
             p_fmt->i_extra = 0;
     }

modules/demux/ogg.c

@@ -154,7 +154,7 @@ static bool Ogg_LogicalStreamResetEsFormat( demux_t *p_demux, logical_stream_t *
 static void Ogg_ResetStream( logical_stream_t *p_stream );
 
 /* */
-static void Ogg_ExtractMeta( demux_t *p_demux, es_format_t *p_fmt, const uint8_t *p_headers, int i_headers );
+static void Ogg_ExtractMeta( demux_t *p_demux, es_format_t *p_fmt, const uint8_t *p_headers, size_t i_headers );
 
 /* Logical bitstream headers */
 static bool Ogg_ReadDaalaHeader( logical_stream_t *, ogg_packet * );
@@ -2564,7 +2564,7 @@ static bool Ogg_LogicalStreamResetEsFormat( demux_t *p_demux, logical_stream_t *
 }
 
 static void Ogg_ExtractComments( demux_t *p_demux, es_format_t *p_fmt,
-                                 const void *p_headers, unsigned i_headers )
+                                 const void *p_headers, size_t i_headers )
 {
     demux_sys_t *p_ogg = p_demux->p_sys;
     int i_cover_score = 0;
@@ -2621,7 +2621,7 @@ static inline uint32_t GetDW24BE( const uint8_t *p )
 }
 
 static void Ogg_ExtractFlacComments( demux_t *p_demux, es_format_t *p_fmt,
-                                     const uint8_t *p_headers, unsigned i_headers )
+                                     const uint8_t *p_headers, size_t i_headers )
 {
     /* Skip Streaminfo 42 bytes / 1st page */
     if(i_headers <= 46)
@@ -2636,7 +2636,7 @@ static void Ogg_ExtractFlacComments( demux_t *p_demux, es_format_t *p_fmt,
 }
 
 static void Ogg_ExtractXiphMeta( demux_t *p_demux, es_format_t *p_fmt,
-                                 const void *p_headers, unsigned i_headers, unsigned i_skip )
+                                 const void *p_headers, size_t i_headers, unsigned i_skip )
 {
     unsigned pi_size[XIPH_MAX_HEADER_COUNT];
     const void *pp_data[XIPH_MAX_HEADER_COUNT];
@@ -2653,7 +2653,7 @@ static void Ogg_ExtractXiphMeta( demux_t *p_demux, es_format_t *p_fmt,
     }
 }
 
-static void Ogg_ExtractMeta( demux_t *p_demux, es_format_t *p_fmt, const uint8_t *p_headers, int i_headers )
+static void Ogg_ExtractMeta( demux_t *p_demux, es_format_t *p_fmt, const uint8_t *p_headers, size_t i_headers )
 {
     demux_sys_t *p_ogg = p_demux->p_sys;
 

modules/demux/ogg.h

@@ -73,7 +73,7 @@ typedef struct logical_stream_s
     int              i_packets_backup;
     int32_t          i_extra_headers_packets;
     void             *p_headers;
-    int              i_headers;
+    size_t           i_headers;
     ogg_int64_t      i_granulepos_offset;/* first granule offset */
 
     /* program clock reference (in units of 90kHz) derived from the previous
@@ -88,7 +88,7 @@ typedef struct logical_stream_s
     int i_next_block_flags;
 
     /* Opus has a starting offset in the headers. */
-    int i_pre_skip;
+    size_t i_pre_skip;
     enum
     {
         OGGPAGE_HEADER,

modules/demux/wav.c

@@ -503,7 +503,7 @@ static int ChunkParseFmt( demux_t *p_demux, uint32_t i_size )
 
     msg_Dbg( p_demux, "format: 0x%4.4x, fourcc: %4.4s, channels: %d, "
              "freq: %u Hz, bitrate: %uKo/s, blockalign: %d, bits/samples: %d, "
-             "extra size: %d",
+             "extra size: %zu",
              GetWLE( &p_wf->wFormatTag ), (char *)&p_sys->fmt.i_codec,
              p_sys->fmt.audio.i_channels, p_sys->fmt.audio.i_rate,
              p_sys->fmt.i_bitrate / 8 / 1024, p_sys->fmt.audio.i_blockalign,

modules/demux/xiph.h

@@ -25,7 +25,7 @@
 #define XIPH_MAX_HEADER_COUNT (256)
 
 /* Temp ffmpeg vorbis format */
-static inline bool xiph_IsLavcFormat(const void *extra, unsigned i_extra,
+static inline bool xiph_IsLavcFormat(const void *extra, size_t i_extra,
                                      vlc_fourcc_t i_codec)
 {
     switch(i_codec)
@@ -39,7 +39,7 @@ static inline bool xiph_IsLavcFormat(const void *extra, unsigned i_extra,
     }
 }
 
-static inline unsigned xiph_CountLavcHeaders(const void *p_extra, unsigned i_extra)
+static inline unsigned xiph_CountLavcHeaders(const void *p_extra, size_t i_extra)
 {
     const uint8_t *p = (const uint8_t*) p_extra;
     const uint8_t *p_end = &p[i_extra];
@@ -56,7 +56,7 @@ static inline unsigned xiph_CountLavcHeaders(const void *p_extra, unsigned i_ext
     return 3;
 }
 
-static inline unsigned xiph_CountHeaders(const void *p_extra, unsigned i_extra)
+static inline unsigned xiph_CountHeaders(const void *p_extra, size_t i_extra)
 {
     const uint8_t *p = (const uint8_t*) p_extra;
     if (!i_extra)
@@ -105,7 +105,7 @@ static inline int xiph_SplitLavcHeaders(unsigned packet_size[],
 
 static inline int xiph_SplitHeaders(unsigned packet_size[],
                                     const void *packet[], unsigned *packet_count,
-                                    unsigned i_extra, const void *p_extra)
+                                    size_t i_extra, const void *p_extra)
 {
     const uint8_t *current = (const uint8_t *)p_extra;
     const uint8_t *end = &current[i_extra];
@@ -158,7 +158,7 @@ static inline int xiph_SplitHeaders(unsigned packet_size[],
     return VLC_SUCCESS;
 }
 
-static inline int xiph_PackHeaders(int *extra_size, void **extra,
+static inline int xiph_PackHeaders(size_t *extra_size, void **extra,
                                    unsigned packet_size[],
                                    const void *const packet[],
                                    unsigned packet_count)
@@ -208,7 +208,7 @@ static inline int xiph_PackHeaders(int *extra_size, void **extra,
     return VLC_SUCCESS;
 }
 
-static inline int xiph_AppendHeaders(int *extra_size, void **extra,
+static inline int xiph_AppendHeaders(size_t *extra_size, void **extra,
                                      unsigned size, const void *data)
 {
     unsigned packet_size[XIPH_MAX_HEADER_COUNT];
@@ -236,7 +236,7 @@ static inline int xiph_AppendHeaders(int *extra_size, void **extra,
 
     free(old);
 
-    if (*extra_size <= 0)
+    if (*extra_size == 0)
         return VLC_EGENERIC;
     return VLC_SUCCESS;
 }

modules/hw/nvdec/nvdec.c

@@ -93,7 +93,7 @@ struct nvdec_ctx {
     CUvideoparser               cuparser;
     union {
         struct hxxx_helper      hh;
-        int                     vc1_header_offset;
+        size_t                  vc1_header_offset;
     };
     bool                        b_is_hxxx;
     bool                        b_xps_pushed; ///< (for xvcC) parameter sets pushed (SPS/PPS/VPS)

modules/mux/mpeg/tables.c

@@ -281,8 +281,9 @@ static void GetPMTmpeg4( vlc_object_t *p_object, dvbpsi_pmt_t *p_dvbpmt,
             bits_write( &bits, 8,   0x05 ); /* tag */
             bits_write( &bits, 24, GetDescriptorLength24b(
                         p_stream->fmt->i_extra ) );
-            for (int j = 0; j < p_stream->fmt->i_extra; j++ )
+            for (size_t j = 0; j < p_stream->fmt->i_extra; j++ )
             {
+            // TODO checks
                 bits_write( &bits, 8,
                     ((uint8_t*)p_stream->fmt->p_extra)[j] );
             }

modules/packetizer/hevc.c

@@ -568,7 +568,7 @@ static void SetsToAnnexB(decoder_sys_t *p_sys,
                          const hevc_picture_parameter_set_t *p_pps,
                          const hevc_sequence_parameter_set_t *p_sps,
                          const hevc_video_parameter_set_t *p_vps,
-                         uint8_t **pp_out, int *pi_out)
+                         uint8_t **pp_out, size_t *pi_out)
 {
     uint8_t *p_data = NULL;
     size_t i_data = 0;

modules/packetizer/mpeg4video.c

@@ -157,7 +157,7 @@ static int Open( vlc_object_t *p_this )
     if( p_dec->fmt_out.i_extra )
     {
         /* We have a vol */
-        msg_Dbg( p_dec, "opening with vol size: %d", p_dec->fmt_out.i_extra );
+        msg_Dbg( p_dec, "opening with vol size: %zu", p_dec->fmt_out.i_extra );
         ParseVOL( p_dec, &p_dec->fmt_out,
                   p_dec->fmt_out.p_extra, p_dec->fmt_out.i_extra );
     }

modules/packetizer/vc1.c

@@ -330,7 +330,7 @@ static void BuildExtraData( decoder_t *p_dec )
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
     es_format_t *p_es = &p_dec->fmt_out;
-    int i_extra;
+    size_t i_extra;
     if( !p_sys->b_sequence_header || !p_sys->b_entry_point )
         return;