packetizer: h264: fix get_profile_level

modules/codec/avcodec/directx_va.c

@@ -490,7 +490,7 @@ static bool profile_supported(const directx_va_mode_t *mode, const es_format_t *
         int profile = fmt->i_profile;
         if (mode->codec == AV_CODEC_ID_H264)
         {
-            size_t h264_profile;
+            uint8_t h264_profile;
             if ( h264_get_profile_level(fmt, &h264_profile, NULL, NULL) )
                 profile = h264_profile;
         }

modules/codec/omxil/mediacodec.c

@@ -517,10 +517,10 @@ static int StartMediaCodec(decoder_t *p_dec)
         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
          && !p_sys->u.video.i_h264_profile)
         {
-            h264_get_profile_level(&p_dec->fmt_in,
-                                   &p_sys->u.video.i_h264_profile, NULL, NULL);
-            if (p_sys->u.video.i_h264_profile)
+            uint8_t i_profile;
+            if(h264_get_profile_level(&p_dec->fmt_in, &i_profile, NULL, NULL))
             {
+                p_sys->u.video.i_h264_profile = i_profile;
                 if (p_sys->api->configure(p_sys->api,
                                           p_sys->u.video.i_h264_profile, 0, 0) != 0 )
                     return VLC_EGENERIC;
@@ -668,7 +668,12 @@ static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
     api->psz_mime = mime;
 
     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
-        h264_get_profile_level(&p_dec->fmt_in, &i_h264_profile, NULL, NULL);
+    {
+        uint8_t i_profile;
+        if( h264_get_profile_level(&p_dec->fmt_in, &i_profile, NULL, NULL) )
+            i_h264_profile = i_profile;
+    }
+
     if (pf_init(api) != 0)
     {
         free(api);

modules/codec/omxil/omxil.c

@@ -174,7 +174,7 @@ static OMX_ERRORTYPE ImplementationSpecificWorkarounds(decoder_t *p_dec,
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
     OMX_PARAM_PORTDEFINITIONTYPE *def = &p_port->definition;
-    size_t i_profile = 0xFFFF, i_level = 0xFFFF;
+    uint8_t i_profile = 0xFF, i_level = 0xFF;
 
     /* Try to find out the profile of the video */
     if(p_fmt->i_cat == VIDEO_ES && def->eDir == OMX_DirInput &&
@@ -187,7 +187,7 @@ static OMX_ERRORTYPE ImplementationSpecificWorkarounds(decoder_t *p_dec,
            p_fmt->i_codec == VLC_CODEC_H264 &&
            (i_profile != PROFILE_H264_BASELINE || i_level > 30))
         {
-            msg_Dbg(p_dec, "h264 profile/level not supported (0x%x, 0x%x)",
+            msg_Dbg(p_dec, "h264 profile/level not supported (0x" PRIx8 ", 0x" PRIx8 ")",
                     i_profile, i_level);
             return OMX_ErrorNotImplemented;
         }

modules/codec/videotoolbox.m

@@ -129,7 +129,7 @@ struct decoder_sys_t
 static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    size_t i_profile = 0xFFFF, i_level = 0xFFFF;
+    uint8_t i_profile = 0xFF, i_level = 0xFF;
     bool b_ret = false;
     CMVideoCodecType codec;
 
@@ -144,7 +144,7 @@ static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
                 return kCMVideoCodecType_H264;
             }
 
-            msg_Dbg(p_dec, "trying to decode MPEG-4 Part 10: profile %zu, level %zu", i_profile, i_level);
+            msg_Dbg(p_dec, "trying to decode MPEG-4 Part 10: profile %" PRIx8 ", level " PRIx8, i_profile, i_level);
 
             switch (i_profile) {
                 case PROFILE_H264_BASELINE:
@@ -160,7 +160,7 @@ static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
 
                 default:
                 {
-                    msg_Dbg(p_dec, "unsupported H264 profile %zu", i_profile);
+                    msg_Dbg(p_dec, "unsupported H264 profile %" PRIx8, i_profile);
                     return -1;
                 }
             }
@@ -169,7 +169,7 @@ static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
             /* a level higher than 5.2 was not tested, so don't dare to
              * try to decode it*/
             if (i_level > 52) {
-                msg_Dbg(p_dec, "unsupported H264 level %zu", i_level);
+                msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
                 return -1;
             }
 #else
@@ -177,7 +177,7 @@ static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
             if (i_level > 42) {
                 /* on Twister, we can do up to 5.2 */
                 if (!deviceSupportsAdvancedLevels() || i_level > 52) {
-                    msg_Dbg(p_dec, "unsupported H264 level %zu", i_level);
+                    msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
                     return -1;
                 }
             }

modules/packetizer/h264_nal.c

@@ -628,32 +628,38 @@ bool h264_get_chroma_luma( const h264_sequence_parameter_set_t *p_sps, uint8_t *
     return true;
 }
 
-bool h264_get_profile_level(const es_format_t *p_fmt, size_t *p_profile,
-                            size_t *p_level, uint8_t *pi_nal_length_size)
+bool h264_get_profile_level(const es_format_t *p_fmt, uint8_t *pi_profile,
+                            uint8_t *pi_level, uint8_t *pi_nal_length_size)
 {
     uint8_t *p = (uint8_t*)p_fmt->p_extra;
-    if(!p || !p_fmt->p_extra) return false;
+    if(p_fmt->i_extra < 8)
+        return false;
 
     /* Check the profile / level */
-    if (p_fmt->i_original_fourcc == VLC_FOURCC('a','v','c','1') && p[0] == 1)
+    if (p[0] == 1 && p_fmt->i_extra >= 12)
     {
-        if (p_fmt->i_extra < 12) return false;
-        if (pi_nal_length_size) *pi_nal_length_size = 1 + (p[4]&0x03);
-        if (!(p[5]&0x1f)) return false;
+        if (pi_nal_length_size)
+            *pi_nal_length_size = 1 + (p[4]&0x03);
         p += 8;
     }
-    else
+    else if(!p[0] && !p[1]) /* FIXME: WTH is setting AnnexB data here ? */
     {
-        if (p_fmt->i_extra < 8) return false;
-        if (!p[0] && !p[1] && !p[2] && p[3] == 1) p += 4;
-        else if (!p[0] && !p[1] && p[2] == 1) p += 3;
-        else return false;
+        if (!p[2] && p[3] == 1)
+            p += 4;
+        else if (p[2] == 1)
+            p += 3;
+        else
+            return false;
     }
+    else return false;
 
     if ( ((*p++)&0x1f) != 7) return false;
 
-    /* Get profile/level out of first SPS */
-    if (p_profile) *p_profile = p[0];
-    if (p_level) *p_level = p[2];
+    if (pi_profile)
+        *pi_profile = p[0];
+
+    if (pi_level)
+        *pi_level = p[2];
+
     return true;
 }

modules/packetizer/h264_nal.h

@@ -180,8 +180,8 @@ bool h264_get_picture_size( const h264_sequence_parameter_set_t *, unsigned *p_w
 bool h264_get_chroma_luma( const h264_sequence_parameter_set_t *, uint8_t *pi_chroma_format,
                            uint8_t *pi_depth_luma, uint8_t *pi_depth_chroma );
 
-/* Get level and Profile */
-bool h264_get_profile_level(const es_format_t *p_fmt, size_t *p_profile,
-                            size_t *p_level, uint8_t *p_nal_length_size);
+/* Get level and Profile from DecoderConfigurationRecord */
+bool h264_get_profile_level(const es_format_t *p_fmt, uint8_t *pi_profile,
+                            uint8_t *pi_level, uint8_t *p_nal_length_size);
 
 #endif /* H264_NAL_H */