ffmpeg chroma: Change GetVlcChroma and GetFfmpegChroma to use a video_format_t, so that we will be able to access and set the rgb mask later on.

This is the first work required to fix #1749

modules/codec/avcodec/chroma.h

@@ -87,27 +87,40 @@ static const struct
     { 0, 0 }
 };
 
-static inline int GetFfmpegChroma( vlc_fourcc_t i_chroma )
+static inline int TestFfmpegChroma( const int i_ffmpeg_id, const vlc_fourcc_t i_vlc_fourcc )
 {
-    int i;
-
-    for( i = 0; chroma_table[i].i_chroma != 0; i++ )
+    for( int i = 0; chroma_table[i].i_chroma != 0; i++ )
     {
-        if( chroma_table[i].i_chroma == i_chroma )
-            return chroma_table[i].i_chroma_id;
+        if( chroma_table[i].i_chroma == i_vlc_fourcc || chroma_table[i].i_chroma_id == i_ffmpeg_id )
+            return VLC_SUCCESS;
     }
-    return -1;
+    return VLC_EGENERIC;
 }
 
-static inline vlc_fourcc_t GetVlcChroma( int i_ffmpeg_chroma )
+/* FIXME special case the RGB formats */
+static inline int GetFfmpegChroma( int *i_ffmpeg_chroma, const video_format_t fmt )
 {
-    int i;
+    for( int i = 0; chroma_table[i].i_chroma != 0; i++ )
+    {
+        if( chroma_table[i].i_chroma == fmt.i_chroma )
+        {
+            *i_ffmpeg_chroma =  chroma_table[i].i_chroma_id;
+            return VLC_SUCCESS;
+        }
+    }
+    return VLC_EGENERIC;
+}
 
+static inline int GetVlcChroma( video_format_t *fmt, const int i_ffmpeg_chroma )
+{
     /* TODO FIXME for rgb format we HAVE to set rgb mask/shift */
-    for( i = 0; chroma_table[i].i_chroma != 0; i++ )
+    for( int i = 0; chroma_table[i].i_chroma != 0; i++ )
     {
         if( chroma_table[i].i_chroma_id == i_ffmpeg_chroma )
-            return chroma_table[i].i_chroma;
+        {
+            fmt->i_chroma = chroma_table[i].i_chroma;
+            return VLC_SUCCESS;
+        }
     }
-    return 0;
+    return VLC_EGENERIC;
 }

modules/codec/avcodec/deinterlace.c

@@ -75,8 +75,9 @@ int OpenDeinterlace( vlc_object_t *p_this )
     filter_sys_t *p_sys;
 
     /* Check if we can handle that formats */
-    if( GetFfmpegChroma( p_filter->fmt_in.video.i_chroma ) < 0 )
+    if( TestFfmpegChroma( -1, p_filter->fmt_in.i_codec  ) != VLC_SUCCESS )
     {
+        msg_Err( p_filter, "Failed to match chroma type" );
         return VLC_EGENERIC;
     }
 
@@ -88,8 +89,12 @@ int OpenDeinterlace( vlc_object_t *p_this )
     }
 
     /* Misc init */
-    p_sys->i_src_ffmpeg_chroma =
-        GetFfmpegChroma( p_filter->fmt_in.video.i_chroma );
+    p_filter->fmt_in.video.i_chroma = p_filter->fmt_in.i_codec;
+    if( GetFfmpegChroma( &p_sys->i_src_ffmpeg_chroma, p_filter->fmt_in.video ) != VLC_SUCCESS )
+    {
+        msg_Err( p_filter, "Failed to match chroma type" );
+        return VLC_EGENERIC;
+    }
     p_filter->pf_video_filter = Deinterlace;
 
     msg_Dbg( p_filter, "deinterlacing" );

modules/codec/avcodec/encoder.c

@@ -212,7 +212,7 @@ int OpenEncoder( vlc_object_t *p_this )
     if( !GetFfmpegCodec( p_enc->fmt_out.i_codec, &i_cat, &i_codec_id,
                              &psz_namecodec ) )
     {
-        if( GetFfmpegChroma( p_enc->fmt_out.i_codec ) < 0 )
+        if( TestFfmpegChroma( -1, p_enc->fmt_out.i_codec ) != VLC_SUCCESS )
         {
             /* handed chroma output */
             return VLC_EGENERIC;
@@ -460,7 +460,9 @@ int OpenEncoder( vlc_object_t *p_this )
         p_sys->p_buffer_out = malloc( p_sys->i_buffer_out );
 
         p_enc->fmt_in.i_codec = VLC_FOURCC('I','4','2','0');
-        p_context->pix_fmt = GetFfmpegChroma( p_enc->fmt_in.i_codec );
+        p_enc->fmt_in.video.i_chroma = p_enc->fmt_in.i_codec;
+        GetFfmpegChroma( &p_context->pix_fmt, p_enc->fmt_in.video );
+
         if( p_codec->pix_fmts )
         {
             const enum PixelFormat *p = p_codec->pix_fmts;
@@ -469,7 +471,8 @@ int OpenEncoder( vlc_object_t *p_this )
                 if( *p == p_context->pix_fmt ) break;
             }
             if( *p == -1 ) p_context->pix_fmt = p_codec->pix_fmts[0];
-            p_enc->fmt_in.i_codec = GetVlcChroma( p_context->pix_fmt );
+            GetVlcChroma( &p_enc->fmt_in.video, p_context->pix_fmt );
+            p_enc->fmt_in.i_codec = p_enc->fmt_in.video.i_chroma;
         }
 
 
@@ -588,8 +591,8 @@ int OpenEncoder( vlc_object_t *p_this )
     if( i_codec_id == CODEC_ID_RAWVIDEO )
     {
         /* XXX: hack: Force same codec (will be handled by transcode) */
-        p_enc->fmt_in.i_codec = p_enc->fmt_out.i_codec;
-        p_context->pix_fmt = GetFfmpegChroma( p_enc->fmt_in.i_codec );
+        p_enc->fmt_in.video.i_chroma = p_enc->fmt_in.i_codec = p_enc->fmt_out.i_codec;
+        GetFfmpegChroma( &p_context->pix_fmt, p_enc->fmt_in.video );
     }
 
     /* Make sure we get extradata filled by the encoder */

modules/codec/avcodec/video.c

@@ -114,18 +114,19 @@ static inline picture_t *ffmpeg_NewPictBuf( decoder_t *p_dec,
 
     p_dec->fmt_out.video.i_width = p_context->width;
     p_dec->fmt_out.video.i_height = p_context->height;
-    p_dec->fmt_out.i_codec = GetVlcChroma( p_context->pix_fmt );
 
     if( !p_context->width || !p_context->height )
     {
         return NULL; /* invalid display size */
     }
 
-    if( !p_dec->fmt_out.i_codec )
+    if( GetVlcChroma( &p_dec->fmt_out.video, p_context->pix_fmt ) != VLC_SUCCESS )
     {
         /* we are doomed */
+        msg_Err( p_dec, "avcodec does not know how to convert this chroma" );
         p_dec->fmt_out.i_codec = VLC_FOURCC('I','4','2','0');
     }
+    p_dec->fmt_out.i_codec = p_dec->fmt_out.video.i_chroma;
 
     /* If an aspect-ratio was specified in the input format then force it */
     if( p_dec->fmt_in.video.i_aspect )
@@ -330,7 +331,13 @@ int InitVideoDec( decoder_t *p_dec, AVCodecContext *p_context,
 
     /* Set output properties */
     p_dec->fmt_out.i_cat = VIDEO_ES;
-    p_dec->fmt_out.i_codec = GetVlcChroma( p_context->pix_fmt );
+    if( GetVlcChroma( &p_dec->fmt_out.video, p_context->pix_fmt ) != VLC_SUCCESS )
+    {
+        /* we are doomed */
+        msg_Err( p_dec, "avcodec does not know how to convert this chroma" );
+        p_dec->fmt_out.i_codec = VLC_FOURCC('I','4','2','0');
+    }
+    p_dec->fmt_out.i_codec = p_dec->fmt_out.video.i_chroma;
 
     /* Setup palette */
     if( p_dec->fmt_in.video.p_palette )
@@ -747,7 +754,7 @@ static void ffmpeg_CopyPicture( decoder_t *p_dec,
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
 
-    if( GetVlcChroma( p_sys->p_context->pix_fmt ) )
+    if( TestFfmpegChroma( p_sys->p_context->pix_fmt, -1 ) )
     {
         int i_plane, i_size, i_line;
         uint8_t *p_dst, *p_src;
@@ -807,13 +814,14 @@ static int ffmpeg_GetFrameBuf( struct AVCodecContext *p_context,
 
     /* Some codecs set pix_fmt only after the 1st frame has been decoded,
      * so this check is necessary. */
-    if( !GetVlcChroma( p_context->pix_fmt ) ||
+    if( GetVlcChroma( &p_dec->fmt_out.video, p_context->pix_fmt ) != VLC_SUCCESS ||
         p_sys->p_context->width % 16 || p_sys->p_context->height % 16 )
     {
         msg_Dbg( p_dec, "disabling direct rendering" );
         p_sys->b_direct_rendering = 0;
         return avcodec_default_get_buffer( p_context, p_ff_pic );
     }
+    p_dec->fmt_out.i_codec = p_dec->fmt_out.video.i_chroma;
 
     /* Get a new picture */
     //p_sys->p_vout->render.b_allow_modify_pics = 0;

modules/demux/avformat/demux.c

@@ -207,17 +207,8 @@ int OpenDemux( vlc_object_t *p_this )
         const char *psz_type = "unknown";
 
         if( !GetVlcFourcc( cc->codec_id, NULL, &fcc, NULL ) )
-        {
             fcc = VLC_FOURCC( 'u', 'n', 'd', 'f' );
 
-            /* Special case for raw video data */
-            if( cc->codec_id == CODEC_ID_RAWVIDEO )
-            {
-                msg_Dbg( p_demux, "raw video, pixel format: %i", cc->pix_fmt );
-                fcc = GetVlcChroma( cc->pix_fmt );
-            }
-        }
-
         switch( cc->codec_type )
         {
         case CODEC_TYPE_AUDIO:
@@ -230,6 +221,19 @@ int OpenDemux( vlc_object_t *p_this )
             break;
         case CODEC_TYPE_VIDEO:
             es_format_Init( &fmt, VIDEO_ES, fcc );
+
+            /* Special case for raw video data */
+            if( cc->codec_id == CODEC_ID_RAWVIDEO )
+            {
+                msg_Dbg( p_demux, "raw video, pixel format: %i", cc->pix_fmt );
+                if( GetVlcChroma( &fmt.video, cc->pix_fmt ) != VLC_SUCCESS)
+                {
+                    msg_Err( p_demux, "was unable to find a FourCC match for raw video" );
+                }
+                else
+                    fmt.i_codec = fmt.video.i_chroma;
+            }
+
             fmt.video.i_width = cc->width;
             fmt.video.i_height = cc->height;
             if( cc->palctrl )

modules/video_filter/swscale.c

@@ -261,8 +261,11 @@ static int GetParameters( ScalerConfiguration *p_cfg,
                           const video_format_t *p_fmto,
                           int i_sws_flags_default )
 {
-    int i_fmti = GetFfmpegChroma( p_fmti->i_chroma );
-    int i_fmto = GetFfmpegChroma( p_fmto->i_chroma );
+    int i_fmti;
+    int i_fmto;
+    if( GetFfmpegChroma( &i_fmti, *p_fmti ) != VLC_SUCCESS ||
+        GetFfmpegChroma( &i_fmto, *p_fmto ) != VLC_SUCCESS )
+        return VLC_EGENERIC;
 
     bool b_has_a = false;
     bool b_add_a = false;