es: change the video_format_t color range to an enum

In many cases we don't know the value so it's better to use UNDEF.

A boolean also prevents from overwriting a value only when it's undefined.

The es_out display now separates the color space and the color range. One may
be known without the other.

include/vlc_es.h

@@ -374,7 +374,7 @@ struct video_format_t
     video_color_primaries_t primaries;                  /**< color primaries */
     video_transfer_func_t transfer;                   /**< transfer function */
     video_color_space_t space;                        /**< YCbCr color space */
-    bool b_color_range_full;                    /**< 0-255 instead of 16-235 */
+    video_color_range_t color_range;            /**< 0-255 instead of 16-235 */
     video_chroma_location_t chroma_location;      /**< YCbCr chroma location */
 
     video_multiview_mode_t multiview_mode;        /** Multiview mode, 2D, 3D */

modules/access/screen/win32.c

@@ -136,7 +136,7 @@ int screen_InitCapture( demux_t *p_demux )
     p_sys->fmt.video.i_sar_num = p_sys->fmt.video.i_sar_den = 1;
     p_sys->fmt.video.i_chroma         = i_chroma;
     p_sys->fmt.video.transfer         = TRANSFER_FUNC_SRGB;
-    p_sys->fmt.video.b_color_range_full = true;
+    p_sys->fmt.video.color_range      = COLOR_RANGE_FULL;
 
     switch( i_chroma )
     {

modules/access/v4l2/demux.c

@@ -436,7 +436,7 @@ static int InitVideo (demux_t *demux, int fd, uint32_t caps)
             es_fmt.video.primaries = COLOR_PRIMARIES_SRGB;
             es_fmt.video.transfer = TRANSFER_FUNC_SRGB;
             es_fmt.video.space = COLOR_SPACE_BT601;
-            es_fmt.video.b_color_range_full = true;
+            es_fmt.video.color_range = COLOR_RANGE_FULL;
             break;
         case V4L2_COLORSPACE_SRGB:
             es_fmt.video.primaries = COLOR_PRIMARIES_SRGB;
@@ -529,10 +529,10 @@ static int InitVideo (demux_t *demux, int fd, uint32_t caps)
         case V4L2_QUANTIZATION_DEFAULT:
             break;
         case V4L2_QUANTIZATION_FULL_RANGE:
-            es_fmt.video.b_color_range_full = true;
+            es_fmt.video.color_range = COLOR_RANGE_FULL;
             break;
         case V4L2_QUANTIZATION_LIM_RANGE:
-            es_fmt.video.b_color_range_full = false;
+            es_fmt.video.color_range = COLOR_RANGE_LIMITED;
             break;
         default:
             msg_Err (demux, "unknown quantization: %u",

modules/codec/aom.c

@@ -225,7 +225,7 @@ static void OutputFrame(decoder_t *dec, const struct aom_image *img)
         v->primaries = iso_23001_8_cp_to_vlc_primaries(img->cp);
         v->transfer = iso_23001_8_tc_to_vlc_xfer(img->tc);
         v->space = iso_23001_8_mc_to_vlc_coeffs(img->mc);
-        v->b_color_range_full = img->range == AOM_CR_FULL_RANGE;
+        v->color_range = img->range == AOM_CR_FULL_RANGE ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
     }
 
     dec->fmt_out.video.projection_mode = dec->fmt_in.video.projection_mode;

modules/codec/avcodec/avcommon.h

@@ -143,8 +143,19 @@ static inline vlc_rational_t FromAVRational(const AVRational rat)
 
 static inline void set_video_color_settings( const video_format_t *p_fmt, AVCodecContext *p_context )
 {
-    if( p_fmt->b_color_range_full )
+    switch( p_fmt->color_range )
+    {
+    case COLOR_RANGE_FULL:
         p_context->color_range = AVCOL_RANGE_JPEG;
+        break;
+    case COLOR_RANGE_LIMITED:
+        p_context->color_range = AVCOL_RANGE_MPEG;
+    case COLOR_RANGE_UNDEF: /* do nothing */
+        break;
+    default:
+        p_context->color_range = AVCOL_RANGE_UNSPECIFIED;
+        break;
+    }
 
     switch( p_fmt->space )
     {

modules/codec/avcodec/video.c

@@ -238,19 +238,15 @@ static int lavc_GetVideoFormat(decoder_t *dec, video_format_t *restrict fmt,
                                  * __MAX(ctx->ticks_per_frame, 1);
     }
 
-    /* FIXME we should only set the known values and let the core decide
-     * later of fallbacks, but we can't do that with a boolean */
     switch ( ctx->color_range )
     {
     case AVCOL_RANGE_JPEG:
-        fmt->b_color_range_full = true;
-        break;
-    case AVCOL_RANGE_UNSPECIFIED:
-        fmt->b_color_range_full = !vlc_fourcc_IsYUV( fmt->i_chroma );
+        fmt->color_range = COLOR_RANGE_FULL;
         break;
     case AVCOL_RANGE_MPEG:
-    default:
-        fmt->b_color_range_full = false;
+        fmt->color_range = COLOR_RANGE_LIMITED;
+        break;
+    default: /* do nothing */
         break;
     }
 

modules/codec/dav1d.c

@@ -127,7 +127,7 @@ static int NewPicture(Dav1dPicture *img, void *cookie)
         v->primaries = iso_23001_8_cp_to_vlc_primaries(img->seq_hdr->pri);
         v->transfer = iso_23001_8_tc_to_vlc_xfer(img->seq_hdr->trc);
         v->space = iso_23001_8_mc_to_vlc_coeffs(img->seq_hdr->mtrx);
-        v->b_color_range_full = img->seq_hdr->color_range;
+        v->color_range = img->seq_hdr->color_range ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
     }
 
     v->projection_mode = dec->fmt_in.video.projection_mode;

modules/codec/jpeg.c

@@ -180,7 +180,7 @@ static int OpenDecoder(vlc_object_t *p_this)
     p_dec->fmt_out.video.i_chroma =
     p_dec->fmt_out.i_codec = VLC_CODEC_RGB24;
     p_dec->fmt_out.video.transfer = TRANSFER_FUNC_SRGB;
-    p_dec->fmt_out.video.b_color_range_full = true;
+    p_dec->fmt_out.video.color_range = COLOR_RANGE_FULL;
     video_format_FixRgb(&p_dec->fmt_out.video);
 
     return VLC_SUCCESS;

modules/codec/png.c

@@ -126,7 +126,7 @@ static int OpenDecoder( vlc_object_t *p_this )
     /* Set output properties */
     p_dec->fmt_out.i_codec = VLC_CODEC_RGBA;
     p_dec->fmt_out.video.transfer = TRANSFER_FUNC_SRGB;
-    p_dec->fmt_out.video.b_color_range_full = true;
+    p_dec->fmt_out.video.color_range = COLOR_RANGE_FULL;
 
     /* Set callbacks */
     p_dec->pf_decode = DecodeBlock;

modules/codec/videotoolbox.m

@@ -488,7 +488,7 @@ static bool ConfigureVoutH264(decoder_t *p_dec)
             p_dec->fmt_out.video.primaries = primaries;
             p_dec->fmt_out.video.transfer = transfer;
             p_dec->fmt_out.video.space = colorspace;
-            p_dec->fmt_out.video.b_color_range_full = full_range;
+            p_dec->fmt_out.video.color_range = full_range ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
         }
     }
 

modules/codec/vpx.c

@@ -269,7 +269,7 @@ static int Decode(decoder_t *dec, block_t *block)
         v->primaries = vpx_color_mapping_table[img->cs].primaries;
         v->transfer = vpx_color_mapping_table[img->cs].transfer;
         v->space = vpx_color_mapping_table[img->cs].space;
-        v->b_color_range_full = img->range == VPX_CR_FULL_RANGE;
+        v->color_range = img->range == VPX_CR_FULL_RANGE ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
     }
 
     dec->fmt_out.video.projection_mode = dec->fmt_in.video.projection_mode;

modules/codec/x264.c

@@ -816,7 +816,7 @@ static int  Open ( vlc_object_t *p_this )
         return VLC_ENOMEM;
 
     fullrange = var_GetBool( p_enc, SOUT_CFG_PREFIX "fullrange" );
-    fullrange |= p_enc->fmt_in.video.b_color_range_full;
+    fullrange |= p_enc->fmt_in.video.color_range == COLOR_RANGE_FULL;
     p_enc->fmt_in.i_codec = fullrange ? VLC_CODEC_J420 : VLC_CODEC_I420;
     p_sys->i_colorspace = X264_CSP_I420;
     char *psz_profile = var_GetString( p_enc, SOUT_CFG_PREFIX "profile" );

modules/demux/mkv/matroska_segment_parse.cpp

@@ -709,11 +709,11 @@ void matroska_segment_c::ParseTrackEntry( const KaxTrackEntry *m )
             switch( static_cast<uint8>(range) )
             {
             case 1:
-                vars.tk->fmt.video.b_color_range_full = 0;
+                vars.tk->fmt.video.color_range = COLOR_RANGE_LIMITED;
                 name ="limited";
                 break;
             case 2:
-                vars.tk->fmt.video.b_color_range_full = 1;
+                vars.tk->fmt.video.color_range = COLOR_RANGE_FULL;
                 name ="full";
                 break;
             case 3: // Matrix coefficients + Transfer characteristics

modules/demux/mp4/essetup.c

@@ -504,8 +504,11 @@ int SetupVideoES( demux_t *p_demux, mp4_track_t *p_track, MP4_Box_t *p_sample )
                     iso_23001_8_tc_to_vlc_xfer( BOXDATA( p_colr )->nclc.i_transfer_function_idx );
             p_track->fmt.video.space =
                     iso_23001_8_mc_to_vlc_coeffs( BOXDATA( p_colr )->nclc.i_matrix_idx );
-            p_track->fmt.video.b_color_range_full = BOXDATA(p_colr)->i_type == VLC_FOURCC( 'n', 'c', 'l', 'x' ) &&
-                    (BOXDATA(p_colr)->nclc.i_full_range >> 7) != 0;
+            if ( BOXDATA(p_colr)->i_type == VLC_FOURCC( 'n', 'c', 'l', 'x' ) &&
+                    (BOXDATA(p_colr)->nclc.i_full_range >> 7) != 0 )
+                p_track->fmt.video.color_range = COLOR_RANGE_FULL;
+            else
+                p_track->fmt.video.color_range = COLOR_RANGE_LIMITED;
         }
     }
 
@@ -725,7 +728,7 @@ int SetupVideoES( demux_t *p_demux, mp4_track_t *p_track, MP4_Box_t *p_sample )
                             iso_23001_8_mc_to_vlc_coeffs( p_data->i_matrix_coeffs );
                 }
 
-                p_track->fmt.video.b_color_range_full = p_data->i_fullrange;
+                p_track->fmt.video.color_range = p_data->i_fullrange ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
                 p_track->fmt.video.i_bits_per_pixel = p_data->i_bit_depth;
 
                 if( p_data->i_codec_init_datasize )

modules/demux/mp4/heif.c

@@ -390,7 +390,8 @@ static int SetPictureProperties( demux_t *p_demux, uint32_t i_item_id,
                                             p_prop->data.p_colr->nclc.i_transfer_function_idx );
                     fmt->video.space = iso_23001_8_mc_to_vlc_coeffs(
                                         p_prop->data.p_colr->nclc.i_matrix_idx );
-                    fmt->video.b_color_range_full = p_prop->data.p_colr->nclc.i_full_range;
+                    fmt->video.color_range = p_prop->data.p_colr->nclc.i_full_range ?
+                                COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
                     break;
                 case ATOM_clli:
                     fmt->video.lighting.MaxCLL = p_prop->data.p_CoLL->i_maxCLL;

modules/hw/d3d9/dxva2_deinterlace.c

@@ -87,7 +87,8 @@ static void Flush(filter_t *filter)
 static void FillExtendedFormat( const video_format_t *p_fmt,
                                 DXVA2_ExtendedFormat *out )
 {
-    out->NominalRange = p_fmt->b_color_range_full ? DXVA2_NominalRange_0_255 : DXVA2_NominalRange_16_235;
+    out->NominalRange = p_fmt->color_range == COLOR_RANGE_FULL ?
+                DXVA2_NominalRange_0_255 : DXVA2_NominalRange_16_235;
     switch (p_fmt->space)
     {
     case COLOR_SPACE_BT601:

modules/mux/mp4/libmp4mux.c

@@ -993,7 +993,7 @@ static bo_t *GetColrBox(const video_format_t *p_vfmt, bool b_mov)
         bo_add_16be(p_box, vlc_primaries_to_iso_23001_8_cp(p_vfmt->primaries));
         bo_add_16be(p_box, vlc_xfer_to_iso_23001_8_tc(p_vfmt->transfer));
         bo_add_16be(p_box, vlc_coeffs_to_iso_23001_8_mc(p_vfmt->space));
-        bo_add_8(p_box, p_vfmt->b_color_range_full ? 0x80 : 0x00);
+        bo_add_8(p_box, p_vfmt->color_range == COLOR_RANGE_FULL ? 0x80 : 0x00);
     }
     return p_box;
 }

modules/packetizer/av1.c

@@ -147,7 +147,7 @@ static void UpdateDecoderFormat(decoder_t *p_dec)
         p_dec->fmt_out.video.primaries = prim;
         p_dec->fmt_out.video.transfer = xfer;
         p_dec->fmt_out.video.space = space;
-        p_dec->fmt_out.video.b_color_range_full = full;
+        p_dec->fmt_out.video.color_range = full ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
     }
 
     if(!p_dec->fmt_in.i_extra && !p_dec->fmt_out.i_extra)

modules/packetizer/av1_obu.c

@@ -194,17 +194,17 @@ static bool av1_parse_color_config(bs_t *p_bs,
 
     if(p_cc->mono_chrome)
     {
-        p_cc->color_range = bs_read1(p_bs);
+        p_cc->color_range = bs_read1(p_bs) ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
     }
     else if( p_cc->color_primaries == 1 &&
              p_cc->transfer_characteristics == 13 &&
              p_cc->matrix_coefficients == 0 )
     {
-        p_cc->color_range = 1;
+        p_cc->color_range = COLOR_RANGE_FULL;
     }
     else
     {
-        p_cc->color_range = bs_read1(p_bs);
+        p_cc->color_range = bs_read1(p_bs) ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
         if(seq_profile > 1)
         {
             if(BitDepth == 12)
@@ -542,7 +542,7 @@ bool AV1_get_colorimetry(const av1_OBU_sequence_header_t *p_seq,
     *p_primaries = iso_23001_8_cp_to_vlc_primaries(p_seq->color_config.color_primaries);
     *p_transfer = iso_23001_8_tc_to_vlc_xfer(p_seq->color_config.transfer_characteristics);
     *p_colorspace = iso_23001_8_mc_to_vlc_coeffs(p_seq->color_config.matrix_coefficients);
-    *p_full_range = p_seq->color_config.color_range;
+    *p_full_range = p_seq->color_config.color_range == COLOR_RANGE_FULL;
     return true;
 }
 

modules/packetizer/h264.c

@@ -227,10 +227,14 @@ static void ActivateSets( decoder_t *p_dec, const h264_sequence_parameter_set_t
                 }
             }
             if( p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF )
+            {
+                bool color_full;
                 h264_get_colorimetry( p_sps, &p_dec->fmt_out.video.primaries,
                                       &p_dec->fmt_out.video.transfer,
                                       &p_dec->fmt_out.video.space,
-                                      &p_dec->fmt_out.video.b_color_range_full );
+                                      &color_full );
+                p_dec->fmt_out.video.color_range = color_full ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
+            }
         }
 
         if( p_dec->fmt_out.i_extra == 0 && p_pps )

modules/packetizer/hevc.c

@@ -560,11 +560,13 @@ static void ActivateSets(decoder_t *p_dec,
 
         if(p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF)
         {
+            bool color_full;
             (void) hevc_get_colorimetry( p_sps,
                                          &p_dec->fmt_out.video.primaries,
                                          &p_dec->fmt_out.video.transfer,
                                          &p_dec->fmt_out.video.space,
-                                         &p_dec->fmt_out.video.b_color_range_full);
+                                         &color_full);
+            p_dec->fmt_out.video.color_range = color_full ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
         }
 
         unsigned sizes[4];

modules/packetizer/mpeg4video.c

@@ -475,7 +475,7 @@ static int ParseVO( decoder_t *p_dec, block_t *p_vo )
             p_dec->fmt_out.video.primaries = iso_23001_8_cp_to_vlc_primaries( colour_primaries );
             p_dec->fmt_out.video.transfer = iso_23001_8_tc_to_vlc_xfer( colour_xfer );
             p_dec->fmt_out.video.space = iso_23001_8_mc_to_vlc_coeffs( colour_matrix_coeff );
-            p_dec->fmt_out.video.b_color_range_full = full_range;
+            p_dec->fmt_out.video.color_range = full_range ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
         }
     }
 

modules/stream_out/transcode/encoder/video.c

@@ -238,7 +238,7 @@ void transcode_encoder_video_configure( vlc_object_t *p_obj,
     p_enc_out->space     = p_src->space;
     p_enc_out->transfer  = p_src->transfer;
     p_enc_out->primaries = p_src->primaries;
-    p_enc_out->b_color_range_full = p_src->b_color_range_full;
+    p_enc_out->color_range = p_src->color_range;
 
      /* set masks when RGB */
     video_format_FixRgb(&p_enc->p_encoder->fmt_in.video);

modules/video_output/placebo_utils.c

@@ -439,7 +439,7 @@ struct pl_color_repr vlc_placebo_ColorRepr(const video_format_t *fmt)
     return (struct pl_color_repr) {
         .sys        = sys,
         .alpha      = PL_ALPHA_PREMULTIPLIED,
-        .levels     = unlikely(fmt->b_color_range_full)
+        .levels     = unlikely(fmt->color_range == COLOR_RANGE_FULL)
                         ? PL_COLOR_LEVELS_PC
                         : PL_COLOR_LEVELS_TV,
         .bits = {

modules/video_output/win32/direct3d11.c

@@ -1049,7 +1049,7 @@ static void D3D11SetColorSpace(vout_display_t *vd)
         goto done;
     }
 
-    bool src_full_range = vd->source.b_color_range_full ||
+    bool src_full_range = vd->source.color_range == COLOR_RANGE_FULL ||
                           /* the YUV->RGB conversion already output full range */
                           is_d3d11_opaque(vd->source.i_chroma) ||
                           vlc_fourcc_IsYUV(vd->source.i_chroma);
@@ -1490,7 +1490,7 @@ static int Direct3D11CreateFormatResources(vout_display_t *vd, const video_forma
 
     hr = D3D11_CompilePixelShader(vd, &sys->hd3d, sys->legacy_shader, &sys->d3d_dev,
                                   &sys->display, fmt->transfer, fmt->primaries,
-                                  fmt->b_color_range_full,
+                                  fmt->color_range == COLOR_RANGE_FULL,
                                   &sys->picQuad);
     if (FAILED(hr))
     {

src/input/es_out.c

@@ -3517,10 +3517,20 @@ static void EsOutUpdateInfo( es_out_t *out, es_out_id_t *es, const vlc_meta_t *p
            };
            static_assert(ARRAY_SIZE(space_names) == COLOR_SPACE_MAX+1,
                          "Color space table mismatch");
-           info_category_AddInfo( p_cat, _("Color space"), _("%s %s Range"),
-               vlc_gettext(space_names[fmt->video.space]),
-               vlc_gettext(fmt->video.b_color_range_full
-                           ? N_("Full") : N_("Limited")) );
+           info_category_AddInfo( p_cat, _("Color space"), "%s",
+               vlc_gettext(space_names[fmt->video.space]) );
+       }
+       if( fmt->video.color_range != COLOR_RANGE_UNDEF )
+       {
+           static const char range_names[][16] = {
+               [COLOR_RANGE_UNDEF]   = N_("Undefined"),
+               [COLOR_RANGE_FULL]    = N_("Full"),
+               [COLOR_RANGE_LIMITED] = N_("Limited"),
+           };
+           static_assert(ARRAY_SIZE(range_names) == COLOR_RANGE_MAX+1,
+                         "Color range table mismatch");
+           info_category_AddInfo( p_cat, _("Color Range"), "%s",
+               vlc_gettext(range_names[fmt->video.color_range]) );
        }
        if( fmt->video.chroma_location != CHROMA_LOCATION_UNDEF )
        {