d3d11: lookup d3d_format_t with or without an alpha texture

modules/access/screen/dxgi.cpp

@@ -248,7 +248,7 @@ int screen_InitCaptureDXGI(demux_t *p_demux)
     DXGI_OUTDUPL_DESC outDesc;
     p_data->duplication->GetDesc(&outDesc);
 
-    p_data->output_format = D3D11_RenderFormat(outDesc.ModeDesc.Format ,true);
+    p_data->output_format = D3D11_RenderFormat(outDesc.ModeDesc.Format, DXGI_FORMAT_UNKNOWN ,true);
     if (unlikely(!p_data->output_format))
     {
         msg_Err(p_demux, "Unknown texture format %d", outDesc.ModeDesc.Format);

modules/codec/avcodec/d3d11va.c

@@ -418,8 +418,8 @@ static int DxSetupOutput(vlc_va_t *va, const directx_va_mode_t *mode, const vide
     }
 
     if (decoder_format == NULL || decoder_format->formatTexture != DXGI_FORMAT_NV12)
-        processorInput[idx++] = D3D11_RenderFormat(DXGI_FORMAT_NV12 ,true);
-    processorInput[idx++] = D3D11_RenderFormat(DXGI_FORMAT_420_OPAQUE ,true);
+        processorInput[idx++] = D3D11_RenderFormat(DXGI_FORMAT_NV12, DXGI_FORMAT_UNKNOWN ,true);
+    processorInput[idx++] = D3D11_RenderFormat(DXGI_FORMAT_420_OPAQUE, DXGI_FORMAT_UNKNOWN ,true);
     processorInput[idx++] = NULL;
 
     /* */

modules/codec/mft_d3d11.cpp

@@ -129,7 +129,7 @@ HRESULT MFHW_d3d11::SetupVideoContext(vlc_logger *logger, ComPtr<IMFDXGIBuffer>
     fmt_out.video.i_width = desc.Width;
     fmt_out.video.i_height = desc.Height;
 
-    cfg = D3D11_RenderFormat(desc.Format ,true);
+    cfg = D3D11_RenderFormat(desc.Format, DXGI_FORMAT_UNKNOWN, true);
 
     fmt_out.i_codec = cfg->fourcc;
     fmt_out.video.i_chroma = cfg->fourcc;

modules/hw/d3d11/d3d11_decoder.cpp

@@ -112,7 +112,7 @@ static int DecodeFrame( decoder_t *p_dec, block_t *p_block )
         D3D11_TEXTURE2D_DESC outDesc;
         src_sys->texture[0]->GetDesc(&outDesc);
 
-        p_sys->output_format = D3D11_RenderFormat(outDesc.Format ,true);
+        p_sys->output_format = D3D11_RenderFormat(outDesc.Format, DXGI_FORMAT_UNKNOWN ,true);
         if (unlikely(!p_sys->output_format->name))
         {
             msg_Err(p_dec, "Unknown texture format %d", outDesc.Format);

modules/hw/d3d11/d3d11_deinterlace.c

@@ -254,7 +254,7 @@ int D3D11OpenDeinterlace(filter_t *filter)
     d3d11_decoder_device_t *dev_sys = GetD3D11OpaqueContext( filter->vctx_in );
     sys->d3d_dev = &dev_sys->d3d_dev;
 
-    sys->output_format = D3D11_RenderFormat(vctx_sys->format ,true);
+    sys->output_format = D3D11_RenderFormat(vctx_sys->format, DXGI_FORMAT_UNKNOWN ,true);
     if (unlikely(sys->output_format == NULL))
         goto error;
 

modules/hw/d3d11/d3d11_filters.c

@@ -188,7 +188,7 @@ static picture_t *AllocPicture( filter_t *p_filter )
 {
     d3d11_video_context_t *vctx_sys = GetD3D11ContextPrivate( p_filter->vctx_out );
 
-    const d3d_format_t *cfg = D3D11_RenderFormat(vctx_sys->format ,true);
+    const d3d_format_t *cfg = D3D11_RenderFormat(vctx_sys->format, DXGI_FORMAT_UNKNOWN ,true);
     if (unlikely(cfg == NULL))
         return NULL;
 

modules/hw/d3d11/d3d11_surface.c

@@ -623,7 +623,7 @@ static picture_t *AllocateCPUtoGPUTexture(filter_t *p_filter, filter_sys_t *p_sy
 
     d3d11_video_context_t *vctx_sys = GetD3D11ContextPrivate( p_filter->vctx_out );
 
-    const d3d_format_t *cfg = D3D11_RenderFormat(vctx_sys->format ,false);
+    const d3d_format_t *cfg = D3D11_RenderFormat(vctx_sys->format, DXGI_FORMAT_UNKNOWN ,false);
     if (unlikely(cfg == NULL))
         return NULL;
 

modules/video_chroma/d3d11_fmt.h

@@ -151,12 +151,13 @@ void AcquireD3D11PictureSys(picture_sys_d3d11_t *p_sys);
 
 void ReleaseD3D11PictureSys(picture_sys_d3d11_t *p_sys);
 
-static inline const d3d_format_t *D3D11_RenderFormat(DXGI_FORMAT opaque, bool gpu_based)
+static inline const d3d_format_t *D3D11_RenderFormat(DXGI_FORMAT opaque, DXGI_FORMAT alpha, bool gpu_based)
 {
     for (const d3d_format_t *output_format = DxgiGetRenderFormatList();
             output_format->name != NULL; ++output_format)
     {
         if (output_format->formatTexture == opaque &&
+            output_format->alphaTexture == alpha &&
             is_d3d11_opaque(output_format->fourcc) == gpu_based)
         {
             return output_format;

modules/video_output/win32/direct3d11.cpp

@@ -215,7 +215,7 @@ static int UpdateDisplayFormat(vout_display_t *vd, const video_format_t *fmt)
 
     display_info_t new_display = { };
 
-    new_display.pixelFormat = D3D11_RenderFormat((DXGI_FORMAT)out.dxgi_format, false);
+    new_display.pixelFormat = D3D11_RenderFormat((DXGI_FORMAT)out.dxgi_format, DXGI_FORMAT_UNKNOWN, false);
     if (unlikely(new_display.pixelFormat == NULL))
     {
         msg_Err(vd, "Could not find the output format.");
@@ -897,7 +897,7 @@ static int SetupOutputFormat(vout_display_t *vd, video_format_t *fmt, vlc_video_
     if (vtcx_sys != NULL &&
         D3D11_DeviceSupportsFormat( sys->d3d_dev, vtcx_sys->format, D3D11_FORMAT_SUPPORT_SHADER_LOAD ))
     {
-        sys->picQuad.generic.textureFormat = D3D11_RenderFormat(vtcx_sys->format ,true);
+        sys->picQuad.generic.textureFormat = D3D11_RenderFormat(vtcx_sys->format, DXGI_FORMAT_UNKNOWN ,true);
     }
 
     // look for the requested pixel format first