d3d11: lock the device when creating the Video Processor

It doesn't seem to be thread safe.

(cherry picked from commit ff39bd6121) (edited)

edited:
- this branch doesn't have the D3D11_CreateProcessor helper
- the d3d11_device_t is stored locally rather than using a point
- the surface conversion filter didn't use a mutex at all

Signed-off-by: Steve Lhomme <robux4@ycbcr.xyz>

modules/hw/d3d11/d3d11_deinterlace.c

@@ -391,6 +391,8 @@ int D3D11OpenDeinterlace(vlc_object_t *obj)
         msg_Warn(filter, "No mutex found to lock the decoder");
     sys->d3d_dev.context_mutex = context_lock;
 
+    d3d11_device_lock(&sys->d3d_dev);
+
     const video_format_t *fmt = &filter->fmt_out.video;
 
     D3D11_VIDEO_PROCESSOR_CONTENT_DESC processorDesc = {
@@ -544,6 +546,7 @@ int D3D11OpenDeinterlace(vlc_object_t *obj)
     {
        goto error;
     }
+    d3d11_device_unlock(&sys->d3d_dev);
 
     sys->buffer_new = filter->owner.video.buffer_new;
     filter->owner.video.buffer_new = NewOutputPicture;
@@ -569,6 +572,7 @@ error:
     if (sys->d3d_dev.d3dcontext)
         D3D11_FilterReleaseInstance(&sys->d3d_dev);
     D3D11_Destroy(&sys->hd3d);
+    d3d11_device_unlock(&sys->d3d_dev);
     free(sys);
 
     return VLC_EGENERIC;

modules/hw/d3d11/d3d11_filters.c

@@ -388,6 +388,8 @@ static int D3D11OpenAdjust(vlc_object_t *obj)
         msg_Warn(filter, "No mutex found to lock the decoder");
     sys->d3d_dev.context_mutex = context_lock;
 
+    d3d11_device_lock(&sys->d3d_dev);
+
     const video_format_t *fmt = &filter->fmt_out.video;
 
     D3D11_VIDEO_PROCESSOR_CONTENT_DESC processorDesc = {
@@ -567,6 +569,7 @@ static int D3D11OpenAdjust(vlc_object_t *obj)
 
     filter->pf_video_filter = Filter;
     filter->p_sys = sys;
+    d3d11_device_unlock(&sys->d3d_dev);
 
     return VLC_SUCCESS;
 error:
@@ -592,6 +595,7 @@ error:
         ID3D11VideoDevice_Release(sys->d3dviddev);
     if (sys->d3d_dev.d3dcontext)
         D3D11_FilterReleaseInstance(&sys->d3d_dev);
+    d3d11_device_unlock(&sys->d3d_dev);
     free(sys);
 
     return VLC_EGENERIC;

modules/hw/d3d11/d3d11_surface.c

@@ -96,6 +96,15 @@ static int SetupProcessor(filter_t *p_filter, ID3D11Device *d3ddevice,
     if (unlikely(FAILED(hr)))
         goto error;
 
+    HANDLE context_lock = INVALID_HANDLE_VALUE;
+    UINT dataSize = sizeof(context_lock);
+    hr = ID3D11DeviceContext_GetPrivateData(sys->d3d_dev.d3dcontext, &GUID_CONTEXT_MUTEX, &dataSize, &context_lock);
+    if (FAILED(hr))
+        msg_Warn(p_filter, "No mutex found to lock the decoder");
+    sys->d3d_dev.context_mutex = context_lock;
+
+    d3d11_device_lock(&sys->d3d_dev);
+
     const video_format_t *fmt = &p_filter->fmt_in.video;
     D3D11_VIDEO_PROCESSOR_CONTENT_DESC processorDesc = {
         .InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
@@ -156,6 +165,7 @@ static int SetupProcessor(filter_t *p_filter, ID3D11Device *d3ddevice,
             else
             {
                 sys->procEnumerator  = processorEnumerator;
+                d3d11_device_unlock(&sys->d3d_dev);
                 return VLC_SUCCESS;
             }
         }
@@ -173,6 +183,7 @@ error:
         ID3D11VideoContext_Release(sys->d3dvidctx);
     if (sys->d3dviddev)
         ID3D11VideoDevice_Release(sys->d3dviddev);
+    d3d11_device_unlock(&sys->d3d_dev);
     return VLC_EGENERIC;
 }
 #endif