d3d11: allow rendering video to DirectComposition surfaces

configure.ac

@@ -2723,6 +2723,19 @@ dnl
 have_iapplication_data2="no"
 AC_CHECK_TYPES([IApplicationData2],[],[],[#include <windows.storage.h>])
 
+dnl
+dnl DirectComposition
+dnl
+AC_LANG_PUSH([C++])
+AC_CHECK_HEADERS([dcomp.h], [
+    have_dcomp="yes"
+], [], [
+  #include <windows.h>
+])
+AC_LANG_POP([C++])
+AM_CONDITIONAL([HAVE_DCOMP], [test "${have_dcomp}" = "yes"])
+
+
 dnl
 dnl  avformat demuxer/muxer plugin
 dnl

include/vlc_vout_window.h

@@ -63,6 +63,7 @@ enum vout_window_type {
     VOUT_WINDOW_TYPE_NSOBJECT /**< macOS/iOS view */,
     VOUT_WINDOW_TYPE_ANDROID_NATIVE /**< Android native window */,
     VOUT_WINDOW_TYPE_WAYLAND /**< Wayland surface */,
+    VOUT_WINDOW_TYPE_DCOMP /**< Win32 DirectComposition */,
 };
 
 /**
@@ -360,6 +361,7 @@ typedef struct vout_window_t {
         void     *nsobject;      /**< macOS/iOS view object */
         void     *anativewindow; /**< Android native window */
         struct wl_surface *wl;   /**< Wayland surface (client pointer) */
+        void     *dcomp_visual;  /**<  Win32 direct composition visual */
     } handle;
 
     /** Display server (mandatory)
@@ -373,6 +375,7 @@ typedef struct vout_window_t {
     union {
         char     *x11; /**< X11 display string (NULL = use default) */
         struct wl_display *wl; /**< Wayland display (client pointer) */
+        void* dcomp_device; /**< DirectComposition device */
     } display;
 
     const struct vout_window_operations *ops; /**< operations handled by the

modules/video_output/Makefile.am

@@ -157,6 +157,9 @@ libdirect3d11_plugin_la_SOURCES = video_output/win32/direct3d11.c \
  video_output/win32/d3d11_swapchain.c video_output/win32/d3d11_swapchain.h \
  video_output/win32/common.c video_output/win32/common.h
 libdirect3d11_plugin_la_LIBADD = libchroma_copy.la libd3d11_common.la $(LIBCOM) -luuid
+if HAVE_DCOMP
+libdirect3d11_plugin_la_SOURCES += video_output/win32/dcomp_wrapper.cpp video_output/win32/dcomp_wrapper.h
+endif
 if !HAVE_WINSTORE
 libdirect3d11_plugin_la_SOURCES += video_output/win32/events.c \
  video_output/win32/events.h \

modules/video_output/win32/d3d11_swapchain.c

@@ -52,11 +52,20 @@
 #include "d3d11_swapchain.h"
 #include "d3d11_shaders.h"
 
+#ifdef HAVE_DCOMP_H
+#  include "dcomp_wrapper.h"
+#endif
+
 typedef enum video_color_axis {
     COLOR_AXIS_RGB,
     COLOR_AXIS_YCBCR,
 } video_color_axis;
 
+typedef enum swapchain_surface_type {
+    SWAPCHAIN_SURFACE_HWND,
+    SWAPCHAIN_SURFACE_DCOMP,
+} swapchain_surface_type;
+
 typedef struct {
     DXGI_COLOR_SPACE_TYPE   dxgi;
     const char              *name;
@@ -75,9 +84,17 @@ struct d3d11_local_swapchain
     const d3d_format_t     *pixelFormat;
     const dxgi_color_space *colorspace;
 
+    swapchain_surface_type  swapchainSurfaceType;
+    union {
 #if !VLC_WINSTORE_APP
-    HWND                   swapchainHwnd;
+        HWND                hwnd;
 #endif /* !VLC_WINSTORE_APP */
+        struct {
+            void*           device;
+            void*           visual;
+        } dcomp;
+    } swapchainSurface;
+
     IDXGISwapChain1        *dxgiswapChain;   /* DXGI 1.2 swap chain */
     IDXGISwapChain4        *dxgiswapChain4;  /* DXGI 1.5 for HDR metadata */
     bool                    send_metadata;
@@ -276,9 +293,10 @@ static void FillSwapChainDesc(struct d3d11_local_swapchain *display, UINT width,
     }
 }
 
-static void CreateSwapchain(struct d3d11_local_swapchain *display, UINT width, UINT height)
+static void CreateSwapchainHwnd(struct d3d11_local_swapchain *display, UINT width, UINT height)
 {
-    if (display->swapchainHwnd == NULL)
+    vlc_assert(display->swapchainSurfaceType == SWAPCHAIN_SURFACE_HWND);
+    if (display->swapchainSurface.hwnd == NULL)
     {
         msg_Err(display->obj, "missing a HWND to create the swapchain");
         return;
@@ -302,14 +320,15 @@ static void CreateSwapchain(struct d3d11_local_swapchain *display, UINT width, U
     }
 
     hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)display->d3d_dev->d3ddevice,
-                                              display->swapchainHwnd, &scd,
+                                                  display->swapchainSurface.hwnd, &scd,
-                                              NULL, NULL, &display->dxgiswapChain);
+                                                  NULL, NULL, &display->dxgiswapChain);
+
     if (hr == DXGI_ERROR_INVALID_CALL && scd.Format == DXGI_FORMAT_R10G10B10A2_UNORM)
     {
         msg_Warn(display->obj, "10 bits swapchain failed, try 8 bits");
         scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
         hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)display->d3d_dev->d3ddevice,
-                                                  display->swapchainHwnd, &scd,
+                                                  display->swapchainSurface.hwnd, &scd,
                                                   NULL, NULL, &display->dxgiswapChain);
     }
     IDXGIFactory2_Release(dxgifactory);
@@ -317,6 +336,65 @@ static void CreateSwapchain(struct d3d11_local_swapchain *display, UINT width, U
         msg_Err(display->obj, "Could not create the SwapChain. (hr=0x%lX)", hr);
     }
 }
+
+#ifdef HAVE_DCOMP_H
+static void CreateSwapchainDComp(struct d3d11_local_swapchain *display, UINT width, UINT height)
+{
+    vlc_assert(display->swapchainSurfaceType == SWAPCHAIN_SURFACE_DCOMP);
+    if (display->swapchainSurface.dcomp.device == NULL || display->swapchainSurface.dcomp.visual == NULL)
+    {
+        msg_Err(display->obj, "missing a HWND to create the swapchain");
+        return;
+    }
+
+    DXGI_SWAP_CHAIN_DESC1 scd;
+    FillSwapChainDesc(display, width, height, &scd);
+    ZeroMemory(&scd, sizeof(scd));
+    scd.BufferCount = 3;
+    scd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+    scd.SampleDesc.Count = 1;
+    scd.SampleDesc.Quality = 0;
+    scd.Width = width;
+    scd.Height = height;
+    scd.Format = display->pixelFormat->formatTexture;
+    scd.Scaling = DXGI_SCALING_STRETCH;
+    scd.SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
+    scd.AlphaMode = DXGI_ALPHA_MODE_IGNORE;
+
+    IDXGIAdapter *dxgiadapter = D3D11DeviceAdapter(display->d3d_dev->d3ddevice);
+    if (unlikely(dxgiadapter==NULL)) {
+        msg_Err(display->obj, "Could not get the DXGI Adapter");
+        return;
+    }
+
+    IDXGIFactory2 *dxgifactory;
+    HRESULT hr = IDXGIAdapter_GetParent(dxgiadapter, &IID_IDXGIFactory2, (void **)&dxgifactory);
+    IDXGIAdapter_Release(dxgiadapter);
+    if (FAILED(hr)) {
+        msg_Err(display->obj, "Could not get the DXGI Factory. (hr=0x%lX)", hr);
+        return;
+    }
+
+    hr = IDXGIFactory2_CreateSwapChainForComposition(dxgifactory, (IUnknown *)display->d3d_dev->d3ddevice,
+                                                    &scd, NULL, &display->dxgiswapChain);
+    if (hr == DXGI_ERROR_INVALID_CALL && scd.Format == DXGI_FORMAT_R10G10B10A2_UNORM)
+    {
+        msg_Warn(display->obj, "10 bits swapchain failed, try 8 bits");
+        scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+        hr = IDXGIFactory2_CreateSwapChainForComposition(dxgifactory, (IUnknown *)display->d3d_dev->d3ddevice,
+                                                        &scd, NULL, &display->dxgiswapChain);
+    }
+    IDXGIFactory2_Release(dxgifactory);
+    if (SUCCEEDED(hr)) {
+        IDCompositionVisual_SetContent(display->swapchainSurface.dcomp.visual, (IUnknown *)display->dxgiswapChain);
+        IDCompositionDevice_Commit(display->swapchainSurface.dcomp.device);
+    }
+    if (FAILED(hr)) {
+        msg_Err(display->obj, "Could not create the SwapChain. (hr=0x%lX)", hr);
+    }
+}
+#endif /* HAVE_DCOMP_H */
+
 #endif /* !VLC_WINSTORE_APP */
 
 static bool UpdateSwapchain( struct d3d11_local_swapchain *display, const libvlc_video_render_cfg_t *cfg )
@@ -400,7 +478,13 @@ static bool UpdateSwapchain( struct d3d11_local_swapchain *display, const libvlc
     if ( display->dxgiswapChain == NULL )
     {
         display->pixelFormat = newPixelFormat;
-        CreateSwapchain(display, cfg->width, cfg->height);
+
+#ifdef HAVE_DCOMP_H
+        if (display->swapchainSurfaceType == SWAPCHAIN_SURFACE_DCOMP)
+            CreateSwapchainDComp(display, cfg->width, cfg->height);
+        else // SWAPCHAIN_TARGET_HWND
+#endif
+            CreateSwapchainHwnd(display, cfg->width, cfg->height);
 
         if (display->dxgiswapChain == NULL)
             return false;
@@ -555,7 +639,7 @@ bool LocalSwapchainSelectPlane( void *opaque, size_t plane )
     return true;
 }
 
-void *CreateLocalSwapchainHandle(vlc_object_t *o, HWND hwnd, d3d11_device_t *d3d_dev)
+void *CreateLocalSwapchainHandleHwnd(vlc_object_t *o, HWND hwnd, d3d11_device_t *d3d_dev)
 {
     struct d3d11_local_swapchain *display = vlc_obj_calloc(o, 1, sizeof(*display));
     if (unlikely(display == NULL))
@@ -563,7 +647,8 @@ void *CreateLocalSwapchainHandle(vlc_object_t *o, HWND hwnd, d3d11_device_t *d3d
 
     display->obj = o;
 #if !VLC_WINSTORE_APP
-    display->swapchainHwnd = hwnd;
+    display->swapchainSurfaceType = SWAPCHAIN_SURFACE_HWND;
+    display->swapchainSurface.hwnd = hwnd;
 #else // VLC_WINSTORE_APP
     VLC_UNUSED(hwnd);
 #endif // VLC_WINSTORE_APP
@@ -571,3 +656,20 @@ void *CreateLocalSwapchainHandle(vlc_object_t *o, HWND hwnd, d3d11_device_t *d3d
 
     return display;
 }
+
+#ifdef HAVE_DCOMP_H
+void *CreateLocalSwapchainHandleDComp(vlc_object_t *o, void* dcompDevice, void* dcompVisual, d3d11_device_t *d3d_dev)
+{
+    struct d3d11_local_swapchain *display = vlc_obj_calloc(o, 1, sizeof(*display));
+    if (unlikely(display == NULL))
+        return NULL;
+
+    display->obj = o;
+    display->swapchainSurfaceType = SWAPCHAIN_SURFACE_DCOMP;
+    display->swapchainSurface.dcomp.device = dcompDevice;
+    display->swapchainSurface.dcomp.visual = dcompVisual;
+    display->d3d_dev = d3d_dev;
+
+    return display;
+}
+#endif

modules/video_output/win32/d3d11_swapchain.h

@@ -28,7 +28,10 @@
 #include <vlc_codec.h>
 #include "../../video_chroma/d3d11_fmt.h"
 
-void *CreateLocalSwapchainHandle(vlc_object_t *, HWND, d3d11_device_t *d3d_dev);
+void *CreateLocalSwapchainHandleHwnd(vlc_object_t *, HWND, d3d11_device_t *d3d_dev);
+#ifdef HAVE_DCOMP_H
+void *CreateLocalSwapchainHandleDComp(vlc_object_t *, void* dcompDevice, void* dcompVisual, d3d11_device_t *d3d_dev);
+#endif
 
 void LocalSwapchainCleanupDevice( void *opaque );
 void LocalSwapchainSwap( void *opaque );

modules/video_output/win32/dcomp_wrapper.cpp

@@ -0,0 +1,35 @@
+/*****************************************************************************
+ * Copyright (c) 2020 VideoLAN
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "dcomp_wrapper.h"
+#include <dcomp.h>
+
+HRESULT IDCompositionVisual_SetContent(void* opaque, IUnknown *content)
+{
+    IDCompositionVisual* visual = (IDCompositionVisual*)opaque;
+    return visual->SetContent(content);
+}
+
+HRESULT IDCompositionDevice_Commit(void* opaque)
+{
+    IDCompositionDevice* device = (IDCompositionDevice*)opaque;
+    return device->Commit();
+}

modules/video_output/win32/dcomp_wrapper.h

@@ -0,0 +1,35 @@
+/*****************************************************************************
+ * Copyright (c) 2020 VideoLAN
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+#ifndef VLC_DCOMP_WRAPPER_H_
+#define VLC_DCOMP_WRAPPER_H_
+
+#include <windows.h>
+#include <unknwn.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+HRESULT IDCompositionVisual_SetContent(void* visual, IUnknown *content);
+HRESULT IDCompositionDevice_Commit(void* device);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

modules/video_output/win32/direct3d11.c

@@ -321,13 +321,22 @@ static int Open(vout_display_t *vd, const vout_display_cfg_t *cfg,
     if ( sys->swapCb == NULL || sys->startEndRenderingCb == NULL || sys->updateOutputCb == NULL )
     {
 #if !VLC_WINSTORE_APP
-        if (CommonWindowInit(VLC_OBJECT(vd), &sys->area, &sys->sys,
+        if (cfg->window->type == VOUT_WINDOW_TYPE_HWND)
+        {
+            if (CommonWindowInit(VLC_OBJECT(vd), &sys->area, &sys->sys,
                        vd->source.projection_mode != PROJECTION_MODE_RECTANGULAR))
-            goto error;
+                goto error;
+        }
+
 #endif /* !VLC_WINSTORE_APP */
 
         /* use our internal swapchain callbacks */
-        sys->outside_opaque      = CreateLocalSwapchainHandle(VLC_OBJECT(vd), sys->sys.hvideownd, sys->d3d_dev);
+#ifdef HAVE_DCOMP_H
+        if (cfg->window->type == VOUT_WINDOW_TYPE_DCOMP)
+            sys->outside_opaque      = CreateLocalSwapchainHandleDComp(VLC_OBJECT(vd), cfg->window->display.dcomp_device, cfg->window->handle.dcomp_visual, sys->d3d_dev);
+        else
+#endif
+            sys->outside_opaque      = CreateLocalSwapchainHandleHwnd(VLC_OBJECT(vd), sys->sys.hvideownd, sys->d3d_dev);
         if (unlikely(sys->outside_opaque == NULL))
             goto error;
         sys->updateOutputCb      = LocalSwapchainUpdateOutput;