qt: fix video window visible twice with X11 compositor

removing BypassWindowManagerHint was causing the window to be displayed twice on
some desktop environments

fix: #26590

modules/gui/qt/maininterface/compositor_x11.cpp

@@ -152,26 +152,33 @@ bool CompositorX11::makeMainInterface(MainCtx* mainCtx)
 {
     m_mainCtx = mainCtx;
 
-    m_videoWidget = std::make_unique<DummyNativeWidget>();
+
+    bool useCSD = m_mainCtx->useClientSideDecoration();
+    m_renderWindow = std::make_unique<vlc::CompositorX11RenderWindow>(m_intf, m_conn, useCSD);
+    if (!m_renderWindow->init())
+        return false;
+
+    m_videoWidget = std::make_unique<DummyNativeWidget>(m_renderWindow.get());
     // widget would normally require WindowTransparentForInput, without this
     // we end up with an invisible area within our window that grabs our mouse events.
     // But using this this causes rendering issues with some VoutDisplay
-    // (xcb_render for instance) instead, we manually, set a null intput region afterwards
+    // (xcb_render for instance) instead, we manually, set a null input region afterwards
     // see setTransparentForMouseEvent
     m_videoWidget->winId();
-    m_videoWidget->setWindowFlag(Qt::WindowStaysOnBottomHint);
-    m_videoWidget->show();
 
-    bool useCSD = m_mainCtx->useClientSideDecoration();
-    m_renderWindow = std::make_unique<vlc::CompositorX11RenderWindow>(m_intf, m_conn, useCSD);
-    if (!m_renderWindow->init())
-        return false;
+    //update manually EventMask as we don't use WindowTransparentForInput
+    const uint32_t mask = XCB_CW_EVENT_MASK;
+    const uint32_t values = XCB_EVENT_MASK_EXPOSURE | XCB_EVENT_MASK_STRUCTURE_NOTIFY
+        | XCB_EVENT_MASK_PROPERTY_CHANGE | XCB_EVENT_MASK_FOCUS_CHANGE;
+    xcb_change_window_attributes(QX11Info::connection(), m_videoWidget->winId(), mask, &values);
+    setTransparentForMouseEvent(QX11Info::connection(), m_videoWidget->winId());
+    m_videoWidget->show();
 
     m_interfaceWindow = m_renderWindow->getWindow();
 
     m_qmlView = std::make_unique<CompositorX11UISurface>(m_interfaceWindow);
-    m_qmlView->setFlag(Qt::WindowType::BypassWindowManagerHint);
     m_qmlView->setFlag(Qt::WindowType::WindowTransparentForInput);
+    m_qmlView->setParent(m_interfaceWindow);
     m_qmlView->winId();
     m_qmlView->show();
 

modules/gui/qt/maininterface/compositor_x11_renderwindow.cpp

@@ -540,12 +540,10 @@ void CompositorX11RenderWindow::setVideoSize(const QSize& size)
 
 void CompositorX11RenderWindow::setVideoWindow( QWindow* window)
 {
-    window->setParent(m_window);
     //ensure Qt x11 pending operation have been forwarded to the server
     xcb_flush(QX11Info::connection());
     m_videoClient = std::make_unique<CompositorX11RenderClient>(m_intf, m_conn, window);
     m_videoPosition = QRect(0,0,0,0);
-    setTransparentForMouseEvent(QX11Info::connection(), window->winId());
     m_videoWindow = window;
     emit videoSurfaceChanged(m_videoClient.get());
 }
@@ -562,8 +560,6 @@ void CompositorX11RenderWindow::disableVideoWindow()
 
 void CompositorX11RenderWindow::setInterfaceWindow(CompositorX11UISurface* window)
 {
-    assert(m_window);
-    window->setParent(m_window);
     //ensure Qt x11 pending operation have been forwarded to the server
     xcb_flush(QX11Info::connection());
     m_interfaceClient = std::make_unique<CompositorX11RenderClient>(m_intf, m_conn, window);

modules/gui/qt/maininterface/compositor_x11_utils.cpp

@@ -19,8 +19,9 @@
 #include <vlc_cxx_helpers.hpp>
 #include "compositor_x11_utils.hpp"
 
-namespace vlc {
+#include <QWindow>
 
+namespace vlc {
 
 DummyNativeWidget::DummyNativeWidget(QWidget* parent, Qt::WindowFlags f)
     : QWidget(parent, f)
@@ -29,6 +30,18 @@ DummyNativeWidget::DummyNativeWidget(QWidget* parent, Qt::WindowFlags f)
     setAttribute(Qt::WA_OpaquePaintEvent, true);
     setAttribute(Qt::WA_PaintOnScreen, true);
     setAttribute(Qt::WA_MouseTracking, true);
+    setAttribute(Qt::WA_TranslucentBackground, false);
+    QWindow* w =  window()->windowHandle();
+    assert(w);
+    /*
+     * force the window not to have an alpha channel, the  parent window
+     * may have an alpha channel and child widget would inhertit the format
+     * even if we set Qt::WA_TranslucentBackground to false. having an alpha
+     * in this surface would lead to the video begin semi-tranparent.
+     */
+    QSurfaceFormat format = w->format();
+    format.setAlphaBufferSize(0);
+    w->setFormat(format);
 }
 
 DummyNativeWidget::~DummyNativeWidget()