directx_va: move the decoder setup out of the generic directx surface setup

modules/codec/avcodec/d3d11va.c

@@ -387,7 +387,12 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     if (err!=VLC_SUCCESS)
         goto error;
 
-    err = directx_va_Setup(va, &sys->dx_sys, ctx, fmt, isXboxHardware(sys->d3d_dev.d3ddevice), &sys->decoder_guid);
+    video_format_t fmt_out;
+    err = directx_va_Setup(va, &sys->dx_sys, ctx, fmt, isXboxHardware(sys->d3d_dev.d3ddevice), &fmt_out, &sys->hw.surface_count, &sys->decoder_guid);
+    if (err != VLC_SUCCESS)
+        goto error;
+
+    err = va_pool_SetupDecoder(va, &sys->dx_sys.va_pool, ctx, &fmt_out, sys->hw.surface_count);
     if (err != VLC_SUCCESS)
         goto error;
 

modules/codec/avcodec/directx_va.c

@@ -280,7 +280,8 @@ char *directx_va_GetDecoderName(const GUID *guid)
 
 /* */
 int directx_va_Setup(vlc_va_t *va, directx_sys_t *dx_sys, const AVCodecContext *avctx,
-                     const es_format_t *fmt, int flag_xbox, GUID *found_guid)
+                     const es_format_t *fmt, int flag_xbox,
+                     video_format_t *fmt_out, unsigned *surfaces, GUID *found_guid)
 {
     /* */
     if (FindVideoServiceConversion(va, dx_sys, fmt, avctx, found_guid)) {
@@ -323,7 +324,30 @@ int directx_va_Setup(vlc_va_t *va, directx_sys_t *dx_sys, const AVCodecContext *
     if ( avctx->active_thread_type & FF_THREAD_FRAME )
         surface_count += avctx->thread_count;
 
-    return va_pool_SetupDecoder(va, &dx_sys->va_pool, avctx, surface_count, surface_alignment);
+    if (avctx->coded_width <= 0 || avctx->coded_height <= 0)
+        return VLC_EGENERIC;
+
+    assert((surface_alignment & (surface_alignment - 1)) == 0); /* power of 2 */
+#define ALIGN(x, y) (((x) + ((y) - 1)) & ~((y) - 1))
+    int surface_width  = ALIGN(avctx->coded_width,  surface_alignment);
+    int surface_height = ALIGN(avctx->coded_height, surface_alignment);
+
+    if (avctx->coded_width != surface_width || avctx->coded_height != surface_height)
+        msg_Warn( va, "surface dimensions (%dx%d) differ from avcodec dimensions (%dx%d)",
+                  surface_width, surface_height,
+                  avctx->coded_width, avctx->coded_height);
+
+    *fmt_out = fmt->video;
+    fmt_out->i_width  = surface_width;
+    fmt_out->i_height = surface_height;
+
+    /* FIXME transmit a video_format_t by VaSetup directly */
+    fmt_out->i_frame_rate      = avctx->framerate.num;
+    fmt_out->i_frame_rate_base = avctx->framerate.den;
+
+
+    *surfaces = surface_count;
+    return VLC_SUCCESS;
 }
 
 void directx_va_Close(vlc_va_t *va, directx_sys_t *dx_sys)

modules/codec/avcodec/directx_va.h

@@ -68,7 +68,8 @@ typedef struct
 
 int directx_va_Open(vlc_va_t *, const struct va_pool_cfg *, directx_sys_t *);
 void directx_va_Close(vlc_va_t *, directx_sys_t *);
-int directx_va_Setup(vlc_va_t *, directx_sys_t *, const AVCodecContext *avctx, const es_format_t *, int flag_xbox, GUID *found_guid);
+int directx_va_Setup(vlc_va_t *, directx_sys_t *, const AVCodecContext *avctx, const es_format_t *, int flag_xbox,
+                     video_format_t *fmt_out, unsigned *surface_count, GUID *found_guid);
 char *directx_va_GetDecoderName(const GUID *guid);
 bool directx_va_canUseDecoder(vlc_va_t *, UINT VendorId, UINT DeviceId, const GUID *pCodec, UINT driverBuild);
 

modules/codec/avcodec/dxva2.c

@@ -314,7 +314,12 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     if (err!=VLC_SUCCESS)
         goto error;
 
-    err = directx_va_Setup(va, &sys->dx_sys, ctx, fmt, 0, &sys->decoder_guid);
+    video_format_t fmt_out;
+    err = directx_va_Setup(va, &sys->dx_sys, ctx, fmt, 0, &fmt_out, &sys->hw.surface_count, &sys->decoder_guid);
+    if (err != VLC_SUCCESS)
+        goto error;
+
+    err = va_pool_SetupDecoder(va, &sys->dx_sys.va_pool, ctx, &fmt_out, sys->hw.surface_count);
     if (err != VLC_SUCCESS)
         goto error;
 

modules/codec/avcodec/va_surface.c

@@ -56,27 +56,14 @@ static void DestroyVideoDecoder(vlc_va_sys_t *sys, va_pool_t *va_pool)
 static int SetupSurfaces(vlc_va_t *, va_pool_t *, unsigned count);
 
 /* */
-int va_pool_SetupDecoder(vlc_va_t *va, va_pool_t *va_pool, const AVCodecContext *avctx, unsigned count, int alignment)
+int va_pool_SetupDecoder(vlc_va_t *va, va_pool_t *va_pool, const AVCodecContext *avctx,
+                         const video_format_t *fmt, unsigned count)
 {
     int err = VLC_ENOMEM;
-    unsigned i = va_pool->surface_count;
-
-    if (avctx->coded_width <= 0 || avctx->coded_height <= 0)
-        return VLC_EGENERIC;
-
-    assert((alignment & (alignment - 1)) == 0); /* power of 2 */
-#define ALIGN(x, y) (((x) + ((y) - 1)) & ~((y) - 1))
-    int surface_width  = ALIGN(avctx->coded_width,  alignment);
-    int surface_height = ALIGN(avctx->coded_height, alignment);
-
-    if (avctx->coded_width != surface_width || avctx->coded_height != surface_height)
-        msg_Warn( va, "surface dimensions (%dx%d) differ from avcodec dimensions (%dx%d)",
-                  surface_width, surface_height,
-                  avctx->coded_width, avctx->coded_height);
 
     if ( va_pool->surface_count >= count &&
-         va_pool->surface_width == surface_width &&
-         va_pool->surface_height == surface_height )
+         va_pool->surface_width  == fmt->i_width &&
+         va_pool->surface_height == fmt->i_height )
     {
         msg_Dbg(va, "reusing surface pool");
         err = VLC_SUCCESS;
@@ -92,23 +79,15 @@ int va_pool_SetupDecoder(vlc_va_t *va, va_pool_t *va_pool, const AVCodecContext
     if (count > MAX_SURFACE_COUNT)
         return VLC_EGENERIC;
 
-    /* FIXME transmit a video_format_t by VaSetup directly */
-    video_format_t fmt;
-    memset(&fmt, 0, sizeof(fmt));
-    fmt.i_width  = surface_width;
-    fmt.i_height = surface_height;
-    fmt.i_frame_rate      = avctx->framerate.num;
-    fmt.i_frame_rate_base = avctx->framerate.den;
-
-    err = va_pool->callbacks->pf_create_decoder_surfaces(va, avctx->codec_id, &fmt, count);
+    err = va_pool->callbacks->pf_create_decoder_surfaces(va, avctx->codec_id, fmt, count);
     if (err == VLC_SUCCESS)
     {
-        va_pool->surface_width  = surface_width;
-        va_pool->surface_height = surface_height;
+        va_pool->surface_width  = fmt->i_width;
+        va_pool->surface_height = fmt->i_height;
+        va_pool->surface_count = va_pool->can_extern_pool ? 0 : count;
     }
 
 done:
-    va_pool->surface_count = i;
     if (err == VLC_SUCCESS)
         err = SetupSurfaces(va, va_pool, count);
 

modules/codec/avcodec/va_surface_internal.h

@@ -76,7 +76,7 @@ struct va_pool_cfg {
 
 int va_pool_Open(vlc_va_t *, const struct va_pool_cfg *, va_pool_t *);
 void va_pool_Close(vlc_va_t *va, va_pool_t *);
-int va_pool_SetupDecoder(vlc_va_t *, va_pool_t *, const AVCodecContext *, unsigned count, int alignment);
+int va_pool_SetupDecoder(vlc_va_t *, va_pool_t *, const AVCodecContext *, const video_format_t *, unsigned count);
 picture_context_t *va_pool_Get(va_pool_t *);
 void va_surface_AddRef(vlc_va_surface_t *surface);
 void va_surface_Release(vlc_va_surface_t *surface);