lua: don't use | operator on VLC errors

This could end up with unknown/invalid error values.

modules/lua/extension.c

@@ -876,7 +876,10 @@ int lua_ExecuteFunctionVa( extensions_manager_t *p_mgr, extension_t *p_ext,
         i_ret = VLC_EGENERIC;
     }
 
-    i_ret |= lua_DialogFlush( L );
+    if (i_ret != VLC_SUCCESS)
+        lua_DialogFlush( L );
+    else
+        i_ret = lua_DialogFlush( L );
 
 exit:
     return i_ret;
@@ -918,7 +921,10 @@ int lua_ExtensionTriggerMenu( extensions_manager_t *p_mgr,
         i_ret = VLC_EGENERIC;
     }
 
-    i_ret |= lua_DialogFlush( L );
+    if (i_ret != VLC_SUCCESS)
+        lua_DialogFlush( L );
+    else
+        i_ret = lua_DialogFlush( L );
     if( i_ret < VLC_SUCCESS )
     {
         msg_Dbg( p_mgr, "Something went wrong in %s (%s:%d)",