riscv
/
qemu
mirror of https://gitea.goldendeliverer.com/riscv/qemu.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

vnc: fix memory leak when vnc disconnect 

Currently when qemu receives a vnc connect, it creates a 'VncState' to
represent this connection. In 'vnc_worker_thread_loop' it creates a
local 'VncState'. The connection 'VcnState' and local 'VncState' exchange
data in 'vnc_async_encoding_start' and 'vnc_async_encoding_end'.
In 'zrle_compress_data' it calls 'deflateInit2' to allocate the libz library
opaque data. The 'VncState' used in 'zrle_compress_data' is the local
'VncState'. In 'vnc_zrle_clear' it calls 'deflateEnd' to free the libz
library opaque data. The 'VncState' used in 'vnc_zrle_clear' is the connection
'VncState'. In currently implementation there will be a memory leak when the
vnc disconnect. Following is the asan output backtrack:

Direct leak of 29760 byte(s) in 5 object(s) allocated from:
    0 0xffffa67ef3c3 in __interceptor_calloc (/lib64/libasan.so.4+0xd33c3)
    1 0xffffa65071cb in g_malloc0 (/lib64/libglib-2.0.so.0+0x571cb)
    2 0xffffa5e968f7 in deflateInit2_ (/lib64/libz.so.1+0x78f7)
    3 0xaaaacec58613 in zrle_compress_data ui/vnc-enc-zrle.c:87
    4 0xaaaacec58613 in zrle_send_framebuffer_update ui/vnc-enc-zrle.c:344
    5 0xaaaacec34e77 in vnc_send_framebuffer_update ui/vnc.c:919
    6 0xaaaacec5e023 in vnc_worker_thread_loop ui/vnc-jobs.c:271
    7 0xaaaacec5e5e7 in vnc_worker_thread ui/vnc-jobs.c:340
    8 0xaaaacee4d3c3 in qemu_thread_start util/qemu-thread-posix.c:502
    9 0xffffa544e8bb in start_thread (/lib64/libpthread.so.0+0x78bb)
    10 0xffffa53965cb in thread_start (/lib64/libc.so.6+0xd55cb)

This is because the opaque allocated in 'deflateInit2' is not freed in
'deflateEnd'. The reason is that the 'deflateEnd' calls 'deflateStateCheck'
and in the latter will check whether 's->strm != strm'(libz's data structure).
This check will be true so in 'deflateEnd' it just return 'Z_STREAM_ERROR' and
not free the data allocated in 'deflateInit2'.

The reason this happens is that the 'VncState' contains the whole 'VncZrle',
so when calling 'deflateInit2', the 's->strm' will be the local address.
So 's->strm != strm' will be true.

To fix this issue, we need to make 'zrle' of 'VncState' to be a pointer.
Then the connection 'VncState' and local 'VncState' exchange mechanism will
work as expection. The 'tight' of 'VncState' has the same issue, let's also turn
it to a pointer.

Reported-by: Ying Fang <fangying1@huawei.com>
Signed-off-by: Li Qiang <liq3ea@163.com>
Message-id: 20190831153922.121308-1-liq3ea@163.com
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
pull/82/head
Li Qiang 7 years ago
committed by Gerd Hoffmann
parent
6105683da3
commit
6bf21f3d83
6 changed files with 170 additions and 162 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 219
      ui/vnc-enc-tight.c
  2. 11
      ui/vnc-enc-zlib.c
  3. 68
      ui/vnc-enc-zrle.c
  4. 2
      ui/vnc-enc-zrle.inc.c
  5. 28
      ui/vnc.c
  6. 4
      ui/vnc.h

219
ui/vnc-enc-tight.c
View File

@ -116,7 +116,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
static bool tight_can_send_png_rect(VncState *vs, int w, int h)
{
if (vs->tight.type != VNC_ENCODING_TIGHT_PNG) {
if (vs->tight->type != VNC_ENCODING_TIGHT_PNG) {
return false;
}
@ -144,7 +144,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
int pixels = 0;
int pix, left[3];
unsigned int errors;
unsigned char *buf = vs->tight.tight.buffer;
unsigned char *buf = vs->tight->tight.buffer;
/*
* If client is big-endian, color samples begin from the second
@ -215,7 +215,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
int pixels = 0; \
int sample, sum, left[3]; \
unsigned int errors; \
unsigned char *buf = vs->tight.tight.buffer; \
unsigned char *buf = vs->tight->tight.buffer; \
\
endian = 0; /* FIXME */ \
\
@ -296,8 +296,8 @@ static int
tight_detect_smooth_image(VncState *vs, int w, int h)
{
unsigned int errors;
int compression = vs->tight.compression;
int quality = vs->tight.quality;
int compression = vs->tight->compression;
int quality = vs->tight->quality;
if (!vs->vd->lossy) {
return 0;
@ -309,7 +309,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
return 0;
}
if (vs->tight.quality != (uint8_t)-1) {
if (vs->tight->quality != (uint8_t)-1) {
if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) {
return 0;
}
@ -320,9 +320,9 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
}
if (vs->client_pf.bytes_per_pixel == 4) {
if (vs->tight.pixel24) {
if (vs->tight->pixel24) {
errors = tight_detect_smooth_image24(vs, w, h);
if (vs->tight.quality != (uint8_t)-1) {
if (vs->tight->quality != (uint8_t)-1) {
return (errors < tight_conf[quality].jpeg_threshold24);
}
return (errors < tight_conf[compression].gradient_threshold24);
@ -352,7 +352,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
uint##bpp##_t c0, c1, ci; \
int i, n0, n1; \
\
data = (uint##bpp##_t *)vs->tight.tight.buffer; \
data = (uint##bpp##_t *)vs->tight->tight.buffer; \
\
c0 = data[0]; \
i = 1; \
@ -423,9 +423,9 @@ static int tight_fill_palette(VncState *vs, int x, int y,
{
int max;
max = count / tight_conf[vs->tight.compression].idx_max_colors_divisor;
max = count / tight_conf[vs->tight->compression].idx_max_colors_divisor;
if (max < 2 &&
count >= tight_conf[vs->tight.compression].mono_min_rect_size) {
count >= tight_conf[vs->tight->compression].mono_min_rect_size) {
max = 2;
}
if (max >= 256) {
@ -558,7 +558,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
int x, y, c;
buf32 = (uint32_t *)buf;
memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int));
if (1 /* FIXME */) {
shift[0] = vs->client_pf.rshift;
@ -575,7 +575,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
upper[c] = 0;
here[c] = 0;
}
prev = (int *)vs->tight.gradient.buffer;
prev = (int *)vs->tight->gradient.buffer;
for (x = 0; x < w; x++) {
pix32 = *buf32++;
for (c = 0; c < 3; c++) {
@ -615,7 +615,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
int prediction; \
int x, y, c; \
\
memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int)); \
memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int)); \
\
endian = 0; /* FIXME */ \
\
@ -631,7 +631,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
upper[c] = 0; \
here[c] = 0; \
} \
prev = (int *)vs->tight.gradient.buffer; \
prev = (int *)vs->tight->gradient.buffer; \
for (x = 0; x < w; x++) { \
pix = *buf; \
if (endian) { \
@ -785,7 +785,7 @@ static void extend_solid_area(VncState *vs, int x, int y, int w, int h,
static int tight_init_stream(VncState *vs, int stream_id,
int level, int strategy)
{
z_streamp zstream = &vs->tight.stream[stream_id];
z_streamp zstream = &vs->tight->stream[stream_id];
if (zstream->opaque == NULL) {
int err;
@ -803,15 +803,15 @@ static int tight_init_stream(VncState *vs, int stream_id,
return -1;
}
vs->tight.levels[stream_id] = level;
vs->tight->levels[stream_id] = level;
zstream->opaque = vs;
}
if (vs->tight.levels[stream_id] != level) {
if (vs->tight->levels[stream_id] != level) {
if (deflateParams(zstream, level, strategy) != Z_OK) {
return -1;
}
vs->tight.levels[stream_id] = level;
vs->tight->levels[stream_id] = level;
}
return 0;
}
@ -839,11 +839,11 @@ static void tight_send_compact_size(VncState *vs, size_t len)
static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
int level, int strategy)
{
z_streamp zstream = &vs->tight.stream[stream_id];
z_streamp zstream = &vs->tight->stream[stream_id];
int previous_out;
if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) {
vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset);
vnc_write(vs, vs->tight->tight.buffer, vs->tight->tight.offset);
return bytes;
}
@ -852,13 +852,13 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
}
/* reserve memory in output buffer */
buffer_reserve(&vs->tight.zlib, bytes + 64);
buffer_reserve(&vs->tight->zlib, bytes + 64);
/* set pointers */
zstream->next_in = vs->tight.tight.buffer;
zstream->avail_in = vs->tight.tight.offset;
zstream->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset;
zstream->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset;
zstream->next_in = vs->tight->tight.buffer;
zstream->avail_in = vs->tight->tight.offset;
zstream->next_out = vs->tight->zlib.buffer + vs->tight->zlib.offset;
zstream->avail_out = vs->tight->zlib.capacity - vs->tight->zlib.offset;
previous_out = zstream->avail_out;
zstream->data_type = Z_BINARY;
@ -868,14 +868,14 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
return -1;
}
vs->tight.zlib.offset = vs->tight.zlib.capacity - zstream->avail_out;
vs->tight->zlib.offset = vs->tight->zlib.capacity - zstream->avail_out;
/* ...how much data has actually been produced by deflate() */
bytes = previous_out - zstream->avail_out;
tight_send_compact_size(vs, bytes);
vnc_write(vs, vs->tight.zlib.buffer, bytes);
vnc_write(vs, vs->tight->zlib.buffer, bytes);
buffer_reset(&vs->tight.zlib);
buffer_reset(&vs->tight->zlib);
return bytes;
}
@ -927,16 +927,17 @@ static int send_full_color_rect(VncState *vs, int x, int y, int w, int h)
vnc_write_u8(vs, stream << 4); /* no flushing, no filter */
if (vs->tight.pixel24) {
tight_pack24(vs, vs->tight.tight.buffer, w * h, &vs->tight.tight.offset);
if (vs->tight->pixel24) {
tight_pack24(vs, vs->tight->tight.buffer, w * h,
&vs->tight->tight.offset);
bytes = 3;
} else {
bytes = vs->client_pf.bytes_per_pixel;
}
bytes = tight_compress_data(vs, stream, w * h * bytes,
tight_conf[vs->tight.compression].raw_zlib_level,
Z_DEFAULT_STRATEGY);
tight_conf[vs->tight->compression].raw_zlib_level,
Z_DEFAULT_STRATEGY);
return (bytes >= 0);
}
@ -947,14 +948,14 @@ static int send_solid_rect(VncState *vs)
vnc_write_u8(vs, VNC_TIGHT_FILL << 4); /* no flushing, no filter */
if (vs->tight.pixel24) {
tight_pack24(vs, vs->tight.tight.buffer, 1, &vs->tight.tight.offset);
if (vs->tight->pixel24) {
tight_pack24(vs, vs->tight->tight.buffer, 1, &vs->tight->tight.offset);
bytes = 3;
} else {
bytes = vs->client_pf.bytes_per_pixel;
}
vnc_write(vs, vs->tight.tight.buffer, bytes);
vnc_write(vs, vs->tight->tight.buffer, bytes);
return 1;
}
@ -963,7 +964,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
{
ssize_t bytes;
int stream = 1;
int level = tight_conf[vs->tight.compression].mono_zlib_level;
int level = tight_conf[vs->tight->compression].mono_zlib_level;
#ifdef CONFIG_VNC_PNG
if (tight_can_send_png_rect(vs, w, h)) {
@ -991,26 +992,26 @@ static int send_mono_rect(VncState *vs, int x, int y,
uint32_t buf[2] = {bg, fg};
size_t ret = sizeof (buf);
if (vs->tight.pixel24) {
if (vs->tight->pixel24) {
tight_pack24(vs, (unsigned char*)buf, 2, &ret);
}
vnc_write(vs, buf, ret);
tight_encode_mono_rect32(vs->tight.tight.buffer, w, h, bg, fg);
tight_encode_mono_rect32(vs->tight->tight.buffer, w, h, bg, fg);
break;
}
case 2:
vnc_write(vs, &bg, 2);
vnc_write(vs, &fg, 2);
tight_encode_mono_rect16(vs->tight.tight.buffer, w, h, bg, fg);
tight_encode_mono_rect16(vs->tight->tight.buffer, w, h, bg, fg);
break;
default:
vnc_write_u8(vs, bg);
vnc_write_u8(vs, fg);
tight_encode_mono_rect8(vs->tight.tight.buffer, w, h, bg, fg);
tight_encode_mono_rect8(vs->tight->tight.buffer, w, h, bg, fg);
break;
}
vs->tight.tight.offset = bytes;
vs->tight->tight.offset = bytes;
bytes = tight_compress_data(vs, stream, bytes, level, Z_DEFAULT_STRATEGY);
return (bytes >= 0);
@ -1040,7 +1041,7 @@ static void write_palette(int idx, uint32_t color, void *opaque)
static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
{
int stream = 3;
int level = tight_conf[vs->tight.compression].gradient_zlib_level;
int level = tight_conf[vs->tight->compression].gradient_zlib_level;
ssize_t bytes;
if (vs->client_pf.bytes_per_pixel == 1) {
@ -1050,23 +1051,23 @@ static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT);
buffer_reserve(&vs->tight.gradient, w * 3 * sizeof (int));
buffer_reserve(&vs->tight->gradient, w * 3 * sizeof(int));
if (vs->tight.pixel24) {
tight_filter_gradient24(vs, vs->tight.tight.buffer, w, h);
if (vs->tight->pixel24) {
tight_filter_gradient24(vs, vs->tight->tight.buffer, w, h);
bytes = 3;
} else if (vs->client_pf.bytes_per_pixel == 4) {
tight_filter_gradient32(vs, (uint32_t *)vs->tight.tight.buffer, w, h);
tight_filter_gradient32(vs, (uint32_t *)vs->tight->tight.buffer, w, h);
bytes = 4;
} else {
tight_filter_gradient16(vs, (uint16_t *)vs->tight.tight.buffer, w, h);
tight_filter_gradient16(vs, (uint16_t *)vs->tight->tight.buffer, w, h);
bytes = 2;
}
buffer_reset(&vs->tight.gradient);
buffer_reset(&vs->tight->gradient);
bytes = w * h * bytes;
vs->tight.tight.offset = bytes;
vs->tight->tight.offset = bytes;
bytes = tight_compress_data(vs, stream, bytes,
level, Z_FILTERED);
@ -1077,7 +1078,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
int w, int h, VncPalette *palette)
{
int stream = 2;
int level = tight_conf[vs->tight.compression].idx_zlib_level;
int level = tight_conf[vs->tight->compression].idx_zlib_level;
int colors;
ssize_t bytes;
@ -1104,12 +1105,12 @@ static int send_palette_rect(VncState *vs, int x, int y,
palette_iter(palette, write_palette, &priv);
vnc_write(vs, header, sizeof(header));
if (vs->tight.pixel24) {
if (vs->tight->pixel24) {
tight_pack24(vs, vs->output.buffer + old_offset, colors, &offset);
vs->output.offset = old_offset + offset;
}
tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h, palette);
break;
}
case 2:
@ -1119,7 +1120,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
palette_iter(palette, write_palette, &priv);
vnc_write(vs, header, sizeof(header));
tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h, palette);
break;
}
default:
@ -1127,7 +1128,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
break;
}
bytes = w * h;
vs->tight.tight.offset = bytes;
vs->tight->tight.offset = bytes;
bytes = tight_compress_data(vs, stream, bytes,
level, Z_DEFAULT_STRATEGY);
@ -1146,7 +1147,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
static void jpeg_init_destination(j_compress_ptr cinfo)
{
VncState *vs = cinfo->client_data;
Buffer *buffer = &vs->tight.jpeg;
Buffer *buffer = &vs->tight->jpeg;
cinfo->dest->next_output_byte = (JOCTET *)buffer->buffer + buffer->offset;
cinfo->dest->free_in_buffer = (size_t)(buffer->capacity - buffer->offset);
@ -1156,7 +1157,7 @@ static void jpeg_init_destination(j_compress_ptr cinfo)
static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo)
{
VncState *vs = cinfo->client_data;
Buffer *buffer = &vs->tight.jpeg;
Buffer *buffer = &vs->tight->jpeg;
buffer->offset = buffer->capacity;
buffer_reserve(buffer, 2048);
@ -1168,7 +1169,7 @@ static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo)
static void jpeg_term_destination(j_compress_ptr cinfo)
{
VncState *vs = cinfo->client_data;
Buffer *buffer = &vs->tight.jpeg;
Buffer *buffer = &vs->tight->jpeg;
buffer->offset = buffer->capacity - cinfo->dest->free_in_buffer;
}
@ -1187,7 +1188,7 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
return send_full_color_rect(vs, x, y, w, h);
}
buffer_reserve(&vs->tight.jpeg, 2048);
buffer_reserve(&vs->tight->jpeg, 2048);
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_compress(&cinfo);
@ -1222,9 +1223,9 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
vnc_write_u8(vs, VNC_TIGHT_JPEG << 4);
tight_send_compact_size(vs, vs->tight.jpeg.offset);
vnc_write(vs, vs->tight.jpeg.buffer, vs->tight.jpeg.offset);
buffer_reset(&vs->tight.jpeg);
tight_send_compact_size(vs, vs->tight->jpeg.offset);
vnc_write(vs, vs->tight->jpeg.buffer, vs->tight->jpeg.offset);
buffer_reset(&vs->tight->jpeg);
return 1;
}
@ -1240,7 +1241,7 @@ static void write_png_palette(int idx, uint32_t pix, void *opaque)
VncState *vs = priv->vs;
png_colorp color = &priv->png_palette[idx];
if (vs->tight.pixel24)
if (vs->tight->pixel24)
{
color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
@ -1267,10 +1268,10 @@ static void png_write_data(png_structp png_ptr, png_bytep data,
{
VncState *vs = png_get_io_ptr(png_ptr);
buffer_reserve(&vs->tight.png, vs->tight.png.offset + length);
memcpy(vs->tight.png.buffer + vs->tight.png.offset, data, length);
buffer_reserve(&vs->tight->png, vs->tight->png.offset + length);
memcpy(vs->tight->png.buffer + vs->tight->png.offset, data, length);
vs->tight.png.offset += length;
vs->tight->png.offset += length;
}
static void png_flush_data(png_structp png_ptr)
@ -1295,8 +1296,8 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
png_infop info_ptr;
png_colorp png_palette = NULL;
pixman_image_t *linebuf;
int level = tight_png_conf[vs->tight.compression].png_zlib_level;
int filters = tight_png_conf[vs->tight.compression].png_filters;
int level = tight_png_conf[vs->tight->compression].png_zlib_level;
int filters = tight_png_conf[vs->tight->compression].png_filters;
uint8_t *buf;
int dy;
@ -1340,21 +1341,23 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
if (vs->client_pf.bytes_per_pixel == 4) {
tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h,
palette);
} else {
tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h,
palette);
}
}
png_write_info(png_ptr, info_ptr);
buffer_reserve(&vs->tight.png, 2048);
buffer_reserve(&vs->tight->png, 2048);
linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
buf = (uint8_t *)pixman_image_get_data(linebuf);
for (dy = 0; dy < h; dy++)
{
if (color_type == PNG_COLOR_TYPE_PALETTE) {
memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
memcpy(buf, vs->tight->tight.buffer + (dy * w), w);
} else {
qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
}
@ -1372,27 +1375,27 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
vnc_write_u8(vs, VNC_TIGHT_PNG << 4);
tight_send_compact_size(vs, vs->tight.png.offset);
vnc_write(vs, vs->tight.png.buffer, vs->tight.png.offset);
buffer_reset(&vs->tight.png);
tight_send_compact_size(vs, vs->tight->png.offset);
vnc_write(vs, vs->tight->png.buffer, vs->tight->png.offset);
buffer_reset(&vs->tight->png);
return 1;
}
#endif /* CONFIG_VNC_PNG */
static void vnc_tight_start(VncState *vs)
{
buffer_reset(&vs->tight.tight);
buffer_reset(&vs->tight->tight);
// make the output buffer be the zlib buffer, so we can compress it later
vs->tight.tmp = vs->output;
vs->output = vs->tight.tight;
vs->tight->tmp = vs->output;
vs->output = vs->tight->tight;
}
static void vnc_tight_stop(VncState *vs)
{
// switch back to normal output/zlib buffers
vs->tight.tight = vs->output;
vs->output = vs->tight.tmp;
vs->tight->tight = vs->output;
vs->output = vs->tight->tmp;
}
static int send_sub_rect_nojpeg(VncState *vs, int x, int y, int w, int h,
@ -1426,9 +1429,9 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
int ret;
if (colors == 0) {
if (force || (tight_jpeg_conf[vs->tight.quality].jpeg_full &&
if (force || (tight_jpeg_conf[vs->tight->quality].jpeg_full &&
tight_detect_smooth_image(vs, w, h))) {
int quality = tight_conf[vs->tight.quality].jpeg_quality;
int quality = tight_conf[vs->tight->quality].jpeg_quality;
ret = send_jpeg_rect(vs, x, y, w, h, quality);
} else {
@ -1440,9 +1443,9 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
ret = send_mono_rect(vs, x, y, w, h, bg, fg);
} else if (colors <= 256) {
if (force || (colors > 96 &&
tight_jpeg_conf[vs->tight.quality].jpeg_idx &&
tight_jpeg_conf[vs->tight->quality].jpeg_idx &&
tight_detect_smooth_image(vs, w, h))) {
int quality = tight_conf[vs->tight.quality].jpeg_quality;
int quality = tight_conf[vs->tight->quality].jpeg_quality;
ret = send_jpeg_rect(vs, x, y, w, h, quality);
} else {
@ -1480,20 +1483,20 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
}
vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type);
vnc_tight_start(vs);
vnc_raw_send_framebuffer_update(vs, x, y, w, h);
vnc_tight_stop(vs);
#ifdef CONFIG_VNC_JPEG
if (!vs->vd->non_adaptive && vs->tight.quality != (uint8_t)-1) {
if (!vs->vd->non_adaptive && vs->tight->quality != (uint8_t)-1) {
double freq = vnc_update_freq(vs, x, y, w, h);
if (freq < tight_jpeg_conf[vs->tight.quality].jpeg_freq_min) {
if (freq < tight_jpeg_conf[vs->tight->quality].jpeg_freq_min) {
allow_jpeg = false;
}
if (freq >= tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) {
if (freq >= tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) {
force_jpeg = true;
vnc_sent_lossy_rect(vs, x, y, w, h);
}
@ -1503,7 +1506,7 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette);
#ifdef CONFIG_VNC_JPEG
if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
if (allow_jpeg && vs->tight->quality != (uint8_t)-1) {
ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
color_count_palette, force_jpeg);
} else {
@ -1520,7 +1523,7 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
static int send_sub_rect_solid(VncState *vs, int x, int y, int w, int h)
{
vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type);
vnc_tight_start(vs);
vnc_raw_send_framebuffer_update(vs, x, y, w, h);
@ -1538,8 +1541,8 @@ static int send_rect_simple(VncState *vs, int x, int y, int w, int h,
int rw, rh;
int n = 0;
max_size = tight_conf[vs->tight.compression].max_rect_size;
max_width = tight_conf[vs->tight.compression].max_rect_width;
max_size = tight_conf[vs->tight->compression].max_rect_size;
max_width = tight_conf[vs->tight->compression].max_rect_width;
if (split && (w > max_width || w * h > max_size)) {
max_sub_width = (w > max_width) ? max_width : w;
@ -1648,16 +1651,16 @@ static int tight_send_framebuffer_update(VncState *vs, int x, int y,
if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF &&
vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) {
vs->tight.pixel24 = true;
vs->tight->pixel24 = true;
} else {
vs->tight.pixel24 = false;
vs->tight->pixel24 = false;
}
#ifdef CONFIG_VNC_JPEG
if (vs->tight.quality != (uint8_t)-1) {
if (vs->tight->quality != (uint8_t)-1) {
double freq = vnc_update_freq(vs, x, y, w, h);
if (freq > tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) {
if (freq > tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) {
return send_rect_simple(vs, x, y, w, h, false);
}
}
@ -1669,8 +1672,8 @@ static int tight_send_framebuffer_update(VncState *vs, int x, int y,
/* Calculate maximum number of rows in one non-solid rectangle. */
max_rows = tight_conf[vs->tight.compression].max_rect_size;
max_rows /= MIN(tight_conf[vs->tight.compression].max_rect_width, w);
max_rows = tight_conf[vs->tight->compression].max_rect_size;
max_rows /= MIN(tight_conf[vs->tight->compression].max_rect_width, w);
return find_large_solid_color_rect(vs, x, y, w, h, max_rows);
}
@ -1678,33 +1681,33 @@ static int tight_send_framebuffer_update(VncState *vs, int x, int y,
int vnc_tight_send_framebuffer_update(VncState *vs, int x, int y,
int w, int h)
{
vs->tight.type = VNC_ENCODING_TIGHT;
vs->tight->type = VNC_ENCODING_TIGHT;
return tight_send_framebuffer_update(vs, x, y, w, h);
}
int vnc_tight_png_send_framebuffer_update(VncState *vs, int x, int y,
int w, int h)
{
vs->tight.type = VNC_ENCODING_TIGHT_PNG;
vs->tight->type = VNC_ENCODING_TIGHT_PNG;
return tight_send_framebuffer_update(vs, x, y, w, h);
}
void vnc_tight_clear(VncState *vs)
{
int i;
for (i=0; i<ARRAY_SIZE(vs->tight.stream); i++) {
if (vs->tight.stream[i].opaque) {
deflateEnd(&vs->tight.stream[i]);
for (i = 0; i < ARRAY_SIZE(vs->tight->stream); i++) {
if (vs->tight->stream[i].opaque) {
deflateEnd(&vs->tight->stream[i]);
}
}
buffer_free(&vs->tight.tight);
buffer_free(&vs->tight.zlib);
buffer_free(&vs->tight.gradient);
buffer_free(&vs->tight->tight);
buffer_free(&vs->tight->zlib);
buffer_free(&vs->tight->gradient);
#ifdef CONFIG_VNC_JPEG
buffer_free(&vs->tight.jpeg);
buffer_free(&vs->tight->jpeg);
#endif
#ifdef CONFIG_VNC_PNG
buffer_free(&vs->tight.png);
buffer_free(&vs->tight->png);
#endif
}

11
ui/vnc-enc-zlib.c
View File

@ -76,7 +76,8 @@ static int vnc_zlib_stop(VncState *vs)
zstream->zalloc = vnc_zlib_zalloc;
zstream->zfree = vnc_zlib_zfree;
err = deflateInit2(zstream, vs->tight.compression, Z_DEFLATED, MAX_WBITS,
err = deflateInit2(zstream, vs->tight->compression, Z_DEFLATED,
MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (err != Z_OK) {
@ -84,16 +85,16 @@ static int vnc_zlib_stop(VncState *vs)
return -1;
}
vs->zlib.level = vs->tight.compression;
vs->zlib.level = vs->tight->compression;
zstream->opaque = vs;
}
if (vs->tight.compression != vs->zlib.level) {
if (deflateParams(zstream, vs->tight.compression,
if (vs->tight->compression != vs->zlib.level) {
if (deflateParams(zstream, vs->tight->compression,
Z_DEFAULT_STRATEGY) != Z_OK) {
return -1;
}
vs->zlib.level = vs->tight.compression;
vs->zlib.level = vs->tight->compression;
}
// reserve memory in output buffer

68
ui/vnc-enc-zrle.c
View File

@ -37,18 +37,18 @@ static const int bits_per_packed_pixel[] = {
static void vnc_zrle_start(VncState *vs)
{
buffer_reset(&vs->zrle.zrle);
buffer_reset(&vs->zrle->zrle);
/* make the output buffer be the zlib buffer, so we can compress it later */
vs->zrle.tmp = vs->output;
vs->output = vs->zrle.zrle;
vs->zrle->tmp = vs->output;
vs->output = vs->zrle->zrle;
}
static void vnc_zrle_stop(VncState *vs)
{
/* switch back to normal output/zlib buffers */
vs->zrle.zrle = vs->output;
vs->output = vs->zrle.tmp;
vs->zrle->zrle = vs->output;
vs->output = vs->zrle->tmp;
}
static void *zrle_convert_fb(VncState *vs, int x, int y, int w, int h,
@ -56,24 +56,24 @@ static void *zrle_convert_fb(VncState *vs, int x, int y, int w, int h,
{
Buffer tmp;
buffer_reset(&vs->zrle.fb);
buffer_reserve(&vs->zrle.fb, w * h * bpp + bpp);
buffer_reset(&vs->zrle->fb);
buffer_reserve(&vs->zrle->fb, w * h * bpp + bpp);
tmp = vs->output;
vs->output = vs->zrle.fb;
vs->output = vs->zrle->fb;
vnc_raw_send_framebuffer_update(vs, x, y, w, h);
vs->zrle.fb = vs->output;
vs->zrle->fb = vs->output;
vs->output = tmp;
return vs->zrle.fb.buffer;
return vs->zrle->fb.buffer;
}
static int zrle_compress_data(VncState *vs, int level)
{
z_streamp zstream = &vs->zrle.stream;
z_streamp zstream = &vs->zrle->stream;
buffer_reset(&vs->zrle.zlib);
buffer_reset(&vs->zrle->zlib);
if (zstream->opaque != vs) {
int err;
@ -93,13 +93,13 @@ static int zrle_compress_data(VncState *vs, int level)
}
/* reserve memory in output buffer */
buffer_reserve(&vs->zrle.zlib, vs->zrle.zrle.offset + 64);
buffer_reserve(&vs->zrle->zlib, vs->zrle->zrle.offset + 64);
/* set pointers */
zstream->next_in = vs->zrle.zrle.buffer;
zstream->avail_in = vs->zrle.zrle.offset;
zstream->next_out = vs->zrle.zlib.buffer + vs->zrle.zlib.offset;
zstream->avail_out = vs->zrle.zlib.capacity - vs->zrle.zlib.offset;
zstream->next_in = vs->zrle->zrle.buffer;
zstream->avail_in = vs->zrle->zrle.offset;
zstream->next_out = vs->zrle->zlib.buffer + vs->zrle->zlib.offset;
zstream->avail_out = vs->zrle->zlib.capacity - vs->zrle->zlib.offset;
zstream->data_type = Z_BINARY;
/* start encoding */
@ -108,8 +108,8 @@ static int zrle_compress_data(VncState *vs, int level)
return -1;
}
vs->zrle.zlib.offset = vs->zrle.zlib.capacity - zstream->avail_out;
return vs->zrle.zlib.offset;
vs->zrle->zlib.offset = vs->zrle->zlib.capacity - zstream->avail_out;
return vs->zrle->zlib.offset;
}
/* Try to work out whether to use RLE and/or a palette. We do this by
@ -259,14 +259,14 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
size_t bytes;
int zywrle_level;
if (vs->zrle.type == VNC_ENCODING_ZYWRLE) {
if (!vs->vd->lossy || vs->tight.quality == (uint8_t)-1
|| vs->tight.quality == 9) {
if (vs->zrle->type == VNC_ENCODING_ZYWRLE) {
if (!vs->vd->lossy || vs->tight->quality == (uint8_t)-1
|| vs->tight->quality == 9) {
zywrle_level = 0;
vs->zrle.type = VNC_ENCODING_ZRLE;
} else if (vs->tight.quality < 3) {
vs->zrle->type = VNC_ENCODING_ZRLE;
} else if (vs->tight->quality < 3) {
zywrle_level = 3;
} else if (vs->tight.quality < 6) {
} else if (vs->tight->quality < 6) {
zywrle_level = 2;
} else {
zywrle_level = 1;
@ -337,30 +337,30 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
vnc_zrle_stop(vs);
bytes = zrle_compress_data(vs, Z_DEFAULT_COMPRESSION);
vnc_framebuffer_update(vs, x, y, w, h, vs->zrle.type);
vnc_framebuffer_update(vs, x, y, w, h, vs->zrle->type);
vnc_write_u32(vs, bytes);
vnc_write(vs, vs->zrle.zlib.buffer, vs->zrle.zlib.offset);
vnc_write(vs, vs->zrle->zlib.buffer, vs->zrle->zlib.offset);
return 1;
}
int vnc_zrle_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
{
vs->zrle.type = VNC_ENCODING_ZRLE;
vs->zrle->type = VNC_ENCODING_ZRLE;
return zrle_send_framebuffer_update(vs, x, y, w, h);
}
int vnc_zywrle_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
{
vs->zrle.type = VNC_ENCODING_ZYWRLE;
vs->zrle->type = VNC_ENCODING_ZYWRLE;
return zrle_send_framebuffer_update(vs, x, y, w, h);
}
void vnc_zrle_clear(VncState *vs)
{
if (vs->zrle.stream.opaque) {
deflateEnd(&vs->zrle.stream);
if (vs->zrle->stream.opaque) {
deflateEnd(&vs->zrle->stream);
}
buffer_free(&vs->zrle.zrle);
buffer_free(&vs->zrle.fb);
buffer_free(&vs->zrle.zlib);
buffer_free(&vs->zrle->zrle);
buffer_free(&vs->zrle->fb);
buffer_free(&vs->zrle->zlib);
}

2
ui/vnc-enc-zrle.inc.c
View File

@ -96,7 +96,7 @@ static void ZRLE_ENCODE(VncState *vs, int x, int y, int w, int h,
static void ZRLE_ENCODE_TILE(VncState *vs, ZRLE_PIXEL *data, int w, int h,
int zywrle_level)
{
VncPalette *palette = &vs->zrle.palette;
VncPalette *palette = &vs->zrle->palette;
int runs = 0;
int single_pixels = 0;

28
ui/vnc.c
View File

@ -1307,6 +1307,8 @@ void vnc_disconnect_finish(VncState *vs)
object_unref(OBJECT(vs->sioc));
vs->sioc = NULL;
vs->magic = 0;
g_free(vs->zrle);
g_free(vs->tight);
g_free(vs);
}
@ -2058,8 +2060,8 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
vs->features = 0;
vs->vnc_encoding = 0;
vs->tight.compression = 9;
vs->tight.quality = -1; /* Lossless by default */
vs->tight->compression = 9;
vs->tight->quality = -1; /* Lossless by default */
vs->absolute = -1;
/*
@ -2127,11 +2129,11 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
vs->features |= VNC_FEATURE_LED_STATE_MASK;
break;
case VNC_ENCODING_COMPRESSLEVEL0 ... VNC_ENCODING_COMPRESSLEVEL0 + 9:
vs->tight.compression = (enc & 0x0F);
vs->tight->compression = (enc & 0x0F);
break;
case VNC_ENCODING_QUALITYLEVEL0 ... VNC_ENCODING_QUALITYLEVEL0 + 9:
if (vs->vd->lossy) {
vs->tight.quality = (enc & 0x0F);
vs->tight->quality = (enc & 0x0F);
}
break;
default:
@ -3034,6 +3036,8 @@ static void vnc_connect(VncDisplay *vd, QIOChannelSocket *sioc,
int i;
trace_vnc_client_connect(vs, sioc);
vs->zrle = g_new0(VncZrle, 1);
vs->tight = g_new0(VncTight, 1);
vs->magic = VNC_MAGIC;
vs->sioc = sioc;
object_ref(OBJECT(vs->sioc));
@ -3045,19 +3049,19 @@ static void vnc_connect(VncDisplay *vd, QIOChannelSocket *sioc,
buffer_init(&vs->output, "vnc-output/%p", sioc);
buffer_init(&vs->jobs_buffer, "vnc-jobs_buffer/%p", sioc);
buffer_init(&vs->tight.tight, "vnc-tight/%p", sioc);
buffer_init(&vs->tight.zlib, "vnc-tight-zlib/%p", sioc);
buffer_init(&vs->tight.gradient, "vnc-tight-gradient/%p", sioc);
buffer_init(&vs->tight->tight, "vnc-tight/%p", sioc);
buffer_init(&vs->tight->zlib, "vnc-tight-zlib/%p", sioc);
buffer_init(&vs->tight->gradient, "vnc-tight-gradient/%p", sioc);
#ifdef CONFIG_VNC_JPEG
buffer_init(&vs->tight.jpeg, "vnc-tight-jpeg/%p", sioc);
buffer_init(&vs->tight->jpeg, "vnc-tight-jpeg/%p", sioc);
#endif
#ifdef CONFIG_VNC_PNG
buffer_init(&vs->tight.png, "vnc-tight-png/%p", sioc);
buffer_init(&vs->tight->png, "vnc-tight-png/%p", sioc);
#endif
buffer_init(&vs->zlib.zlib, "vnc-zlib/%p", sioc);
buffer_init(&vs->zrle.zrle, "vnc-zrle/%p", sioc);
buffer_init(&vs->zrle.fb, "vnc-zrle-fb/%p", sioc);
buffer_init(&vs->zrle.zlib, "vnc-zrle-zlib/%p", sioc);
buffer_init(&vs->zrle->zrle, "vnc-zrle/%p", sioc);
buffer_init(&vs->zrle->fb, "vnc-zrle-fb/%p", sioc);
buffer_init(&vs->zrle->zlib, "vnc-zrle-zlib/%p", sioc);
if (skipauth) {
vs->auth = VNC_AUTH_NONE;

4
ui/vnc.h
View File

@ -338,10 +338,10 @@ struct VncState
/* Encoding specific, if you add something here, don't forget to
* update vnc_async_encoding_start()
*/
VncTight tight;
VncTight *tight;
VncZlib zlib;
VncHextile hextile;
VncZrle zrle;
VncZrle *zrle;
VncZywrle zywrle;
Notifier mouse_mode_notifier;

Write Preview
Loading…
Cancel
Save