Add zlib encoding support (Alexander Graf)

This patch adds zlib encoding support for VNC. It basically runs
the raw traffic through zlib, providing a pretty good compression
ratio.

Signed-off-by: Alexander Graf <agraf@suse.de>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>


git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@6499 c046a42c-6fe2-441c-8c8c-71466251a162

vnc.c

@@ -29,6 +29,7 @@
 #include "qemu_socket.h"
 #include "qemu-timer.h"
 #include "audio/audio.h"
+#include <zlib.h>
 
 #define VNC_REFRESH_INTERVAL (1000 / 30)
 
@@ -144,6 +145,10 @@ struct VncState
     size_t read_handler_expect;
     /* input */
     uint8_t modifiers_state[256];
+
+    Buffer zlib;
+    Buffer zlib_tmp;
+    z_stream zlib_stream[4];
 };
 
 static VncState *vnc_state; /* needed for info vnc */
@@ -481,9 +486,108 @@ static void send_framebuffer_update_hextile(VncState *vs, int x, int y, int w, i
 
 }
 
+static void vnc_zlib_init(VncState *vs)
+{
+    int i;
+    for (i=0; i<(sizeof(vs->zlib_stream) / sizeof(z_stream)); i++)
+        vs->zlib_stream[i].opaque = NULL;
+}
+
+static void vnc_zlib_start(VncState *vs)
+{
+    buffer_reset(&vs->zlib);
+
+    // make the output buffer be the zlib buffer, so we can compress it later
+    vs->zlib_tmp = vs->output;
+    vs->output = vs->zlib;
+}
+
+static int vnc_zlib_stop(VncState *vs, int stream_id)
+{
+    z_streamp zstream = &vs->zlib_stream[stream_id];
+    int previous_out;
+
+    // switch back to normal output/zlib buffers
+    vs->zlib = vs->output;
+    vs->output = vs->zlib_tmp;
+
+    // compress the zlib buffer
+
+    // initialize the stream
+    // XXX need one stream per session
+    if (zstream->opaque != vs) {
+        int err;
+
+        VNC_DEBUG("VNC: initializing zlib stream %d\n", stream_id);
+        VNC_DEBUG("VNC: opaque = %p | vs = %p\n", zstream->opaque, vs);
+        zstream->zalloc = Z_NULL;
+        zstream->zfree = Z_NULL;
+
+        err = deflateInit2(zstream, vs->tight_compression, Z_DEFLATED, MAX_WBITS,
+                           MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
+
+        if (err != Z_OK) {
+            fprintf(stderr, "VNC: error initializing zlib\n");
+            return -1;
+        }
+
+        zstream->opaque = vs;
+    }
+
+    // XXX what to do if tight_compression changed in between?
+
+    // reserve memory in output buffer
+    buffer_reserve(&vs->output, vs->zlib.offset + 64);
+
+    // set pointers
+    zstream->next_in = vs->zlib.buffer;
+    zstream->avail_in = vs->zlib.offset;
+    zstream->next_out = vs->output.buffer + vs->output.offset;
+    zstream->avail_out = vs->output.capacity - vs->output.offset;
+    zstream->data_type = Z_BINARY;
+    previous_out = zstream->total_out;
+
+    // start encoding
+    if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
+        fprintf(stderr, "VNC: error during zlib compression\n");
+        return -1;
+    }
+
+    vs->output.offset = vs->output.capacity - zstream->avail_out;
+    return zstream->total_out - previous_out;
+}
+
+static void send_framebuffer_update_zlib(VncState *vs, int x, int y, int w, int h)
+{
+    int old_offset, new_offset, bytes_written;
+
+    vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_ZLIB);
+
+    // remember where we put in the follow-up size
+    old_offset = vs->output.offset;
+    vnc_write_s32(vs, 0);
+
+    // compress the stream
+    vnc_zlib_start(vs);
+    send_framebuffer_update_raw(vs, x, y, w, h);
+    bytes_written = vnc_zlib_stop(vs, 0);
+
+    if (bytes_written == -1)
+        return;
+
+    // hack in the size
+    new_offset = vs->output.offset;
+    vs->output.offset = old_offset;
+    vnc_write_u32(vs, bytes_written);
+    vs->output.offset = new_offset;
+}
+
 static void send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     switch(vs->vnc_encoding) {
+	case VNC_ENCODING_ZLIB:
+	    send_framebuffer_update_zlib(vs, x, y, w, h);
+	    break;
 	case VNC_ENCODING_HEXTILE:
 	    vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_HEXTILE);
 	    send_framebuffer_update_hextile(vs, x, y, w, h);
@@ -1169,6 +1273,7 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
     int i;
     unsigned int enc = 0;
 
+    vnc_zlib_init(vs);
     vs->features = 0;
     vs->vnc_encoding = 0;
     vs->tight_compression = 9;
@@ -1189,6 +1294,10 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
             vs->features |= VNC_FEATURE_HEXTILE_MASK;
             vs->vnc_encoding = enc;
             break;
+        case VNC_ENCODING_ZLIB:
+            vs->features |= VNC_FEATURE_ZLIB_MASK;
+            vs->vnc_encoding = enc;
+            break;
         case VNC_ENCODING_DESKTOPRESIZE:
             vs->features |= VNC_FEATURE_RESIZE_MASK;
             break;