Merge pull request #82 from glyptodon/experimental

GUAC-240 / GUAC-1290: Merge experimental JPEG streaming changes

src/common/guac_surface.c

@@ -29,9 +29,11 @@
 #include <guacamole/layer.h>
 #include <guacamole/protocol.h>
 #include <guacamole/socket.h>
+#include <guacamole/timestamp.h>
 
 #include <stdlib.h>
 #include <stdint.h>
+#include <string.h>
 
 /**
  * The width of an update which should be considered negible and thus
@@ -77,6 +79,18 @@
 #define cairo_format_stride_for_width(format, width) (width*4)
 #endif
 
+/**
+ * The JPEG image quality ('quantization') setting to use. Range 0-100 where
+ * 100 is the highest quality/largest file size, and 0 is the lowest
+ * quality/smallest file size.
+ */
+#define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90
+
+/**
+ * The framerate which, if exceeded, indicates that JPEG is preferred.
+ */
+#define GUAC_COMMON_SURFACE_JPEG_FRAMERATE 3
+
 /**
  * Updates the coordinates of the given rectangle to be within the bounds of
  * the given surface.
@@ -222,22 +236,252 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co
 }
 
 /**
- * Flushes the PNG update currently described by the dirty rectangle within the
+ * Calculate the current average framerate for a given area on the surface.
- * given surface to that surface's PNG queue. There MUST be space within the
+ *
+ * @param surface
+ *     The surface on which the framerate will be calculated.
+ *
+ * @param rect
+ *     The rect containing the area for which the average framerate will be 
+ *     calculated.
+ *
+ * @return
+ *     The average framerate of the given area, in frames per second.
+ */
+static unsigned int __guac_common_surface_calculate_framerate(
+        guac_common_surface* surface, const guac_common_rect* rect) {
+
+    int x, y;
+
+    /* Calculate minimum X/Y coordinates intersecting given rect */
+    int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+    int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+
+    /* Calculate maximum X/Y coordinates intersecting given rect */
+    int max_x = min_x + (rect->width  - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+    int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+
+    unsigned int sum_framerate = 0;
+    unsigned int count = 0;
+
+    /* Get start of buffer at given coordinates */
+    const guac_common_surface_heat_cell* heat_row =
+        surface->heat_map + min_y * surface->width + min_x;
+
+    /* Iterate over all the heat map cells for the area
+     * and calculate the average framerate */
+    for (y = min_y; y < max_y; y++) {
+
+        /* Get current row of heat map */
+        const guac_common_surface_heat_cell* heat_cell = heat_row;
+
+        /* For each cell in subset of row */
+        for (x = min_x; x < max_x; x++) {
+
+            /* Calculate indicies for latest and oldest history entries */
+            int oldest_entry = heat_cell->oldest_entry;
+            int latest_entry = oldest_entry - 1;
+            if (latest_entry < 0)
+                latest_entry = GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE - 1;
+
+            /* Calculate elapsed time covering entire history for this cell */
+            int elapsed_time = heat_cell->history[latest_entry]
+                             - heat_cell->history[oldest_entry];
+
+            /* Calculate and add framerate */
+            if (elapsed_time)
+                sum_framerate += GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE
+                    * 1000 / elapsed_time;
+
+            /* Next heat map cell */
+            heat_cell++;
+            count++;
+
+        }
+
+        /* Next heat map row */
+        heat_row += surface->width;
+
+    }
+
+    /* Calculate the average framerate over entire rect */
+    if (count)
+        return sum_framerate / count;
+
+    return 0;
+
+}
+
+ /**
+ * Guesses whether a rectangle within a particular surface would be better
+ * compressed as PNG or as JPEG. Positive values indicate PNG is likely to
+ * be superior, while negative values indicate JPEG.
+ *
+ * @param surface
+ *     The surface containing the image data to check.
+ *
+ * @param rect
+ *     The rect to check within the given surface.
+ *
+ * @return
+ *     Positive values if PNG compression is likely to perform better than
+ *     JPEG, or negative values if JPEG is likely to perform better than PNG.
+ */
+static int __guac_common_surface_png_optimality(guac_common_surface* surface,
+        const guac_common_rect* rect) {
+
+    int x, y;
+
+    int num_same = 0;
+    int num_different = 1;
+
+    /* Get image/buffer metrics */
+    int width = rect->width;
+    int height = rect->height;
+    int stride = surface->stride;
+
+    /* Get buffer from surface */
+    unsigned char* buffer = surface->buffer + rect->y * stride + rect->x * 4;
+
+    /* For each row */
+    for (y = 0; y < height; y++) {
+
+        uint32_t last_pixel;
+        uint32_t* row = (uint32_t*) buffer;
+
+        /* For each pixel in current row */
+        for (x = 0; x < width; x++) {
+
+            /* Get next pixel */
+            uint32_t current_pixel = *(row++) | 0xFF000000;
+
+            /* Update same/different counts according to pixel value */
+            if (x != 0) {
+                if (current_pixel == last_pixel)
+                    num_same++;
+                else
+                    num_different++;
+            }
+
+            last_pixel = current_pixel;
+
+        }
+
+        /* Advance to next row */
+        buffer += stride;
+
+    }
+
+    /* Return rough approximation of optimality for PNG compression */
+    return 0x100 * num_same / num_different - 0x400;
+
+}
+
+/**
+ * Returns whether the given rectangle would be optimally encoded as JPEG
+ * rather than PNG.
+ *
+ * @param surface
+ *     The surface to be queried.
+ *
+ * @param rect
+ *     The rectangle to check.
+ *
+ * @return
+ *     Non-zero if the rectangle would be optimally encoded as JPEG, zero
+ *     otherwise.
+ */
+static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface,
+        const guac_common_rect* rect) {
+
+    /* Calculate the average framerate for the given rect */
+    int framerate = __guac_common_surface_calculate_framerate(surface, rect);
+
+    /* JPEG is preferred if framerate is high enough */
+    return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE
+        && __guac_common_surface_png_optimality(surface, rect) < 0;
+
+}
+
+/**
+ * Updates the heat map cells which intersect the given rectangle using the
+ * given timestamp. This timestamp, along with timestamps from past updates,
+ * is used to calculate the framerate of each heat cell.
+ *
+ * @param surface
+ *     The surface containing the heat map cells to be updated.
+ *
+ * @param rect
+ *     The rectangle containing the heat map cells to be updated.
+ *
+ * @param time
+ *     The timestamp to use when updating the heat map cells which intersect
+ *     the given rectangle.
+ */
+static void __guac_common_surface_touch_rect(guac_common_surface* surface,
+        guac_common_rect* rect, guac_timestamp time) {
+
+    int x, y;
+
+    /* Calculate minimum X/Y coordinates intersecting given rect */
+    int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+    int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+
+    /* Calculate maximum X/Y coordinates intersecting given rect */
+    int max_x = min_x + (rect->width  - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+    int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
+
+    /* Get start of buffer at given coordinates */
+    guac_common_surface_heat_cell* heat_row =
+        surface->heat_map + min_y * surface->width + min_x;
+
+    /* Update all heat map cells which intersect with rectangle */
+    for (y = min_y; y <= max_y; y++) {
+
+        /* Get current row of heat map */
+        guac_common_surface_heat_cell* heat_cell = heat_row;
+
+        /* For each cell in subset of row */
+        for (x = min_x; x <= max_x; x++) {
+
+            /* Replace oldest entry with new timestamp */
+            heat_cell->history[heat_cell->oldest_entry] = time;
+
+            /* Update to next oldest entry */
+            heat_cell->oldest_entry++;
+            if (heat_cell->oldest_entry >=
+                    GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE)
+                heat_cell->oldest_entry = 0;
+
+            /* Advance to next heat map cell */
+            heat_cell++;
+
+        }
+
+        /* Next heat map row */
+        heat_row += surface->width;
+
+    }
+
+}
+
+/**
+ * Flushes the bitmap update currently described by the dirty rectangle within the
+ * given surface to that surface's bitmap queue. There MUST be space within the
  * queue.
  *
  * @param surface The surface to flush.
  */
 static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) {
 
-    guac_common_surface_png_rect* rect;
+    guac_common_surface_bitmap_rect* rect;
 
     /* Do not flush if not dirty */
     if (!surface->dirty)
         return;
 
     /* Add new rect to queue */
-    rect = &(surface->png_queue[surface->png_queue_length++]);
+    rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]);
     rect->rect = surface->dirty_rect;
     rect->flushed = 0;
 
@@ -254,7 +498,7 @@ void guac_common_surface_flush_deferred(guac_common_surface* surface) {
 
     /* Flush if queue size has reached maximum (space is reserved for the final dirty rect,
      * as guac_common_surface_flush() MAY add an additional rect to the queue */
-    if (surface->png_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
+    if (surface->bitmap_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
         guac_common_surface_flush(surface);
 
     /* Append dirty rect to queue */
@@ -665,19 +909,20 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client,
         guac_socket* socket, const guac_layer* layer, int w, int h) {
 
     /* Init surface */
-    guac_common_surface* surface = malloc(sizeof(guac_common_surface));
+    guac_common_surface* surface = calloc(1, sizeof(guac_common_surface));
     surface->client = client;
     surface->socket = socket;
     surface->layer = layer;
     surface->width = w;
     surface->height = h;
-    surface->dirty = 0;
-    surface->png_queue_length = 0;
 
     /* Create corresponding Cairo surface */
     surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w);
     surface->buffer = calloc(h, surface->stride);
 
+    /* Create corresponding heat map */
+    surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell));
+
     /* Reset clipping rect */
     guac_common_surface_reset_clip(surface);
 
@@ -700,6 +945,7 @@ void guac_common_surface_free(guac_common_surface* surface) {
     if (surface->realized)
         guac_protocol_send_dispose(surface->socket, surface->layer);
 
+    free(surface->heat_map);
     free(surface->buffer);
     free(surface);
 
@@ -736,6 +982,10 @@ void guac_common_surface_resize(guac_common_surface* surface, int w, int h) {
     /* Free old data */
     free(old_buffer);
 
+    /* Allocate completely new heat map (can safely discard old stats) */
+    free(surface->heat_map);
+    surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell));
+
     /* Resize dirty rect to fit new surface dimensions */
     if (surface->dirty) {
         __guac_common_bound_rect(surface, &surface->dirty_rect, NULL, NULL);
@@ -773,6 +1023,10 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_
     if (rect.width <= 0 || rect.height <= 0)
         return;
 
+    /* Update the heat map for the update rectangle. */
+    guac_timestamp time = guac_timestamp_current();
+    __guac_common_surface_touch_rect(surface, &rect, time);
+
     /* Flush if not combining */
     if (!__guac_common_should_combine(surface, &rect, 0))
         guac_common_surface_flush_deferred(surface);
@@ -948,11 +1202,13 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) {
 }
 
 /**
- * Flushes the PNG update currently described by the dirty rectangle within the
+ * Flushes the bitmap update currently described by the dirty rectangle within
- * given surface directly to a "png" instruction, which is sent on the socket
+ * the given surface directly via an "img" instruction as PNG data. The
- * associated with the surface.
+ * resulting instructions will be sent over the socket associated with the
+ * given surface.
  *
- * @param surface The surface to flush.
+ * @param surface
+ *     The surface to flush.
  */
 static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
 
@@ -982,15 +1238,52 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
 }
 
 /**
- * Comparator for instances of guac_common_surface_png_rect, the elements
+ * Flushes the bitmap update currently described by the dirty rectangle within
- * which make up a surface's PNG buffer.
+ * the given surface directly via an "img" instruction as JPEG data. The
+ * resulting instructions will be sent over the socket associated with the
+ * given surface.
+ *
+ * @param surface
+ *     The surface to flush.
+ */
+static void __guac_common_surface_flush_to_jpeg(guac_common_surface* surface) {
+
+    if (surface->dirty) {
+
+        guac_socket* socket = surface->socket;
+        const guac_layer* layer = surface->layer;
+
+        /* Get Cairo surface for specified rect */
+        unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4;
+        cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24,
+                                                                    surface->dirty_rect.width,
+                                                                    surface->dirty_rect.height,
+                                                                    surface->stride);
+
+        /* Send JPEG for rect */
+        guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer,
+                surface->dirty_rect.x, surface->dirty_rect.y, rect,
+                GUAC_SURFACE_JPEG_IMAGE_QUALITY);
+        cairo_surface_destroy(rect);
+        surface->realized = 1;
+
+        /* Surface is no longer dirty */
+        surface->dirty = 0;
+
+    }
+
+}
+
+/**
+ * Comparator for instances of guac_common_surface_bitmap_rect, the elements
+ * which make up a surface's bitmap buffer.
  *
  * @see qsort
  */
-static int __guac_common_surface_png_rect_compare(const void* a, const void* b) {
+static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* b) {
 
-    guac_common_surface_png_rect* ra = (guac_common_surface_png_rect*) a;
+    guac_common_surface_bitmap_rect* ra = (guac_common_surface_bitmap_rect*) a;
-    guac_common_surface_png_rect* rb = (guac_common_surface_png_rect*) b;
+    guac_common_surface_bitmap_rect* rb = (guac_common_surface_bitmap_rect*) b;
 
     /* Order roughly top to bottom, left to right */
     if (ra->rect.y != rb->rect.y) return ra->rect.y - rb->rect.y;
@@ -1006,31 +1299,31 @@ static int __guac_common_surface_png_rect_compare(const void* a, const void* b)
 
 void guac_common_surface_flush(guac_common_surface* surface) {
 
-    guac_common_surface_png_rect* current = surface->png_queue;
+    /* Flush final dirty rectangle to queue. */
+    __guac_common_surface_flush_to_queue(surface);
 
+    guac_common_surface_bitmap_rect* current = surface->bitmap_queue;
     int i, j;
     int original_queue_length;
     int flushed = 0;
 
-    /* Flush final dirty rect to queue */
+    original_queue_length = surface->bitmap_queue_length;
-    __guac_common_surface_flush_to_queue(surface);
-    original_queue_length = surface->png_queue_length;
 
     /* Sort updates to make combination less costly */
-    qsort(surface->png_queue, surface->png_queue_length, sizeof(guac_common_surface_png_rect),
+    qsort(surface->bitmap_queue, surface->bitmap_queue_length, sizeof(guac_common_surface_bitmap_rect),
-          __guac_common_surface_png_rect_compare);
+          __guac_common_surface_bitmap_rect_compare);
 
     /* Flush all rects in queue */
-    for (i=0; i < surface->png_queue_length; i++) {
+    for (i=0; i < surface->bitmap_queue_length; i++) {
 
         /* Get next unflushed candidate */
-        guac_common_surface_png_rect* candidate = current;
+        guac_common_surface_bitmap_rect* candidate = current;
         if (!candidate->flushed) {
 
             int combined = 0;
 
             /* Build up rect as much as possible */
-            for (j=i; j < surface->png_queue_length; j++) {
+            for (j=i; j < surface->bitmap_queue_length; j++) {
 
                 if (!candidate->flushed) {
 
@@ -1054,13 +1347,23 @@ void guac_common_surface_flush(guac_common_surface* surface) {
 
             /* Re-add to queue if there's room and this update was modified or we expect others might be */
             if ((combined > 1 || i < original_queue_length)
-                    && surface->png_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
+                    && surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
                 __guac_common_surface_flush_to_queue(surface);
 
-            /* Flush as PNG otherwise */
+            /* Flush as bitmap otherwise */
-            else {
+            else if (surface->dirty) {
-                if (surface->dirty) flushed++;
+
+                flushed++;
+
+                /* Flush as JPEG if JPEG is preferred */
+                if (__guac_common_surface_should_use_jpeg(surface,
+                            &surface->dirty_rect))
+                    __guac_common_surface_flush_to_jpeg(surface);
+
+                /* Otherwise, use PNG */
+                else
                     __guac_common_surface_flush_to_png(surface);
+
             }
 
         }
@@ -1070,7 +1373,7 @@ void guac_common_surface_flush(guac_common_surface* surface) {
     }
 
     /* Flush complete */
-    surface->png_queue_length = 0;
+    surface->bitmap_queue_length = 0;
 
 }
 

src/common/guac_surface.h

@@ -33,15 +33,51 @@
 #include <guacamole/socket.h>
 
 /**
- * The maximum number of updates to allow within the PNG queue.
+ * The maximum number of updates to allow within the bitmap queue.
  */
 #define GUAC_COMMON_SURFACE_QUEUE_SIZE 256
 
 /**
- * Representation of a PNG update, having a rectangle of image data (stored
+ * Heat map cell size in pixels. Each side of each heat map cell will consist
+ * of this many pixels.
+ */
+#define GUAC_COMMON_SURFACE_HEAT_CELL_SIZE 64
+
+/**
+ * The number of entries to collect within each heat map cell. Collected
+ * history entries are used to determine the framerate of the region associated
+ * with that cell.
+ */
+#define GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE 5
+
+/**
+ * Representation of a cell in the refresh heat map. This cell is used to keep
+ * track of how often an area on a surface is refreshed.
+ */
+typedef struct guac_common_surface_heat_cell {
+
+    /**
+     * Timestamps of each of the last N updates covering the location
+     * associated with this heat map cell. This is used to calculate the
+     * framerate. This array is structured as a ring buffer containing history
+     * entries in chronologically-ascending order, starting at the entry
+     * pointed to by oldest_entry and proceeding through all other entries,
+     * wrapping around if the end of the array is reached.
+     */
+    guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE];
+
+    /**
+     * Index of the oldest entry within the history.
+     */
+    int oldest_entry;
+
+} guac_common_surface_heat_cell;
+
+/**
+ * Representation of a bitmap update, having a rectangle of image data (stored
  * elsewhere) and a flushed/not-flushed state.
  */
-typedef struct guac_common_surface_png_rect {
+typedef struct guac_common_surface_bitmap_rect {
 
     /**
      * Whether this rectangle has been flushed.
@@ -49,11 +85,11 @@ typedef struct guac_common_surface_png_rect {
     int flushed;
 
     /**
-     * The rectangle containing the PNG update.
+     * The rectangle containing the bitmap update.
      */
     guac_common_rect rect;
 
-} guac_common_surface_png_rect;
+} guac_common_surface_bitmap_rect;
 
 /**
  * Surface which backs a Guacamole buffer or layer, automatically
@@ -123,14 +159,20 @@ typedef struct guac_common_surface {
     guac_common_rect clip_rect;
 
     /**
-     * The number of updates in the PNG queue.
+     * The number of updates in the bitmap queue.
      */
-    int png_queue_length;
+    int bitmap_queue_length;
 
     /**
-     * All queued PNG updates.
+     * All queued bitmap updates.
      */
-    guac_common_surface_png_rect png_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
+    guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
+
+    /**
+     * A heat map keeping track of the refresh frequency of
+     * the areas of the screen.
+     */
+    guac_common_surface_heat_cell* heat_map;
 
 } guac_common_surface;
 

src/protocols/vnc/client.h

@@ -53,7 +53,7 @@
  * milliseconds. If the server is silent for at least this amount of time, the
  * frame will be considered finished.
  */
-#define GUAC_VNC_FRAME_TIMEOUT 0
+#define GUAC_VNC_FRAME_TIMEOUT 10
 
 /**
  * The number of milliseconds to wait between connection attempts.