maride/guacamole-spice-protocol
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #71 from glyptodon/adaptive-jpeg

Browse Source 
GUAC-240: Implement adaptive JPEG encoding
This commit is contained in:
James Muehlner 2015-08-18 11:31:53 -07:00
commit 22a307f422
2 changed files with 386 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

365
src/common/guac_surface.c
View File

@ -29,9 +29,11 @@
#include <guacamole/layer.h>
#include <guacamole/protocol.h>
#include <guacamole/socket.h>
#include <guacamole/timestamp.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
/**
* The width of an update which should be considered negible and thus
@ -77,6 +79,18 @@
#define cairo_format_stride_for_width(format, width) (width*4)
#endif
/**
* The JPEG image quality ('quantization') setting to use. Range 0-100 where
* 100 is the highest quality/largest file size, and 0 is the lowest
* quality/smallest file size.
*/
#define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90
/**
* The framerate which, if exceeded, indicates that JPEG is preferred.
*/
#define GUAC_COMMON_SURFACE_JPEG_FRAMERATE 3
/**
* Updates the coordinates of the given rectangle to be within the bounds of
* the given surface.
@ -222,22 +236,252 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co
}
/**
* Flushes the PNG update currently described by the dirty rectangle within the
* given surface to that surface's PNG queue. There MUST be space within the
* Calculate the current average framerate for a given area on the surface.
*
* @param surface
* The surface on which the framerate will be calculated.
*
* @param rect
* The rect containing the area for which the average framerate will be
* calculated.
*
* @return
* The average framerate of the given area, in frames per second.
*/
static unsigned int __guac_common_surface_calculate_framerate(
guac_common_surface* surface, const guac_common_rect* rect) {
int x, y;
/* Calculate minimum X/Y coordinates intersecting given rect */
int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
/* Calculate maximum X/Y coordinates intersecting given rect */
int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
unsigned int sum_framerate = 0;
unsigned int count = 0;
/* Get start of buffer at given coordinates */
const guac_common_surface_heat_cell* heat_row =
surface->heat_map + min_y * surface->width + min_x;
/* Iterate over all the heat map cells for the area
* and calculate the average framerate */
for (y = min_y; y < max_y; y++) {
/* Get current row of heat map */
const guac_common_surface_heat_cell* heat_cell = heat_row;
/* For each cell in subset of row */
for (x = min_x; x < max_x; x++) {
/* Calculate indicies for latest and oldest history entries */
int oldest_entry = heat_cell->oldest_entry;
int latest_entry = oldest_entry - 1;
if (latest_entry < 0)
latest_entry = GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE - 1;
/* Calculate elapsed time covering entire history for this cell */
int elapsed_time = heat_cell->history[latest_entry]
- heat_cell->history[oldest_entry];
/* Calculate and add framerate */
if (elapsed_time)
sum_framerate += GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE
* 1000 / elapsed_time;
/* Next heat map cell */
heat_cell++;
count++;
}
/* Next heat map row */
heat_row += surface->width;
}
/* Calculate the average framerate over entire rect */
if (count)
return sum_framerate / count;
return 0;
}
/**
* Guesses whether a rectangle within a particular surface would be better
* compressed as PNG or as JPEG. Positive values indicate PNG is likely to
* be superior, while negative values indicate JPEG.
*
* @param surface
* The surface containing the image data to check.
*
* @param rect
* The rect to check within the given surface.
*
* @return
* Positive values if PNG compression is likely to perform better than
* JPEG, or negative values if JPEG is likely to perform better than PNG.
*/
static int __guac_common_surface_png_optimality(guac_common_surface* surface,
const guac_common_rect* rect) {
int x, y;
int num_same = 0;
int num_different = 1;
/* Get image/buffer metrics */
int width = rect->width;
int height = rect->height;
int stride = surface->stride;
/* Get buffer from surface */
unsigned char* buffer = surface->buffer + rect->y * stride + rect->x * 4;
/* For each row */
for (y = 0; y < height; y++) {
uint32_t last_pixel;
uint32_t* row = (uint32_t*) buffer;
/* For each pixel in current row */
for (x = 0; x < width; x++) {
/* Get next pixel */
uint32_t current_pixel = *(row++) | 0xFF000000;
/* Update same/different counts according to pixel value */
if (x != 0) {
if (current_pixel == last_pixel)
num_same++;
else
num_different++;
}
last_pixel = current_pixel;
}
/* Advance to next row */
buffer += stride;
}
/* Return rough approximation of optimality for PNG compression */
return 0x100 * num_same / num_different - 0x400;
}
/**
* Returns whether the given rectangle would be optimally encoded as JPEG
* rather than PNG.
*
* @param surface
* The surface to be queried.
*
* @param rect
* The rectangle to check.
*
* @return
* Non-zero if the rectangle would be optimally encoded as JPEG, zero
* otherwise.
*/
static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface,
const guac_common_rect* rect) {
/* Calculate the average framerate for the given rect */
int framerate = __guac_common_surface_calculate_framerate(surface, rect);
/* JPEG is preferred if framerate is high enough */
return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE
&& __guac_common_surface_png_optimality(surface, rect) < 0;
}
/**
* Updates the heat map cells which intersect the given rectangle using the
* given timestamp. This timestamp, along with timestamps from past updates,
* is used to calculate the framerate of each heat cell.
*
* @param surface
* The surface containing the heat map cells to be updated.
*
* @param rect
* The rectangle containing the heat map cells to be updated.
*
* @param time
* The timestamp to use when updating the heat map cells which intersect
* the given rectangle.
*/
static void __guac_common_surface_touch_rect(guac_common_surface* surface,
guac_common_rect* rect, guac_timestamp time) {
int x, y;
/* Calculate minimum X/Y coordinates intersecting given rect */
int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
/* Calculate maximum X/Y coordinates intersecting given rect */
int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE;
/* Get start of buffer at given coordinates */
guac_common_surface_heat_cell* heat_row =
surface->heat_map + min_y * surface->width + min_x;
/* Update all heat map cells which intersect with rectangle */
for (y = min_y; y <= max_y; y++) {
/* Get current row of heat map */
guac_common_surface_heat_cell* heat_cell = heat_row;
/* For each cell in subset of row */
for (x = min_x; x <= max_x; x++) {
/* Replace oldest entry with new timestamp */
heat_cell->history[heat_cell->oldest_entry] = time;
/* Update to next oldest entry */
heat_cell->oldest_entry++;
if (heat_cell->oldest_entry >=
GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE)
heat_cell->oldest_entry = 0;
/* Advance to next heat map cell */
heat_cell++;
}
/* Next heat map row */
heat_row += surface->width;
}
}
/**
* Flushes the bitmap update currently described by the dirty rectangle within the
* given surface to that surface's bitmap queue. There MUST be space within the
* queue.
*
* @param surface The surface to flush.
*/
static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) {
guac_common_surface_png_rect* rect;
guac_common_surface_bitmap_rect* rect;
/* Do not flush if not dirty */
if (!surface->dirty)
return;
/* Add new rect to queue */
rect = &(surface->png_queue[surface->png_queue_length++]);
rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]);
rect->rect = surface->dirty_rect;
rect->flushed = 0;
@ -254,7 +498,7 @@ void guac_common_surface_flush_deferred(guac_common_surface* surface) {
/* Flush if queue size has reached maximum (space is reserved for the final dirty rect,
* as guac_common_surface_flush() MAY add an additional rect to the queue */
if (surface->png_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
if (surface->bitmap_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
guac_common_surface_flush(surface);
/* Append dirty rect to queue */
@ -665,19 +909,20 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client,
guac_socket* socket, const guac_layer* layer, int w, int h) {
/* Init surface */
guac_common_surface* surface = malloc(sizeof(guac_common_surface));
guac_common_surface* surface = calloc(1, sizeof(guac_common_surface));
surface->client = client;
surface->socket = socket;
surface->layer = layer;
surface->width = w;
surface->height = h;
surface->dirty = 0;
surface->png_queue_length = 0;
/* Create corresponding Cairo surface */
surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w);
surface->buffer = calloc(h, surface->stride);
/* Create corresponding heat map */
surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell));
/* Reset clipping rect */
guac_common_surface_reset_clip(surface);
@ -700,6 +945,7 @@ void guac_common_surface_free(guac_common_surface* surface) {
if (surface->realized)
guac_protocol_send_dispose(surface->socket, surface->layer);
free(surface->heat_map);
free(surface->buffer);
free(surface);
@ -736,6 +982,10 @@ void guac_common_surface_resize(guac_common_surface* surface, int w, int h) {
/* Free old data */
free(old_buffer);
/* Allocate completely new heat map (can safely discard old stats) */
free(surface->heat_map);
surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell));
/* Resize dirty rect to fit new surface dimensions */
if (surface->dirty) {
__guac_common_bound_rect(surface, &surface->dirty_rect, NULL, NULL);
@ -773,6 +1023,10 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_
if (rect.width <= 0 || rect.height <= 0)
return;
/* Update the heat map for the update rectangle. */
guac_timestamp time = guac_timestamp_current();
__guac_common_surface_touch_rect(surface, &rect, time);
/* Flush if not combining */
if (!__guac_common_should_combine(surface, &rect, 0))
guac_common_surface_flush_deferred(surface);
@ -948,11 +1202,13 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) {
}
/**
* Flushes the PNG update currently described by the dirty rectangle within the
* given surface directly to a "png" instruction, which is sent on the socket
* associated with the surface.
* Flushes the bitmap update currently described by the dirty rectangle within
* the given surface directly via an "img" instruction as PNG data. The
* resulting instructions will be sent over the socket associated with the
* given surface.
*
* @param surface The surface to flush.
* @param surface
* The surface to flush.
*/
static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
@ -982,15 +1238,52 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
}
/**
* Comparator for instances of guac_common_surface_png_rect, the elements
* which make up a surface's PNG buffer.
* Flushes the bitmap update currently described by the dirty rectangle within
* the given surface directly via an "img" instruction as JPEG data. The
* resulting instructions will be sent over the socket associated with the
* given surface.
*
* @param surface
* The surface to flush.
*/
static void __guac_common_surface_flush_to_jpeg(guac_common_surface* surface) {
if (surface->dirty) {
guac_socket* socket = surface->socket;
const guac_layer* layer = surface->layer;
/* Get Cairo surface for specified rect */
unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4;
cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24,
surface->dirty_rect.width,
surface->dirty_rect.height,
surface->stride);
/* Send JPEG for rect */
guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer,
surface->dirty_rect.x, surface->dirty_rect.y, rect,
GUAC_SURFACE_JPEG_IMAGE_QUALITY);
cairo_surface_destroy(rect);
surface->realized = 1;
/* Surface is no longer dirty */
surface->dirty = 0;
}
}
/**
* Comparator for instances of guac_common_surface_bitmap_rect, the elements
* which make up a surface's bitmap buffer.
*
* @see qsort
*/
static int __guac_common_surface_png_rect_compare(const void* a, const void* b) {
static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* b) {
guac_common_surface_png_rect* ra = (guac_common_surface_png_rect*) a;
guac_common_surface_png_rect* rb = (guac_common_surface_png_rect*) b;
guac_common_surface_bitmap_rect* ra = (guac_common_surface_bitmap_rect*) a;
guac_common_surface_bitmap_rect* rb = (guac_common_surface_bitmap_rect*) b;
/* Order roughly top to bottom, left to right */
if (ra->rect.y != rb->rect.y) return ra->rect.y - rb->rect.y;
@ -1006,31 +1299,31 @@ static int __guac_common_surface_png_rect_compare(const void* a, const void* b)
void guac_common_surface_flush(guac_common_surface* surface) {
guac_common_surface_png_rect* current = surface->png_queue;
/* Flush final dirty rectangle to queue. */
__guac_common_surface_flush_to_queue(surface);
guac_common_surface_bitmap_rect* current = surface->bitmap_queue;
int i, j;
int original_queue_length;
int flushed = 0;
/* Flush final dirty rect to queue */
__guac_common_surface_flush_to_queue(surface);
original_queue_length = surface->png_queue_length;
original_queue_length = surface->bitmap_queue_length;
/* Sort updates to make combination less costly */
qsort(surface->png_queue, surface->png_queue_length, sizeof(guac_common_surface_png_rect),
__guac_common_surface_png_rect_compare);
qsort(surface->bitmap_queue, surface->bitmap_queue_length, sizeof(guac_common_surface_bitmap_rect),
__guac_common_surface_bitmap_rect_compare);
/* Flush all rects in queue */
for (i=0; i < surface->png_queue_length; i++) {
for (i=0; i < surface->bitmap_queue_length; i++) {
/* Get next unflushed candidate */
guac_common_surface_png_rect* candidate = current;
guac_common_surface_bitmap_rect* candidate = current;
if (!candidate->flushed) {
int combined = 0;
/* Build up rect as much as possible */
for (j=i; j < surface->png_queue_length; j++) {
for (j=i; j < surface->bitmap_queue_length; j++) {
if (!candidate->flushed) {
@ -1054,13 +1347,23 @@ void guac_common_surface_flush(guac_common_surface* surface) {
/* Re-add to queue if there's room and this update was modified or we expect others might be */
if ((combined > 1 || i < original_queue_length)
&& surface->png_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
&& surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
__guac_common_surface_flush_to_queue(surface);
/* Flush as PNG otherwise */
else {
if (surface->dirty) flushed++;
/* Flush as bitmap otherwise */
else if (surface->dirty) {
flushed++;
/* Flush as JPEG if JPEG is preferred */
if (__guac_common_surface_should_use_jpeg(surface,
&surface->dirty_rect))
__guac_common_surface_flush_to_jpeg(surface);
/* Otherwise, use PNG */
else
__guac_common_surface_flush_to_png(surface);
}
}
@ -1070,7 +1373,7 @@ void guac_common_surface_flush(guac_common_surface* surface) {
}
/* Flush complete */
surface->png_queue_length = 0;
surface->bitmap_queue_length = 0;
}

60
src/common/guac_surface.h
View File

@ -33,15 +33,51 @@
#include <guacamole/socket.h>
/**
* The maximum number of updates to allow within the PNG queue.
* The maximum number of updates to allow within the bitmap queue.
*/
#define GUAC_COMMON_SURFACE_QUEUE_SIZE 256
/**
* Representation of a PNG update, having a rectangle of image data (stored
* Heat map cell size in pixels. Each side of each heat map cell will consist
* of this many pixels.
*/
#define GUAC_COMMON_SURFACE_HEAT_CELL_SIZE 64
/**
* The number of entries to collect within each heat map cell. Collected
* history entries are used to determine the framerate of the region associated
* with that cell.
*/
#define GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE 5
/**
* Representation of a cell in the refresh heat map. This cell is used to keep
* track of how often an area on a surface is refreshed.
*/
typedef struct guac_common_surface_heat_cell {
/**
* Timestamps of each of the last N updates covering the location
* associated with this heat map cell. This is used to calculate the
* framerate. This array is structured as a ring buffer containing history
* entries in chronologically-ascending order, starting at the entry
* pointed to by oldest_entry and proceeding through all other entries,
* wrapping around if the end of the array is reached.
*/
guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE];
/**
* Index of the oldest entry within the history.
*/
int oldest_entry;
} guac_common_surface_heat_cell;
/**
* Representation of a bitmap update, having a rectangle of image data (stored
* elsewhere) and a flushed/not-flushed state.
*/
typedef struct guac_common_surface_png_rect {
typedef struct guac_common_surface_bitmap_rect {
/**
* Whether this rectangle has been flushed.
@ -49,11 +85,11 @@ typedef struct guac_common_surface_png_rect {
int flushed;
/**
* The rectangle containing the PNG update.
* The rectangle containing the bitmap update.
*/
guac_common_rect rect;
} guac_common_surface_png_rect;
} guac_common_surface_bitmap_rect;
/**
* Surface which backs a Guacamole buffer or layer, automatically
@ -123,14 +159,20 @@ typedef struct guac_common_surface {
guac_common_rect clip_rect;
/**
* The number of updates in the PNG queue.
* The number of updates in the bitmap queue.
*/
int png_queue_length;
int bitmap_queue_length;
/**
* All queued PNG updates.
* All queued bitmap updates.
*/
guac_common_surface_png_rect png_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
/**
* A heat map keeping track of the refresh frequency of
* the areas of the screen.
*/
guac_common_surface_heat_cell* heat_map;
} guac_common_surface;