diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 8780c052..fce6902c 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -29,9 +29,11 @@ #include #include #include +#include #include #include +#include /** * The width of an update which should be considered negible and thus @@ -77,6 +79,18 @@ #define cairo_format_stride_for_width(format, width) (width*4) #endif +/** + * The JPEG image quality ('quantization') setting to use. Range 0-100 where + * 100 is the highest quality/largest file size, and 0 is the lowest + * quality/smallest file size. + */ +#define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90 + +/** + * The framerate which, if exceeded, indicates that JPEG is preferred. + */ +#define GUAC_COMMON_SURFACE_JPEG_FRAMERATE 3 + /** * Updates the coordinates of the given rectangle to be within the bounds of * the given surface. @@ -222,22 +236,252 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } /** - * Flushes the PNG update currently described by the dirty rectangle within the - * given surface to that surface's PNG queue. There MUST be space within the + * Calculate the current average framerate for a given area on the surface. + * + * @param surface + * The surface on which the framerate will be calculated. + * + * @param rect + * The rect containing the area for which the average framerate will be + * calculated. + * + * @return + * The average framerate of the given area, in frames per second. + */ +static unsigned int __guac_common_surface_calculate_framerate( + guac_common_surface* surface, const guac_common_rect* rect) { + + int x, y; + + /* Calculate minimum X/Y coordinates intersecting given rect */ + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + + /* Calculate maximum X/Y coordinates intersecting given rect */ + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + + unsigned int sum_framerate = 0; + unsigned int count = 0; + + /* Get start of buffer at given coordinates */ + const guac_common_surface_heat_cell* heat_row = + surface->heat_map + min_y * surface->width + min_x; + + /* Iterate over all the heat map cells for the area + * and calculate the average framerate */ + for (y = min_y; y < max_y; y++) { + + /* Get current row of heat map */ + const guac_common_surface_heat_cell* heat_cell = heat_row; + + /* For each cell in subset of row */ + for (x = min_x; x < max_x; x++) { + + /* Calculate indicies for latest and oldest history entries */ + int oldest_entry = heat_cell->oldest_entry; + int latest_entry = oldest_entry - 1; + if (latest_entry < 0) + latest_entry = GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE - 1; + + /* Calculate elapsed time covering entire history for this cell */ + int elapsed_time = heat_cell->history[latest_entry] + - heat_cell->history[oldest_entry]; + + /* Calculate and add framerate */ + if (elapsed_time) + sum_framerate += GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE + * 1000 / elapsed_time; + + /* Next heat map cell */ + heat_cell++; + count++; + + } + + /* Next heat map row */ + heat_row += surface->width; + + } + + /* Calculate the average framerate over entire rect */ + if (count) + return sum_framerate / count; + + return 0; + +} + + /** + * Guesses whether a rectangle within a particular surface would be better + * compressed as PNG or as JPEG. Positive values indicate PNG is likely to + * be superior, while negative values indicate JPEG. + * + * @param surface + * The surface containing the image data to check. + * + * @param rect + * The rect to check within the given surface. + * + * @return + * Positive values if PNG compression is likely to perform better than + * JPEG, or negative values if JPEG is likely to perform better than PNG. + */ +static int __guac_common_surface_png_optimality(guac_common_surface* surface, + const guac_common_rect* rect) { + + int x, y; + + int num_same = 0; + int num_different = 1; + + /* Get image/buffer metrics */ + int width = rect->width; + int height = rect->height; + int stride = surface->stride; + + /* Get buffer from surface */ + unsigned char* buffer = surface->buffer + rect->y * stride + rect->x * 4; + + /* For each row */ + for (y = 0; y < height; y++) { + + uint32_t last_pixel; + uint32_t* row = (uint32_t*) buffer; + + /* For each pixel in current row */ + for (x = 0; x < width; x++) { + + /* Get next pixel */ + uint32_t current_pixel = *(row++) | 0xFF000000; + + /* Update same/different counts according to pixel value */ + if (x != 0) { + if (current_pixel == last_pixel) + num_same++; + else + num_different++; + } + + last_pixel = current_pixel; + + } + + /* Advance to next row */ + buffer += stride; + + } + + /* Return rough approximation of optimality for PNG compression */ + return 0x100 * num_same / num_different - 0x400; + +} + +/** + * Returns whether the given rectangle would be optimally encoded as JPEG + * rather than PNG. + * + * @param surface + * The surface to be queried. + * + * @param rect + * The rectangle to check. + * + * @return + * Non-zero if the rectangle would be optimally encoded as JPEG, zero + * otherwise. + */ +static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface, + const guac_common_rect* rect) { + + /* Calculate the average framerate for the given rect */ + int framerate = __guac_common_surface_calculate_framerate(surface, rect); + + /* JPEG is preferred if framerate is high enough */ + return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE + && __guac_common_surface_png_optimality(surface, rect) < 0; + +} + +/** + * Updates the heat map cells which intersect the given rectangle using the + * given timestamp. This timestamp, along with timestamps from past updates, + * is used to calculate the framerate of each heat cell. + * + * @param surface + * The surface containing the heat map cells to be updated. + * + * @param rect + * The rectangle containing the heat map cells to be updated. + * + * @param time + * The timestamp to use when updating the heat map cells which intersect + * the given rectangle. + */ +static void __guac_common_surface_touch_rect(guac_common_surface* surface, + guac_common_rect* rect, guac_timestamp time) { + + int x, y; + + /* Calculate minimum X/Y coordinates intersecting given rect */ + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + + /* Calculate maximum X/Y coordinates intersecting given rect */ + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + + /* Get start of buffer at given coordinates */ + guac_common_surface_heat_cell* heat_row = + surface->heat_map + min_y * surface->width + min_x; + + /* Update all heat map cells which intersect with rectangle */ + for (y = min_y; y <= max_y; y++) { + + /* Get current row of heat map */ + guac_common_surface_heat_cell* heat_cell = heat_row; + + /* For each cell in subset of row */ + for (x = min_x; x <= max_x; x++) { + + /* Replace oldest entry with new timestamp */ + heat_cell->history[heat_cell->oldest_entry] = time; + + /* Update to next oldest entry */ + heat_cell->oldest_entry++; + if (heat_cell->oldest_entry >= + GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE) + heat_cell->oldest_entry = 0; + + /* Advance to next heat map cell */ + heat_cell++; + + } + + /* Next heat map row */ + heat_row += surface->width; + + } + +} + +/** + * Flushes the bitmap update currently described by the dirty rectangle within the + * given surface to that surface's bitmap queue. There MUST be space within the * queue. * * @param surface The surface to flush. */ static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) { - guac_common_surface_png_rect* rect; + guac_common_surface_bitmap_rect* rect; /* Do not flush if not dirty */ if (!surface->dirty) return; /* Add new rect to queue */ - rect = &(surface->png_queue[surface->png_queue_length++]); + rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); rect->rect = surface->dirty_rect; rect->flushed = 0; @@ -254,7 +498,7 @@ void guac_common_surface_flush_deferred(guac_common_surface* surface) { /* Flush if queue size has reached maximum (space is reserved for the final dirty rect, * as guac_common_surface_flush() MAY add an additional rect to the queue */ - if (surface->png_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1) + if (surface->bitmap_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1) guac_common_surface_flush(surface); /* Append dirty rect to queue */ @@ -665,19 +909,20 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, guac_socket* socket, const guac_layer* layer, int w, int h) { /* Init surface */ - guac_common_surface* surface = malloc(sizeof(guac_common_surface)); + guac_common_surface* surface = calloc(1, sizeof(guac_common_surface)); surface->client = client; surface->socket = socket; surface->layer = layer; surface->width = w; surface->height = h; - surface->dirty = 0; - surface->png_queue_length = 0; /* Create corresponding Cairo surface */ surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w); surface->buffer = calloc(h, surface->stride); + /* Create corresponding heat map */ + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell)); + /* Reset clipping rect */ guac_common_surface_reset_clip(surface); @@ -700,6 +945,7 @@ void guac_common_surface_free(guac_common_surface* surface) { if (surface->realized) guac_protocol_send_dispose(surface->socket, surface->layer); + free(surface->heat_map); free(surface->buffer); free(surface); @@ -736,6 +982,10 @@ void guac_common_surface_resize(guac_common_surface* surface, int w, int h) { /* Free old data */ free(old_buffer); + /* Allocate completely new heat map (can safely discard old stats) */ + free(surface->heat_map); + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell)); + /* Resize dirty rect to fit new surface dimensions */ if (surface->dirty) { __guac_common_bound_rect(surface, &surface->dirty_rect, NULL, NULL); @@ -773,6 +1023,10 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_ if (rect.width <= 0 || rect.height <= 0) return; + /* Update the heat map for the update rectangle. */ + guac_timestamp time = guac_timestamp_current(); + __guac_common_surface_touch_rect(surface, &rect, time); + /* Flush if not combining */ if (!__guac_common_should_combine(surface, &rect, 0)) guac_common_surface_flush_deferred(surface); @@ -948,11 +1202,13 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) { } /** - * Flushes the PNG update currently described by the dirty rectangle within the - * given surface directly to a "png" instruction, which is sent on the socket - * associated with the surface. + * Flushes the bitmap update currently described by the dirty rectangle within + * the given surface directly via an "img" instruction as PNG data. The + * resulting instructions will be sent over the socket associated with the + * given surface. * - * @param surface The surface to flush. + * @param surface + * The surface to flush. */ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { @@ -982,15 +1238,52 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { } /** - * Comparator for instances of guac_common_surface_png_rect, the elements - * which make up a surface's PNG buffer. + * Flushes the bitmap update currently described by the dirty rectangle within + * the given surface directly via an "img" instruction as JPEG data. The + * resulting instructions will be sent over the socket associated with the + * given surface. + * + * @param surface + * The surface to flush. + */ +static void __guac_common_surface_flush_to_jpeg(guac_common_surface* surface) { + + if (surface->dirty) { + + guac_socket* socket = surface->socket; + const guac_layer* layer = surface->layer; + + /* Get Cairo surface for specified rect */ + unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4; + cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, + surface->dirty_rect.width, + surface->dirty_rect.height, + surface->stride); + + /* Send JPEG for rect */ + guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer, + surface->dirty_rect.x, surface->dirty_rect.y, rect, + GUAC_SURFACE_JPEG_IMAGE_QUALITY); + cairo_surface_destroy(rect); + surface->realized = 1; + + /* Surface is no longer dirty */ + surface->dirty = 0; + + } + +} + +/** + * Comparator for instances of guac_common_surface_bitmap_rect, the elements + * which make up a surface's bitmap buffer. * * @see qsort */ -static int __guac_common_surface_png_rect_compare(const void* a, const void* b) { +static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* b) { - guac_common_surface_png_rect* ra = (guac_common_surface_png_rect*) a; - guac_common_surface_png_rect* rb = (guac_common_surface_png_rect*) b; + guac_common_surface_bitmap_rect* ra = (guac_common_surface_bitmap_rect*) a; + guac_common_surface_bitmap_rect* rb = (guac_common_surface_bitmap_rect*) b; /* Order roughly top to bottom, left to right */ if (ra->rect.y != rb->rect.y) return ra->rect.y - rb->rect.y; @@ -1006,31 +1299,31 @@ static int __guac_common_surface_png_rect_compare(const void* a, const void* b) void guac_common_surface_flush(guac_common_surface* surface) { - guac_common_surface_png_rect* current = surface->png_queue; + /* Flush final dirty rectangle to queue. */ + __guac_common_surface_flush_to_queue(surface); + guac_common_surface_bitmap_rect* current = surface->bitmap_queue; int i, j; int original_queue_length; int flushed = 0; - /* Flush final dirty rect to queue */ - __guac_common_surface_flush_to_queue(surface); - original_queue_length = surface->png_queue_length; + original_queue_length = surface->bitmap_queue_length; /* Sort updates to make combination less costly */ - qsort(surface->png_queue, surface->png_queue_length, sizeof(guac_common_surface_png_rect), - __guac_common_surface_png_rect_compare); + qsort(surface->bitmap_queue, surface->bitmap_queue_length, sizeof(guac_common_surface_bitmap_rect), + __guac_common_surface_bitmap_rect_compare); /* Flush all rects in queue */ - for (i=0; i < surface->png_queue_length; i++) { + for (i=0; i < surface->bitmap_queue_length; i++) { /* Get next unflushed candidate */ - guac_common_surface_png_rect* candidate = current; + guac_common_surface_bitmap_rect* candidate = current; if (!candidate->flushed) { int combined = 0; /* Build up rect as much as possible */ - for (j=i; j < surface->png_queue_length; j++) { + for (j=i; j < surface->bitmap_queue_length; j++) { if (!candidate->flushed) { @@ -1054,13 +1347,23 @@ void guac_common_surface_flush(guac_common_surface* surface) { /* Re-add to queue if there's room and this update was modified or we expect others might be */ if ((combined > 1 || i < original_queue_length) - && surface->png_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE) + && surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE) __guac_common_surface_flush_to_queue(surface); - /* Flush as PNG otherwise */ - else { - if (surface->dirty) flushed++; - __guac_common_surface_flush_to_png(surface); + /* Flush as bitmap otherwise */ + else if (surface->dirty) { + + flushed++; + + /* Flush as JPEG if JPEG is preferred */ + if (__guac_common_surface_should_use_jpeg(surface, + &surface->dirty_rect)) + __guac_common_surface_flush_to_jpeg(surface); + + /* Otherwise, use PNG */ + else + __guac_common_surface_flush_to_png(surface); + } } @@ -1070,7 +1373,7 @@ void guac_common_surface_flush(guac_common_surface* surface) { } /* Flush complete */ - surface->png_queue_length = 0; + surface->bitmap_queue_length = 0; } diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index cacfc2d9..ad99af0e 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -33,15 +33,51 @@ #include /** - * The maximum number of updates to allow within the PNG queue. + * The maximum number of updates to allow within the bitmap queue. */ #define GUAC_COMMON_SURFACE_QUEUE_SIZE 256 /** - * Representation of a PNG update, having a rectangle of image data (stored + * Heat map cell size in pixels. Each side of each heat map cell will consist + * of this many pixels. + */ +#define GUAC_COMMON_SURFACE_HEAT_CELL_SIZE 64 + +/** + * The number of entries to collect within each heat map cell. Collected + * history entries are used to determine the framerate of the region associated + * with that cell. + */ +#define GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE 5 + +/** + * Representation of a cell in the refresh heat map. This cell is used to keep + * track of how often an area on a surface is refreshed. + */ +typedef struct guac_common_surface_heat_cell { + + /** + * Timestamps of each of the last N updates covering the location + * associated with this heat map cell. This is used to calculate the + * framerate. This array is structured as a ring buffer containing history + * entries in chronologically-ascending order, starting at the entry + * pointed to by oldest_entry and proceeding through all other entries, + * wrapping around if the end of the array is reached. + */ + guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE]; + + /** + * Index of the oldest entry within the history. + */ + int oldest_entry; + +} guac_common_surface_heat_cell; + +/** + * Representation of a bitmap update, having a rectangle of image data (stored * elsewhere) and a flushed/not-flushed state. */ -typedef struct guac_common_surface_png_rect { +typedef struct guac_common_surface_bitmap_rect { /** * Whether this rectangle has been flushed. @@ -49,11 +85,11 @@ typedef struct guac_common_surface_png_rect { int flushed; /** - * The rectangle containing the PNG update. + * The rectangle containing the bitmap update. */ guac_common_rect rect; -} guac_common_surface_png_rect; +} guac_common_surface_bitmap_rect; /** * Surface which backs a Guacamole buffer or layer, automatically @@ -123,14 +159,20 @@ typedef struct guac_common_surface { guac_common_rect clip_rect; /** - * The number of updates in the PNG queue. + * The number of updates in the bitmap queue. */ - int png_queue_length; + int bitmap_queue_length; /** - * All queued PNG updates. + * All queued bitmap updates. */ - guac_common_surface_png_rect png_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE]; + guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE]; + + /** + * A heat map keeping track of the refresh frequency of + * the areas of the screen. + */ + guac_common_surface_heat_cell* heat_map; } guac_common_surface; diff --git a/src/protocols/vnc/client.h b/src/protocols/vnc/client.h index ccbac69e..428ffff9 100644 --- a/src/protocols/vnc/client.h +++ b/src/protocols/vnc/client.h @@ -53,7 +53,7 @@ * milliseconds. If the server is silent for at least this amount of time, the * frame will be considered finished. */ -#define GUAC_VNC_FRAME_TIMEOUT 0 +#define GUAC_VNC_FRAME_TIMEOUT 10 /** * The number of milliseconds to wait between connection attempts.